func (db *DBStore) readBytes(name string) (*bytes.Buffer, error) {
if db.env == nil {
return nil, ErrDBStoreNotInitialized
}
var b *bytes.Buffer
err := db.env.View(func(txn *lmdb.Txn) error {
dbi, err := txn.OpenRoot(0)
if err != nil {
return err
}
data, err := txn.Get(dbi, []byte(name))
if err != nil {
return err
}
b = bytes.NewBuffer(data)
return nil
})
if err != nil {
// conform to semantics of store read operations and return
// os.ErrNotExist if the entry was not found
if lmdb.IsNotFound(err) {
return nil, os.ErrNotExist
}
return nil, errors.Wrapf(err, "failed to read data for key %s", name)
}
return b, nil
}
// Contains ???
// BUG(bmatsuo):
// Contains is surely broken. It just looks for v in the given database. It
// seems like it should buffer the output, or run the iteration again.
func (it *AllIterator) Contains(v graph.Value) (ok bool) {
graph.ContainsLogIn(it, v)
defer func() { graph.ContainsLogOut(it, v, ok) }()
tok, ok := v.(*Token)
if !ok {
return false
}
if tok.db != it.db {
return false
}
err := it.qs.env.View(func(tx *lmdb.Txn) (err error) {
tx.RawRead = true
_, err = tx.Get(it.qs.dbis[tok.db], tok.key)
ok = !lmdb.IsNotFound(err)
return err
})
if err != nil {
return false
}
if ok {
it.result = tok
}
return ok
}
func (c *cmdRead) run() {
// create environment and dbi.
numDB := 10
c.sizeDB = 1 << 30
var err error
c.env, err = createEnv(c.dbfile, numDB, c.sizeDB)
for lmdb.IsMapFull(err) {
c.sizeDB *= 2
c.env, err = createEnv(c.dbfile, numDB, c.sizeDB)
}
raiseError(err)
defer c.env.Close()
var sizeDB int64 = 1 << 30
c.featureEnv, err = createNoLockEnv(c.featureDB, numDB, sizeDB)
for lmdb.IsMapFull(err) {
sizeDB *= 2
c.featureEnv, err = createNoLockEnv(c.featureDB, numDB, sizeDB)
}
raiseError(err)
defer c.featureEnv.Close()
createDBI(c.env, "gene")
// open pileup file and read SNP.
snpChan := readPileup(c.pileupFile)
// group SNPs into genes.
geneChan := c.groupSNPs(snpChan)
c.loadGenes(geneChan)
err = c.env.View(func(tx *lmdb.Txn) error {
dbi, err := tx.OpenDBI("gene", 0)
if err != nil {
return err
}
cur, err := tx.OpenCursor(dbi)
if err != nil {
return err
}
count := 0
for {
_, _, err := cur.Get(nil, nil, lmdb.Next)
if lmdb.IsNotFound(err) {
return nil
}
if err != nil {
return err
}
count++
}
log.Printf("Total gene: %d\n", count)
return nil
})
if err != nil {
log.Panicln(err)
}
}
// Get a K/V pair
func (m *MDBStore) Get(key []byte) ([]byte, error) {
var val []byte
err := m.env.View(func(txn *lmdb.Txn) (err error) {
val, err = txn.Get(m.dbConf, key)
if lmdb.IsNotFound(err) {
return fmt.Errorf("not found")
}
return err
})
return val, err
}
func (m *MDBStore) GetUint64(key []byte) (v64 uint64, err error) {
err = m.env.View(func(txn *lmdb.Txn) (err error) {
txn.RawRead = true
val, err := txn.Get(m.dbConf, key)
if lmdb.IsNotFound(err) {
return fmt.Errorf("not found")
}
if err == nil {
v64 = bytesToUint64(val)
}
return err
})
return v64, err
}
func (qs *QuadStore) getInt64ForMetaKey(tx *lmdb.Txn, key string, empty int64) (int64, error) {
var out int64
data, err := tx.Get(qs.metaDBI, []byte(key))
if lmdb.IsNotFound(err) {
return empty, nil
}
if err != nil {
return 0, err
}
buf := bytes.NewBuffer(data)
err = binary.Read(buf, binary.LittleEndian, &out)
if err != nil {
return 0, err
}
return out, nil
}
// Gets a log entry at a given index
func (m *MDBStore) GetLog(index uint64, logOut *raft.Log) error {
key := uint64ToBytes(index)
return m.env.View(func(txn *lmdb.Txn) error {
txn.RawRead = true
val, err := txn.Get(m.dbLogs, key)
if lmdb.IsNotFound(err) {
return raft.ErrLogNotFound
} else if err != nil {
return err
}
// Convert the value to a log
return decodeMsgPack(val, logOut)
})
}
// Iterate all cr results,
// output a channel of CovRes.
func (c *cmdReport) getCr() chan CovRes {
ch := make(chan CovRes)
fn := func(txn *lmdb.Txn) error {
dbi, err := txn.OpenDBI("cr", 0)
if err != nil {
return err
}
cur, err := txn.OpenCursor(dbi)
if err != nil {
return err
}
defer cur.Close()
for {
k, v, err := cur.Get(nil, nil, lmdb.Next)
if lmdb.IsNotFound(err) {
break
}
if err != nil {
return err
}
values := []float64{}
if err := msgpack.Unmarshal(v, &values); err != nil {
return err
}
ch <- CovRes{Key: k, Values: values}
}
return nil
}
go func() {
defer close(ch)
err := c.resultsDB.View(fn)
if err != nil {
log.Panicln(err)
}
}()
return ch
}
func getAllCr(env *lmdb.Env, dbname string) chan KeyValue {
ch := make(chan KeyValue)
go func() {
defer close(ch)
fn := func(txn *lmdb.Txn) error {
dbi, err := txn.OpenDBI(dbname, 0)
if err != nil {
return err
}
cur, err := txn.OpenCursor(dbi)
if err != nil {
return err
}
for {
k, v, err := cur.Get(nil, nil, lmdb.Next)
if lmdb.IsNotFound(err) {
break
}
if err != nil {
return err
}
ch <- KeyValue{Key: k, Value: v}
}
return nil
}
err := env.View(fn)
if err != nil {
if *debug {
log.Panicln(err)
} else {
log.Fatalln(err)
}
}
}()
return ch
}
func (m *MDBStore) getIndex(k, v []byte, op uint) (uint64, error) {
var k64 uint64
err := m.env.View(func(txn *lmdb.Txn) error {
txn.RawRead = true
cursor, err := txn.OpenCursor(m.dbLogs)
if err != nil {
return err
}
k, _, err = cursor.Get(nil, nil, op)
cursor.Close()
if lmdb.IsNotFound(err) {
return nil
} else if err == nil {
k64 = bytesToUint64(k)
}
return err
})
return k64, err
}
func getAllSNPs(env *lmdb.Env) chan SNPArr {
ch := make(chan SNPArr)
fn := func(tx *lmdb.Txn) error {
dbi, err := tx.OpenDBI("gene", 0)
if err != nil {
return err
}
cur, err := tx.OpenCursor(dbi)
if err != nil {
return err
}
defer cur.Close()
for {
k, v, err := cur.Get(nil, nil, lmdb.Next)
if lmdb.IsNotFound(err) {
return nil
} else if err != nil {
return err
}
arr := []pileup.SNP{}
if err := msgpack.Unmarshal(v, &arr); err != nil {
return err
}
ch <- SNPArr{Key: k, Arr: arr}
}
}
go func() {
defer close(ch)
env.View(fn)
}()
return ch
}
func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) {
env, err := createLMDB(path, options)
if err != nil {
glog.Errorln("Error, couldn't open! ", err)
return nil, err
}
var qs QuadStore
qs.env = env
err = qs.openDBIs()
if lmdb.IsNotFound(err) {
return nil, errors.New("lmdb: quadstore has not been initialised")
}
err = qs.getMetadata()
if err != nil {
return nil, err
}
if qs.version != latestDataVersion {
return nil, errors.New("lmdb: data version is out of date. Run cayleyupgrade for your config to update the data")
}
return &qs, nil
}
func (c *cmdRead) queryGenome(reference string) *Genome {
features := []*Feature{}
fn := func(tx *lmdb.Txn) error {
// first, we query genome bucket to
// obtain a list of gene IDs.
dbi, err := tx.OpenDBI("genome", 0)
if err != nil {
return err
}
v, err := tx.Get(dbi, []byte(reference))
if err != nil {
return err
}
// unpack gene IDs.
geneIDs := []string{}
err = msgpack.Unmarshal(v, &geneIDs)
if err != nil {
log.Panicln(err)
}
// now, we try obtain gene feature informations
// from feature bucket.
dbi, err = tx.OpenDBI("feature", 0)
for _, id := range geneIDs {
if len(id) == 0 {
continue
}
v, err := tx.Get(dbi, []byte(id))
if err != nil {
return err
}
if len(v) > 0 {
f := Feature{}
if err := msgpack.Unmarshal(v, &f); err != nil {
log.Panicln(err)
}
if f.Type == "CDS" || strings.Contains(f.Type, "RNA") {
features = append(features, &f)
}
}
}
return nil
}
err := c.featureEnv.View(fn)
if err != nil {
if lmdb.IsNotFound(err) {
if *debug {
log.Printf("key %s is not found\n", reference)
}
return nil
}
log.Panicln(err)
}
// sort features.
if len(features) > 0 {
sort.Sort(ByEnd{features})
}
return &Genome{Reference: reference, Features: features}
}
func (d *DiversityFilter) filter(buf []*sam.Record, acc string, genome []byte) (out []*sam.Record, acc1 string, genome1 []byte) {
fn := func(txn *lmdb.Txn) error {
dbi, err := txn.OpenDBI("read", 0)
if err != nil {
return err
}
for _, r := range buf {
key := []byte(r.Name)
val, err := txn.Get(dbi, key)
if err != nil {
if lmdb.IsNotFound(err) {
val, err = r.MarshalText()
if err != nil {
return err
}
err = txn.Put(dbi, key, val, 0)
if err != nil {
return err
}
} else {
return err
}
} else {
var mate *sam.Record = &sam.Record{}
err := mate.UnmarshalText(val)
raiseError(err)
if r.Ref.Name() == mate.Ref.Name() {
if acc != r.Ref.Name() {
genome, err = d.findGenome(r, d.featureDB, "fna")
raiseError(err)
acc = r.Ref.Name()
if *debug {
log.Println(acc)
}
}
diff1, len1 := d.Diff(r, genome)
diff2, len2 := d.Diff(mate, genome)
if len1 > 0 && len2 > 0 && float64(diff1+diff2)/float64(len1+len2) <= d.Cutoff {
out = append(out, r)
out = append(out, mate)
} else {
if *debug {
log.Printf("%d, %d, %d, %d\n", diff1, diff2, len1, len2)
}
}
}
txn.Del(dbi, key, val)
}
}
return nil
}
retry:
err := d.db.Update(fn)
if lmdb.IsMapFull(err) {
d.sizeDB *= 2
err = d.db.SetMapSize(d.sizeDB)
raiseError(err)
goto retry
}
raiseError(err)
genome1 = genome
acc1 = acc
return
}