func processMultipleCursor(
cursor *gorethink.Cursor,
messageConstructor func() proto.Message,
) ([]interface{}, error) {
var data []string
if err := cursor.All(&data); err != nil {
return nil, err
}
result := make([]interface{}, len(data))
for i, datum := range data {
message := messageConstructor()
if err := jsonpb.UnmarshalString(datum, message); err != nil {
return nil, err
}
result[i] = message
}
return result, nil
}
func packageCursorToUrl(c *r.Cursor) (UrlMap, error) {
rows := []UrlMap{}
err := c.All(&rows)
return rows[0], err
}
func (bmo *BMO) Compute(input *os.File) {
var err error
var cur *r.Cursor
var session *r.Session
// set up database connection pool
session, err = r.Connect(r.ConnectOpts{
Addresses: bmo.nodes,
Database: bmo.database,
DiscoverHosts: true,
})
session.SetMaxOpenConns(POOL_SIZE)
if err != nil {
log.Fatalln(err)
}
// ensure table is present
var tableNames []string
cur, err = r.DB(bmo.database).TableList().Run(session)
if err != nil {
log.Fatalln(err)
}
cur.All(&tableNames)
set := make(map[string]bool)
for _, v := range tableNames {
set[v] = true
}
if !set[bmo.table] {
log.Println("Creating table ", bmo.table)
_, err = r.DB(bmo.database).TableCreate(bmo.table).RunWrite(session)
if err != nil {
log.Fatalln("Error creating table: ", err)
os.Exit(1)
}
}
// deliver the messages
decoder := json.NewDecoder(input)
ms := make([]Message, INSERT_BATCH_SIZE)
var m *Message
var i uint64
var ignoreLast bool
pool, _ := tunny.CreatePoolGeneric(POOL_SIZE).Open()
defer pool.Close()
table := r.Table(bmo.table)
insertOptions := r.InsertOpts{Durability: "soft"}
insert := func() {
j := i
if !ignoreLast {
j += 1
}
_, err = table.Insert(ms[:j], insertOptions).RunWrite(session)
if err != nil {
log.Fatal(err)
os.Exit(1)
}
}
for {
i = bmo.seq % INSERT_BATCH_SIZE
m = &ms[i]
err = decoder.Decode(&m)
switch {
case err == io.EOF:
ignoreLast = true
pool.SendWork(insert)
return
case err != nil:
ignoreLast = true
pool.SendWork(insert)
log.Fatal("Can't parse json input, \"", err, "\". Object #", bmo.seq, ", after ", m)
os.Exit(1)
default:
if i+1 == INSERT_BATCH_SIZE {
ignoreLast = false
pool.SendWork(insert)
}
}
bmo.seq += 1
}
}