func (mgr *QueueManager) heartbeatReservations() {
batch := gocql.NewBatch(gocql.UnloggedBatch)
mgr.mu.RLock()
for queueID := range mgr.writers {
batch.Query(`UPDATE queue_managers USING TTL ? SET manager_id = ? WHERE queue_id = ?`,
mgr.config.reservationTTL(), mgr.Name, queueID)
}
mgr.mu.RUnlock()
// TODO: Panic hard if an error occurs
mgr.db.ExecuteBatch(batch)
}
func insertBatch() error {
batch := gocql.NewBatch(gocql.LoggedBatch)
for _, page := range pageTestData {
batch.Query(`INSERT INTO page
(title, revid, body, views, protected, modified, tags, attachments)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
page.Title, page.RevId, page.Body, page.Views, page.Protected,
page.Modified, page.Tags, page.Attachments)
}
if err := session.ExecuteBatch(batch); err != nil {
return err
}
return nil
}
func (writer *queueWriter) writeBatch(db *gocql.Session, requests []*batchRequest) {
dbBatch := gocql.NewBatch(gocql.UnloggedBatch)
i := int64(0)
for _, request := range requests {
for _, item := range request.items {
itemID := writer.nextIndex + i
dbBatch.Query(`INSERT INTO queue_items (queue_id, item_id, item_value) VALUES (?, ?, ?)`, writer.id, itemID, item)
i++
}
}
err := db.ExecuteBatch(dbBatch)
writer.respond(requests, err)
}
func main() {
var in *os.File
var err error
var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
var event_file = flag.String("event_file", "", "event file")
flag.Parse()
if *event_file == "" {
panic("must pass an event file")
}
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
panic(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
init_globals()
fmt.Printf("Opening event file %s....\n", *event_file)
if in, err = os.Open(*event_file); err != nil {
panic(err)
}
// connect to the cluster
cluster := gocql.NewCluster("10.168.16.195", "10.168.16.196")
cluster.Keyspace = "logs_ks"
cluster.Consistency = gocql.Quorum
session, _ := cluster.CreateSession()
defer session.Close()
scanner := bufio.NewScanner(in)
line := ""
var parts []string
line_num := 0
var batch *gocql.Batch
rowdata := make([]interface{}, len(columns))
for scanner.Scan() {
line = scanner.Text()
line_num++
parts = strings.SplitN(line, "\t", 7)
if len(parts) != 7 {
log.Printf("Bad line %d in event file: %s\n", line_num, *event_file)
continue
}
if batch == nil {
batch = gocql.NewBatch(gocql.LoggedBatch)
}
col_idx := 0
var val_fl float64
var val_bl bool
var val_int int
var ts time.Time
for _, k := range columns {
switch logfile_spec[k].data_type {
case "integer":
val_int, err = strconv.Atoi(parts[logfile_spec[k].indx])
if err == nil {
val_int = 0
}
rowdata[col_idx] = val_int
case "float":
val_fl, err = strconv.ParseFloat(parts[logfile_spec[k].indx], 32)
if err == nil {
val_fl = 0.0
}
rowdata[col_idx] = float32(val_fl)
case "boolean":
val_bl, err = strconv.ParseBool(parts[logfile_spec[k].indx])
if err == nil {
val_bl = false
}
rowdata[col_idx] = val_bl
case "timestamp":
// Reference MAGIC time: Mon Jan 2 15:04:05 -0700 MST 2006
ts, err = time.Parse("20060102150405", parts[logfile_spec[k].indx])
if err != nil {
log.Printf("Bad time in line %d in event file: %s\n", line_num, *event_file)
continue // can't make fake timestamp
}
rowdata[col_idx] = ts
default: // strings varchar / ascii etc
rowdata[col_idx] = parts[logfile_spec[k].indx]
}
col_idx += 1
}
//log.Printf("parts = %v", parts)
//log.Printf("rowdata = %v", rowdata)
batch.Query(insert_query1, rowdata...)
batch_count := len(batch.Entries)
if batch_count >= 2000 {
if err := session.ExecuteBatch(batch); err != nil {
log.Printf("Failed to execute batch: %v", err)
}
batch = nil
log.Printf("Done with batch of %d", batch_count)
}
}
if batch != nil && len(batch.Entries) > 0 {
batch_count := len(batch.Entries)
if err := session.ExecuteBatch(batch); err != nil {
log.Printf("Failed to execute batch: %v", err)
}
batch = nil
log.Printf("Done with batch of %d", batch_count)
}
in.Close()
}
