Beispiel #1
0
func (mgr *QueueManager) heartbeatReservations() {
	batch := gocql.NewBatch(gocql.UnloggedBatch)

	mgr.mu.RLock()
	for queueID := range mgr.writers {
		batch.Query(`UPDATE queue_managers USING TTL ? SET manager_id = ? WHERE queue_id = ?`,
			mgr.config.reservationTTL(), mgr.Name, queueID)
	}
	mgr.mu.RUnlock()

	// TODO: Panic hard if an error occurs
	mgr.db.ExecuteBatch(batch)
}
Beispiel #2
0
func insertBatch() error {
	batch := gocql.NewBatch(gocql.LoggedBatch)
	for _, page := range pageTestData {
		batch.Query(`INSERT INTO page
			(title, revid, body, views, protected, modified, tags, attachments)
			VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
			page.Title, page.RevId, page.Body, page.Views, page.Protected,
			page.Modified, page.Tags, page.Attachments)
	}
	if err := session.ExecuteBatch(batch); err != nil {
		return err
	}
	return nil
}
Beispiel #3
0
func (writer *queueWriter) writeBatch(db *gocql.Session, requests []*batchRequest) {
	dbBatch := gocql.NewBatch(gocql.UnloggedBatch)
	i := int64(0)

	for _, request := range requests {
		for _, item := range request.items {
			itemID := writer.nextIndex + i
			dbBatch.Query(`INSERT INTO queue_items (queue_id, item_id, item_value) VALUES (?, ?, ?)`, writer.id, itemID, item)
			i++
		}
	}

	err := db.ExecuteBatch(dbBatch)
	writer.respond(requests, err)
}
Beispiel #4
0
func main() {

	var in *os.File
	var err error

	var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
	var event_file = flag.String("event_file", "", "event file")

	flag.Parse()
	if *event_file == "" {
		panic("must pass an event file")
	}

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			panic(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	init_globals()

	fmt.Printf("Opening event file %s....\n", *event_file)
	if in, err = os.Open(*event_file); err != nil {
		panic(err)
	}

	// connect to the cluster
	cluster := gocql.NewCluster("10.168.16.195", "10.168.16.196")
	cluster.Keyspace = "logs_ks"
	cluster.Consistency = gocql.Quorum
	session, _ := cluster.CreateSession()
	defer session.Close()

	scanner := bufio.NewScanner(in)
	line := ""
	var parts []string

	line_num := 0
	var batch *gocql.Batch
	rowdata := make([]interface{}, len(columns))

	for scanner.Scan() {
		line = scanner.Text()
		line_num++
		parts = strings.SplitN(line, "\t", 7)
		if len(parts) != 7 {
			log.Printf("Bad line %d in event file: %s\n", line_num, *event_file)
			continue
		}

		if batch == nil {
			batch = gocql.NewBatch(gocql.LoggedBatch)
		}

		col_idx := 0
		var val_fl float64
		var val_bl bool
		var val_int int
		var ts time.Time

		for _, k := range columns {
			switch logfile_spec[k].data_type {
			case "integer":
				val_int, err = strconv.Atoi(parts[logfile_spec[k].indx])
				if err == nil {
					val_int = 0
				}
				rowdata[col_idx] = val_int
			case "float":
				val_fl, err = strconv.ParseFloat(parts[logfile_spec[k].indx], 32)
				if err == nil {
					val_fl = 0.0
				}
				rowdata[col_idx] = float32(val_fl)
			case "boolean":
				val_bl, err = strconv.ParseBool(parts[logfile_spec[k].indx])
				if err == nil {
					val_bl = false
				}
				rowdata[col_idx] = val_bl
			case "timestamp":
				// Reference MAGIC time: Mon Jan 2 15:04:05 -0700 MST 2006
				ts, err = time.Parse("20060102150405", parts[logfile_spec[k].indx])
				if err != nil {
					log.Printf("Bad time in line %d in event file: %s\n", line_num, *event_file)
					continue // can't make fake timestamp
				}
				rowdata[col_idx] = ts
			default: // strings varchar / ascii etc
				rowdata[col_idx] = parts[logfile_spec[k].indx]
			}
			col_idx += 1
		}
		//log.Printf("parts = %v", parts)
		//log.Printf("rowdata = %v", rowdata)

		batch.Query(insert_query1, rowdata...)

		batch_count := len(batch.Entries)
		if batch_count >= 2000 {
			if err := session.ExecuteBatch(batch); err != nil {
				log.Printf("Failed to execute batch: %v", err)
			}
			batch = nil
			log.Printf("Done with batch of %d", batch_count)
		}
	}
	if batch != nil && len(batch.Entries) > 0 {
		batch_count := len(batch.Entries)
		if err := session.ExecuteBatch(batch); err != nil {
			log.Printf("Failed to execute batch: %v", err)
		}
		batch = nil
		log.Printf("Done with batch of %d", batch_count)
	}
	in.Close()

}