Beispiel #1
0
func main() {

	session := integration.TestSession("127.0.0.1", "cqlc")
	integration.Truncate(session, REVERSE_TIMESERIES)

	result := "FAILED"

	ctx := cqlc.NewContext()
	batch := gocql.NewBatch(gocql.LoggedBatch)

	events := 100

	for i := 0; i < events; i++ {

		unix := time.Now().Unix()
		t := time.Unix(unix+int64(i), 0)

		ctx.Upsert(REVERSE_TIMESERIES).
			SetString(REVERSE_TIMESERIES.EVENT_TYPE, "x").
			SetTimestamp(REVERSE_TIMESERIES.INSERTION_TIME, t).
			SetBytes(REVERSE_TIMESERIES.EVENT, []byte("neb")).
			Batch(batch)
	}

	if err := session.ExecuteBatch(batch); err != nil {
		log.Fatalf("Could not execute batch: %v", err)
		os.Exit(1)
	}

	iter, err := ctx.Select().From(REVERSE_TIMESERIES).Fetch(session)
	if err != nil {
		log.Fatalf("Could not read REVERSE_TIMESERIES: %v", err)
		os.Exit(1)
	}

	var previous time.Time

	err = MapReverseTimeseries(iter, func(e ReverseTimeseries) (bool, error) {

		current := e.InsertionTime

		if !previous.IsZero() {
			if current.After(previous) {
				return false, fmt.Errorf("Wrong ordering (DESC): previous was %v but current is %v", previous, current)
			}
		}

		previous = current
		return true, nil
	})

	if err != nil {
		log.Fatalf("Unexpected order of REVERSE_TIMESERIES: %v", err)
		os.Exit(1)
	}

	result = "PASSED"

	os.Stdout.WriteString(result)
}
//StoreBatchEvents stores multiple events from models.EventTrackingPayload within a batch
func (d *StorageDatastore) StoreBatchEvents(p *models.EventTrackingPayload) error {
	batch := gocql.NewBatch(gocql.LoggedBatch)

	statement := `INSERT INTO tracking.events (id, client, name, date, properties) VALUES (?, ?, ?, ?, ?)`

	for _, e := range p.Events {
		//Generate a new UUID
		u4, err := gocql.RandomUUID()
		if err != nil {
			return err
		}
		//map properties
		var propMap map[string]string
		propMap = make(map[string]string)
		for _, property := range e.Properties {
			propMap[property.Name] = property.Value
		}
		//Add to batch
		batch.Query(statement, u4.String(), p.Token, e.Name, e.Date, propMap)
	}

	err := d.Session.ExecuteBatch(batch)
	if err != nil {
		return err
	}

	return nil
}
Beispiel #3
0
func main() {

	session := integration.TestSession("127.0.0.1", "cqlc")
	session.SetPageSize(1000)
	integration.Truncate(session, EVENTS)

	result := "FAILED"

	ctx := cqlc.NewContext()
	batch := gocql.NewBatch(gocql.LoggedBatch)

	rounds := 10
	distinct := 10

	for i := 0; i < rounds; i++ {
		for j := 0; j < distinct; j++ {
			ctx.Upsert(EVENTS).
				SetInt64(EVENTS.SENSOR, int64(j)).
				SetTimeUUID(EVENTS.TIMESTAMP, gocql.TimeUUID()).
				SetFloat32(EVENTS.TEMPERATURE, 19.8).
				SetInt32(EVENTS.PRESSURE, 357).
				Batch(batch)
		}
	}

	err := session.ExecuteBatch(batch)

	if err != nil {
		log.Fatalf("Could not execute batch: %v", err)
		os.Exit(1)
	}

	iter, err := ctx.SelectDistinct(EVENTS.SENSOR).From(EVENTS).Fetch(session)
	if err != nil {
		log.Fatalf("Could not prepare query: %v", err)
		os.Exit(1)
	}

	count := 0

	MapEvents(iter, func(e Events) (bool, error) {
		count++
		return true, nil
	})

	if err := iter.Close(); err != nil {
		log.Fatalf("Could not close iterator: %v", err)
		os.Exit(1)
	}

	if count == distinct {
		result = "PASSED"
	} else {
		result = fmt.Sprintf("Expected %d distinct rows; got %d", distinct, count)
	}

	os.Stdout.WriteString(result)
}
Beispiel #4
0
func main() {

	session := integration.TestSession("127.0.0.1", "cqlc")
	integration.Truncate(session, BASIC_CLUSTERED)

	result := "FAILED"

	rows := 10

	ctx := cqlc.NewContext()
	batch := gocql.NewBatch(gocql.LoggedBatch)

	for i := 0; i < rows; i++ {
		err := ctx.Upsert(BASIC_CLUSTERED).
			SetInt64(BASIC_CLUSTERED.ID, int64(0)).
			SetInt32(BASIC_CLUSTERED.INT32_CLUSTER, int32(i)).
			SetInt32(BASIC_CLUSTERED.INT32_COLUMN, int32(i)).
			Batch(batch)

		if err != nil {
			log.Fatalf("Could not batch upsert: %v", err)
			os.Exit(1)
		}
	}

	err := session.ExecuteBatch(batch)

	if err != nil {
		log.Fatalf("Could not execute batch: %v", err)
		os.Exit(1)
	}

	iter, err := ctx.Select().From(BASIC_CLUSTERED).Where(BASIC_CLUSTERED.ID.Eq(0)).Fetch(session)

	basics, err := BindBasicClustered(iter)
	if err != nil {
		log.Fatalf("Could not bind data: %v", err)
		os.Exit(1)
	}

	err = iter.Close()
	if err != nil {
		log.Fatalf("Could not bind data: %v", err)
		os.Exit(1)
	}

	if len(basics) == rows {
		result = "PASSED"
	}

	os.Stdout.WriteString(result)
}
Beispiel #5
0
func storeUsers(conf *Config, users chan models.User) {
	// connect to the cluster
	cluster := gocql.NewCluster(conf.CassNodes...)
	cluster.Keyspace = "am"
	cluster.Consistency = gocql.Quorum
	cluster.ProtoVersion = 3
	session, err := cluster.CreateSession()
	if err != nil {
		log.Fatalf("Can't create cassandra session. Err: %v", err)
	}
	defer session.Close()

	var enqued int
	batch := session.NewBatch(gocql.UnloggedBatch)
	stmt := `INSERT INTO users(id, email, dob, weight, height, nickname, country, city, caption, longitude, latitude, gender) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
	for user := range users {
		if enqued > 0 && enqued%BatchSize == 0 {
			err := session.ExecuteBatch(batch)
			if err != nil {
				log.Fatalf("Can't execute batch. Err: %v", err)
			}

			log.Printf("Batch with %v users saved! Total users inserted: %v", BatchSize, enqued)

			batch = gocql.NewBatch(gocql.UnloggedBatch)
		}

		// insert a user
		batch.Query(stmt,
			user.Pnum,
			user.Email,
			user.Dob,
			user.Weight,
			user.Height,
			user.Nickname,
			user.Country,
			user.City,
			user.Caption,
			user.Location.Longitude,
			user.Location.Latitude,
			user.Gender)

		enqued++
	}

	if batch.Size() > 0 {
		if err := session.ExecuteBatch(batch); err != nil {
			log.Fatalf("Can't execute batch. Err: %v", err)
		}
	}

}
Beispiel #6
0
func main() {

	session := integration.TestSession("127.0.0.1", "cqlc")
	integration.Truncate(session, EVENTS)

	result := "FAILED"

	ctx := cqlc.NewContext()
	batch := gocql.NewBatch(gocql.LoggedBatch)

	events := 50 * 1000
	batchSize := 1000

	for i := 0; i < events; i++ {
		ctx.Upsert(EVENTS).
			SetInt64(EVENTS.SENSOR, int64(i)).
			SetTimeUUID(EVENTS.TIMESTAMP, gocql.TimeUUID()).
			SetFloat32(EVENTS.TEMPERATURE, 19.8).
			SetInt32(EVENTS.PRESSURE, 357).
			Batch(batch)

		if i%batchSize == 0 {
			if err := session.ExecuteBatch(batch); err != nil {
				log.Fatalf("Could not execute batch: %v", err)
				os.Exit(1)
			}
			batch = gocql.NewBatch(gocql.LoggedBatch)
		}
	}

	err := session.ExecuteBatch(batch)

	if err != nil {
		log.Fatalf("Could not execute batch: %v", err)
		os.Exit(1)
	}

	query, err := ctx.Select().From(EVENTS).Prepare(session)
	if err != nil {
		log.Fatalf("Could not prepare query: %v", err)
		os.Exit(1)
	}

	query.PageSize(100)
	iter := query.Iter()
	count := 0

	MapEvents(iter, func(e Events) (bool, error) {
		count++
		return true, nil
	})

	if err := iter.Close(); err != nil {
		log.Fatalf("Could not close iterator: %v", err)
		os.Exit(1)
	}

	if count == events {
		result = "PASSED"
	}

	os.Stdout.WriteString(result)
}
Beispiel #7
0
func main() {

	session := integration.TestSession("127.0.0.1", "cqlc")
	integration.Truncate(session, EVENTS)

	result := "FAILED"

	ctx := cqlc.NewContext()
	batch := gocql.NewBatch(gocql.LoggedBatch)

	events := 100

	for i := 0; i < events; i++ {
		ctx.Upsert(EVENTS).
			SetInt64(EVENTS.SENSOR, int64(100)).
			SetTimeUUID(EVENTS.TIMESTAMP, gocql.TimeUUID()).
			SetFloat32(EVENTS.TEMPERATURE, 19.8).
			SetInt32(EVENTS.PRESSURE, 357).
			Batch(batch)
	}

	if err := session.ExecuteBatch(batch); err != nil {
		log.Fatalf("Could not execute batch: %v", err)
		os.Exit(1)
	}

	limit := 11

	iter, err := ctx.Select(EVENTS.SENSOR).
		From(EVENTS).
		Limit(limit).
		Fetch(session)

	if err != nil {
		log.Fatalf("Could not execute query: %v", err)
		os.Exit(1)
	}

	count := 0

	MapEvents(iter, func(e Events) (bool, error) {
		count++
		return true, nil
	})

	if count == limit {

		limit = 14

		iter, err = ctx.Select().
			From(EVENTS).
			Where(EVENTS.SENSOR.Eq(int64(100))).
			Limit(limit).
			Fetch(session)

		count := 0

		MapEvents(iter, func(e Events) (bool, error) {
			count++
			return true, nil
		})

		if count == limit {
			result = "PASSED"
		} else {
			result = fmt.Sprintf("Expected limit of %d; got %d rows", limit, count)
		}

	} else {
		result = fmt.Sprintf("Expected limit of %d; got %d rows", limit, count)
	}

	if err != nil {
		log.Fatalf("Could not execute query: %v", err)
		os.Exit(1)
	}

	os.Stdout.WriteString(result)
}
Beispiel #8
0
func main() {

	session := integration.TestSession("127.0.0.1", "cqlc")
	integration.Truncate(session, SIGNIFICANT_EVENTS)

	result := "FAILED"

	ctx := cqlc.NewContext()
	batch := gocql.NewBatch(gocql.LoggedBatch)

	events := 100

	var first, last gocql.UUID

	for i := 0; i < events; i++ {
		ts := gocql.TimeUUID()
		ctx.Upsert(SIGNIFICANT_EVENTS).
			SetInt64(SIGNIFICANT_EVENTS.SENSOR, 100).
			SetTimeUUID(SIGNIFICANT_EVENTS.TIMESTAMP, ts).
			SetInt32(SIGNIFICANT_EVENTS.SIGNIFICANCE, int32(i/10)).
			SetFloat32(SIGNIFICANT_EVENTS.TEMPERATURE, 19.8).
			SetInt32(SIGNIFICANT_EVENTS.PRESSURE, 357).
			Batch(batch)
		switch i {
		case 0:
			first = ts
		case events - 1:
			last = ts
		}
	}

	if err := session.ExecuteBatch(batch); err != nil {
		log.Fatalf("Could not execute batch: %v", err)
		os.Exit(1)
	}

	count, err := checkOrdering(session, SIGNIFICANT_EVENTS.TIMESTAMP)

	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	if count == events {

		count, err = checkOrdering(session, SIGNIFICANT_EVENTS.TIMESTAMP.Desc())

		if err != nil {
			log.Fatal(err)
			os.Exit(1)
		}

		if count == events {

			firstRead, err := checkOrderedLimit(session, SIGNIFICANT_EVENTS.TIMESTAMP, SIGNIFICANT_EVENTS.SIGNIFICANCE)
			if err != nil {
				log.Fatal(err)
				os.Exit(1)
			}

			lastRead, err := checkOrderedLimit(session, SIGNIFICANT_EVENTS.TIMESTAMP.Desc())
			if err != nil {
				log.Fatal(err)
				os.Exit(1)
			}

			if first == firstRead {
				if last == lastRead {
					result = "PASSED"
				} else {
					result = fmt.Sprintf("Expected last %v; got %v", last.Time(), lastRead.Time())
				}

			} else {
				result = fmt.Sprintf("Expected first %v; got %v", first.Time(), firstRead.Time())
			}
		}

	} else {
		result = fmt.Sprintf("Expected %d rows; got %d rows", events, count)
	}

	os.Stdout.WriteString(result)
}
Beispiel #9
0
func main() {

	session := integration.TestSession("127.0.0.1", "cqlc")
	integration.Truncate(session, USER_ACCOUNTS)

	result := "FAILED"

	ctx := cqlc.NewContext()
	batch := gocql.NewBatch(gocql.LoggedBatch)

	f, err := os.Open(os.Args[1])
	if err != nil {
		log.Fatalf("Could not open CSV file: %s", err)
		os.Exit(1)
	}

	b, err := csvb.NewBinder(f, nil)
	if err != nil {
		log.Fatalf("Could not create binder: %s", err)
		os.Exit(1)
	}

	s := make(map[string]string)
	s["username"] = "******"
	s["email"] = "Email"
	s["password"] = "******"
	s["last_visited"] = "LastVisited"
	s["country"] = "Country"

	err = b.ForEach(func(row csvb.Row) (bool, error) {
		var u UserAccounts

		if err := row.Bind(&u, s); err != nil {
			return false, err
		}

		ctx.Store(USER_ACCOUNTS.Bind(u)).Batch(batch)

		return true, nil
	})

	if err != nil {
		log.Fatalf("Could not bind CSV file: %s", err)
		os.Exit(1)
	}

	if err := session.ExecuteBatch(batch); err != nil {
		log.Fatalf("Could not execute batch: %v", err)
		os.Exit(1)
	}

	iter, err := ctx.Select().
		From(USER_ACCOUNTS).
		Where(USER_ACCOUNTS.COUNTRY.Eq("uk")).
		Fetch(session)

	if err != nil {
		log.Fatalf("Could not query by secondary index: %s", err)
		os.Exit(1)
	}

	count := 0
	err = MapUserAccounts(iter, func(u UserAccounts) (bool, error) {
		count++
		if u.Country != "uk" {
			return false, fmt.Errorf("Expected country %s; got %s", "uk", u.Country)
		}
		return true, nil
	})

	if err != nil {
		log.Fatalf("Could not query by secondary index: %s", err)
		os.Exit(1)
	}

	if count == 1 {
		result = "PASSED"
	}

	os.Stdout.WriteString(result)
}
Beispiel #10
0
func insertData(inChan chan struct {
	k string
	v interface{}
}) {
	var val string
	tick := time.NewTicker(10000 * time.Millisecond)
	var insertmap map[string]map[string]int
	insertmap = make(map[string]map[string]int)
	batch := gocql.NewBatch(gocql.CounterBatch)
	stmt := "UPDATE tick.dist_over_time set count=count+? WHERE event_time=? AND stream=? AND attr_name=? AND attr_value=?"
	//count := 0
	for {
		select {
		case m := <-inChan:
			//fmt.Println(m.k, m.v)
			switch j := m.v.(type) {
			case bool:
				val = strconv.FormatBool(j)
			case int:
				val = strconv.Itoa(j)
			case float64:
				val = strconv.FormatFloat(j, 'f', 2, 64)
			case string:
				val = j
			default:
				val = "" //do not worry about this
			}

			var ok bool
			if val != "" {
				_, ok = insertmap[m.k]
				if !ok {
					innermap := make(map[string]int)
					innermap[val] = 1
					insertmap[m.k] = innermap
				} else {
					insertmap[m.k][val] = insertmap[m.k][val] + 1

				}
			}
		case <-tick.C:
			for k, values := range insertmap {
				for v, c := range values {
					//get current minute
					now := time.Now()
					t := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), 0, 0, time.Local)
					batch.Query(stmt, c, t, *stream, k, v)
					//delete(values, v)
				}
				//delete(insertmap, k)
			}
			err := session.ExecuteBatch(batch)
			if err != nil {
				// if error than lose 10 seconds worth of data --shudder--
				log.Println(err)
			}
			// re-initialize insertmap and batch
			batch = gocql.NewBatch(gocql.CounterBatch)
			insertmap = make(map[string]map[string]int)
		}
	}
}