// NewRedisClient return a new RedisClient given the provided RedisOptions
func NewRedisClient(opts RedisOptions) (*RedisClient, error) {
if opts.ClientID == "" {
id, err := gocql.RandomUUID()
if err != nil {
return nil, err
}
opts.ClientID = id.String()
}
if opts.Network == "" {
opts.Network = "tcp"
}
if opts.Namespace == "" {
opts.Namespace = "glock:"
}
if opts.DialFunc == nil {
opts.DialFunc = redis.Dial
}
c := RedisClient{nil, opts}
err := c.Reconnect()
if err != nil {
return nil, err
}
return &c, nil
}
func TestUUID(t *testing.T) {
type KeyValue struct {
Id gocql.UUID `json:"id" cql:"key"`
Text string `json:"id" cql:"value"`
}
s := setup(t, "key_value")
defer s.Close()
id, err := gocql.RandomUUID()
assert.NoError(t, err)
kv := KeyValue{
Id: id,
Text: "hello world",
}
if err := Bind(`INSERT INTO key_value (key, value) VALUES (?, ?)`, &kv).Exec(s); err != nil {
t.Fatal(err)
}
q := s.Query(`SELECT key, value FROM key_value LIMIT 1`)
b := BindQuery(q)
var read KeyValue
b.Scan(&read)
if err := b.Close(); err != nil {
t.Fatal(err)
}
assert.Equal(t, kv, read)
}
//StoreBatchEvents stores multiple events from models.EventTrackingPayload within a batch
func (d *StorageDatastore) StoreBatchEvents(p *models.EventTrackingPayload) error {
batch := gocql.NewBatch(gocql.LoggedBatch)
statement := `INSERT INTO tracking.events (id, client, name, date, properties) VALUES (?, ?, ?, ?, ?)`
for _, e := range p.Events {
//Generate a new UUID
u4, err := gocql.RandomUUID()
if err != nil {
return err
}
//map properties
var propMap map[string]string
propMap = make(map[string]string)
for _, property := range e.Properties {
propMap[property.Name] = property.Value
}
//Add to batch
batch.Query(statement, u4.String(), p.Token, e.Name, e.Date, propMap)
}
err := d.Session.ExecuteBatch(batch)
if err != nil {
return err
}
return nil
}
func generateSerieIds(count int) []gocql.UUID {
result := make([]gocql.UUID, count)
for i := 0; i < count; i++ {
result[i], _ = gocql.RandomUUID()
}
return result
}
func requestID(inner http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
//uid := uuid.TimeUUID()
uuid, err := gocql.RandomUUID()
if err == nil {
r.Header.Set("Request-Id", uuid.String())
}
w.Header().Set("Request-Id", r.Header.Get("Request-Id"))
inner.ServeHTTP(w, r)
})
}
// NewCassandraLockClient creates a new client from options
func NewCassandraLockClient(opts CassandraOptions) (*CassandraClient, error) {
if opts.ReplicationFactor <= 0 {
opts.ReplicationFactor = 1
}
consistency := gocql.Quorum
if opts.ReplicationFactor == 0 {
consistency = gocql.One
}
var session *gocql.Session
var err error
var c CassandraClient
for proto := 4; proto > 1; proto-- {
c = CassandraClient{nil, opts.Hosts, "", "", "", nil, proto, consistency}
c.cluster = gocql.NewCluster(opts.Hosts...)
c.cluster.ProtoVersion = proto
session, err = c.cluster.CreateSession()
if err == nil {
break
}
}
if err != nil {
return nil, err
}
err = session.Query(
fmt.Sprintf(createKs, opts.KeySpace, opts.ReplicationFactor),
).Exec()
if err != nil {
return nil, err
}
err = session.Query(
fmt.Sprintf(createTable, opts.KeySpace, opts.TableName),
).Exec()
if err != nil {
return nil, err
}
id, err := gocql.RandomUUID()
if err != nil {
return nil, err
}
c.hosts = opts.Hosts
c.keyspace = opts.KeySpace
c.table = opts.TableName
c.clientID = id.String()
c.Reconnect()
return &c, nil
}
func fakeUuid() gocql.UUID {
initUuids.Do(func() {
for i := 0; i < 5; i++ {
u, err := gocql.RandomUUID()
if err != nil {
panic(err)
}
selectUuids = append(selectUuids, u)
}
})
return selectUuids[rand.Intn(len(selectUuids))]
}
func getRandom(session *gocql.Session) [][]string {
var first, rest, defn string
rval := make([][]string, 0)
random, _ := gocql.RandomUUID()
iter := session.Query(`select first, rest, defn from defs where TOKEN(first) > TOKEN(?) limit 1`, random.String()).Consistency(gocql.One).Iter()
for iter.Scan(&first, &rest, &defn) {
rval = append(rval, []string{first, rest, defn})
}
if err := iter.Close(); err != nil {
log.Fatal(err)
}
return rval
}
func (conn *CassConnection) CreateDevice(
name string,
uuid *gocql.UUID,
secretKey string,
publicAccessLevel datalayer.AccessLevel) (datalayer.Device, error) {
// TODO: validate parameters
var id gocql.UUID
var err error
if uuid == nil {
id, err = gocql.RandomUUID()
if err != nil {
return nil, err
}
} else {
id = *uuid
}
if secretKey == "" {
secretKey, err = random.Base64String(24)
if err != nil {
return nil, err
}
}
err = conn.session.Query(`
INSERT INTO devices (device_id, secret_key, friendly_name, public_access_level)
VALUES (?, ?, ?, ?)
`, id, secretKey, name, publicAccessLevel).Exec()
if err != nil {
canolog.Error("Error creating device:", err)
return nil, err
}
return &CassDevice{
conn: conn,
deviceId: id,
secretKey: secretKey,
name: name,
doc: sddl.Sys.NewEmptyDocument(),
docString: "",
publicAccessLevel: publicAccessLevel,
locationNote: "",
wsConnected: false,
}, nil
}
// Retweet creates a new retweet (copy of) an existing tweet
func Retweet(s *gocql.Session, username string, id gocql.UUID) (*Tweet, error) {
ot, err := GetTweet(s, id)
if err != nil {
return &Tweet{}, err
}
nid, err := gocql.RandomUUID()
if err != nil {
return &Tweet{}, err
}
t := Tweet{
ID: nid,
Username: username,
Message: ot.Message,
IsRetweet: true,
OriginalID: id,
}
return &t, CreateTweet(s, &t)
}
func NewCassandraDatastore() (*CassandraDatastore, error) {
ds := &CassandraDatastore{
cf: GetCassandraConfig(),
}
var err error
ds.db, err = ds.cf.CreateSession()
if err != nil {
return nil, fmt.Errorf("Failed to create cassandra datastore: %v", err)
}
ds.addedDomains = lrucache.New(Config.AddedDomainsCacheSize)
u, err := gocql.RandomUUID()
if err != nil {
return ds, err
}
ds.crawlerUuid = u
return ds, nil
}
func Create() (*Fields, error) {
var err error
m := &Fields{Exists: false}
for {
m.UUID, err = gocql.RandomUUID()
if err != nil {
return nil, err
}
var row = map[string]interface{}{}
var apply bool
if apply, err = CQLSession.Query(`insert into live_planets (planet_uuid,create_time) values (?,now()) if not exists`, m.UUID).MapScanCAS(row); err != nil {
logger.Error(errors.New(err))
return nil, err
}
if apply {
break
}
}
return m, nil
}
func login(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
if r.Method == "POST" {
r.ParseForm()
login := r.Form["login"][0]
password := password_hash((r.Form["password"][0]))
var existed_login string
var existed_cookie gocql.UUID
var cookie gocql.UUID
if err := session.Query("SELECT login FROM users WHERE login = ? and password = ?", login, password).Consistency(gocql.One).Scan(&existed_login); err != nil {
fmt.Println(err)
}
if existed_login != "" {
if err := session.Query("SELECT cookie FROM cookies WHERE login = ?", login).Consistency(gocql.One).Scan(&existed_cookie); err != nil {
if err == gocql.ErrNotFound {
cookie, _ = gocql.RandomUUID()
if err := session.Query("INSERT INTO cookies (login, cookie) VALUES (?, ?)", login, cookie).Exec(); err != nil {
w.WriteHeader(500)
return
}
} else {
w.WriteHeader(500)
return
}
} else {
cookie = existed_cookie
}
w.Header().Set("Set-Cookie", "sessionToken="+cookie.String())
fmt.Fprintf(w, "{\"error\":\"\"}\n")
} else {
w.WriteHeader(403)
fmt.Fprintf(w, "{\"error\":\"User or password is wrong\"}\n")
return
}
} else {
w.WriteHeader(405)
fmt.Fprintf(w, "{\"error\":\"Allowed methods: POST\"}\n")
}
}
func main() {
session := integration.TestSession("127.0.0.1", "cqlc")
cqlc.Truncate(session, BASIC)
result := "FAILED"
ctx := cqlc.NewContext()
uuid, _ := gocql.RandomUUID()
biggie := new(big.Int)
biggie.SetString("830169365738487321165427203929228", 10)
basic := Basic{
Id: "x",
Int32Column: 111,
Int64Column: 1 << 32,
AsciiColumn: "ABC",
TimestampColumn: time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC), // Keep it simple for reflect.DeepEqual
BooleanColumn: true,
TextColumn: "foo",
VarcharColumn: "bar",
FloatColumn: math.MaxFloat32,
DoubleColumn: math.MaxFloat64,
DecimalColumn: inf.NewDec(1, 3),
VarintColumn: biggie,
TimeuuidColumn: gocql.TimeUUID(),
UuidColumn: uuid,
MapColumn: map[string]string{"baz": "quux"},
ArrayColumn: []string{"baz", "quux"},
SetColumn: []string{"baz", "quux"},
}
create(ctx, session, basic)
iter, err := ctx.Select(
BASIC.ID,
BASIC.ASCII_COLUMN,
BASIC.INT32_COLUMN,
BASIC.INT64_COLUMN,
BASIC.FLOAT_COLUMN,
BASIC.DOUBLE_COLUMN,
BASIC.DECIMAL_COLUMN,
BASIC.VARINT_COLUMN,
BASIC.TIMESTAMP_COLUMN,
BASIC.TIMEUUID_COLUMN,
BASIC.UUID_COLUMN,
BASIC.BOOLEAN_COLUMN,
BASIC.TEXT_COLUMN,
BASIC.VARCHAR_COLUMN,
BASIC.MAP_COLUMN,
BASIC.ARRAY_COLUMN,
BASIC.SET_COLUMN).
From(BASIC).
Where(BASIC.ID.Eq("x")).
Fetch(session)
if err != nil {
log.Fatalf("Could not bind data: %v", err)
os.Exit(1)
}
result, _ = checkBasics(iter, basic)
iter, err = ctx.Select(
BASIC.ID,
BASIC.ASCII_COLUMN,
BASIC.INT32_COLUMN,
BASIC.INT64_COLUMN,
BASIC.FLOAT_COLUMN,
BASIC.DOUBLE_COLUMN,
BASIC.DECIMAL_COLUMN,
BASIC.VARINT_COLUMN,
BASIC.TIMESTAMP_COLUMN,
BASIC.TIMEUUID_COLUMN,
BASIC.UUID_COLUMN,
BASIC.BOOLEAN_COLUMN,
BASIC.TEXT_COLUMN,
BASIC.VARCHAR_COLUMN,
BASIC.MAP_COLUMN,
BASIC.ARRAY_COLUMN,
BASIC.SET_COLUMN).
From(BASIC).
Fetch(session)
if err != nil {
log.Fatalf("Could not bind data: %v", err)
os.Exit(1)
}
result, _ = checkBasics(iter, basic)
// TODO write test case for a non-matching WHERE clause
os.Stdout.WriteString(result)
}
//
// Config alteration right up front
//
func modifyConfigDataSource() {
walker.Config.Cassandra.Keyspace = "walker_test_model"
walker.Config.Cassandra.Hosts = []string{"localhost"}
walker.Config.Cassandra.ReplicationFactor = 1
}
//
// Some global data
//
var fooTime = time.Now().AddDate(0, 0, -1)
var testTime = time.Now().AddDate(0, 0, -2)
var bazUuid, _ = gocql.RandomUUID()
var testComLinkOrder []console.LinkInfo
var testComLinkHash = map[string]console.LinkInfo{
"http://test.com/page1.html": console.LinkInfo{
Url: "http://test.com/page1.html",
Status: 200,
Error: "",
RobotsExcluded: false,
CrawlTime: walker.NotYetCrawled,
},
"http://test.com/page2.html": console.LinkInfo{
Url: "http://test.com/page2.html",
Status: 200,
Error: "",
RobotsExcluded: false,
