func (u *Universe) SpawnAsteroid(redis_client *redis.Client) {
if rand.Float32() < 0.1 {
live_asteroids := redis_client.Cmd("KEYS", "asteroid-*")
num_asteroids := 1000
if len(live_asteroids.Elems) < num_asteroids {
temp_asteroid := &asteroid.Asteroid{gameobject.GameObject{
Id: rand.Intn(500000),
X: rand.Float64(),
Y: rand.Float64(),
Velx: rand.Float64() * 100,
Vely: rand.Float64() * 100},
100}
u.Asteroids = append(u.Asteroids, temp_asteroid)
asteroid_json, _ := json.Marshal(temp_asteroid)
redis_client.Cmd("SET", fmt.Sprintf("asteroid-%v", temp_asteroid.Id), asteroid_json)
}
// temp_bullet := &bullet.Bullet{gameobject.GameObject{
// Id: rand.Intn(500000),
// X: rand.Float64(),
// Y: rand.Float64(),
// Velx: rand.Float64() * 100,
// Vely: rand.Float64() * 100},
// 100}
// u.Bullets = append(u.Bullets,temp_bullet)
// redis_client.Cmd("SET", fmt.Sprintf("bullet-%v", temp_bullet.Id), temp_bullet)
}
}
func insertDocument(domain, title, url, group string, popularity float64, c *redis.Client) {
encoded := encodeString(domain)
encodedURL := encodeString(url)
existingTitle, _ := c.Cmd("HGET", encoded+"$titles", encodedURL).Str()
if existingTitle == "" {
err := c.Cmd("HINCRBY", "$domaindocuments", domain, 1).Err
errorHandler(err)
}
pipedCommands := 0
for _, prefix := range getPrefixes(title) {
if group != "" {
encodedGroup := encodeString(group)
c.Append("ZADD", encoded+encodedGroup+prefix, popularity, encodedURL)
c.Append("HSET", encoded+"$groups", encodedURL, encodedGroup)
pipedCommands++
} else {
c.Append("ZADD", encoded+prefix, popularity, encodedURL)
}
pipedCommands++
}
c.Append("HSET", encoded+"$titles", encodedURL, title)
pipedCommands++
c.Append("HSET", encoded+"$urls", encodedURL, url)
pipedCommands++
for i := 1; i <= pipedCommands; i++ {
if err := c.GetReply().Err; err != nil {
errorHandler(err)
}
}
}
func authenticateByPassword(c *redis.Client, password string) error {
if r := c.Cmd("AUTH", password); r.Err != nil {
logger.Errorf("Faild to authenticate. %s", r.Err)
return r.Err
}
return nil
}
func getRedisInfo(c *redis.Client) (*map[string]string, error) {
info := make(map[string]string)
r := c.Cmd("info")
if r.Err != nil {
return nil, errors.New("couldn't execute query")
}
str, err := r.Str()
if err != nil {
return nil, errors.New("couldn't execute query")
}
for _, line := range strings.Split(str, "\r\n") {
if line == "" {
continue
}
if re, _ := regexp.MatchString("^#", line); re {
continue
}
record := strings.SplitN(line, ":", 2)
if len(record) < 2 {
continue
}
key, value := record[0], record[1]
info[key] = value
}
return &info, nil
}
func fetchPercentageOfMemory(c *redis.Client, stat map[string]float64) error {
r := c.Cmd("CONFIG", "GET", "maxmemory")
if r.Err != nil {
logger.Errorf("Failed to run `CONFIG GET maxmemory` command. %s", r.Err)
return r.Err
}
res, err := r.Hash()
if err != nil {
logger.Errorf("Failed to fetch maxmemory. %s", err)
return err
}
maxsize, err := strconv.ParseFloat(res["maxmemory"], 64)
if err != nil {
logger.Errorf("Failed to parse maxmemory. %s", err)
return err
}
if maxsize == 0.0 {
stat["percentage_of_memory"] = 0.0
} else {
stat["percentage_of_memory"] = 100.0 * stat["used_memory"] / maxsize
}
return nil
}
func getBackend(hostname string, defaultBackendType string, redisClient *redis.Client) (string, error) {
fmt.Println("Looking up", hostname)
backends, error := redisClient.Cmd("smembers", "hostnames:"+hostname+":backends").List()
if error != nil {
fmt.Println("Error in redis lookup for hostname backend", error)
return "", error
}
if len(backends) == 0 {
backends, error = redisClient.Cmd("smembers", "hostnames:"+defaultBackendType+":backends").List()
if error != nil {
fmt.Println("Error in redis lookup for default backend", error)
return "", error
}
if len(backends) == 0 {
fmt.Println("No default backend of type", defaultBackendType)
return "", errors.New("Could not find default backend of type " + defaultBackendType)
}
}
fmt.Println("Found backends:", backends)
backend := backends[int(rand.Float32()*float32(len(backends)))]
return backend, nil
}
func list(connection *redis.Client) map[string][]string {
var blacklisted []string
var whitelisted []string
var marked []string
repsheet := connection.Cmd("KEYS", "*:repsheet:*:*")
for i := 0; i < len(repsheet.Elems); i++ {
value, _ := repsheet.Elems[i].Str()
parts := strings.Split(value, ":")
repsheetType := parts[len(parts)-1]
switch repsheetType {
case "blacklisted":
blacklisted = append(blacklisted, parts[0])
case "whitelisted":
whitelisted = append(whitelisted, parts[0])
case "marked":
marked = append(marked, parts[0])
}
}
var list = make(map[string][]string)
list["blacklisted"] = blacklisted
list["whitelisted"] = whitelisted
list["marked"] = marked
return list
}
func SetRedisValue(redisClient *redis.Client, key string, value string) {
var reply *redis.Reply = redisClient.Cmd("SET", key, value)
if reply.Err != nil {
panic(reply.Err)
}
}
func storeCacheValues(rascalData []byte, cdnName string, sampleTime int64, cacheGroupMap map[string]string, redisClient *redis.Client) error {
/* note about the redis data:
keys are cdnName:deliveryService:cacheGroup:cacheName:statName
*/
type CacheStatsJson struct {
Pp string `json:"pp"`
Date string `json:"date"`
Caches map[string]map[string][]struct {
Index uint64 `json:"index"`
Time uint64 `json:"time"`
Value string `json:"value"`
Span uint64 `json:"span"`
} `json:"caches"`
}
var jData CacheStatsJson
err := json.Unmarshal(rascalData, &jData)
errHndlr(err, ERROR)
statCount := 0
statTotals := make(map[string]float64)
for cacheName, cacheData := range jData.Caches {
for statName, statData := range cacheData {
redisKey := cdnName + ":all:" + cacheGroupMap[cacheName] + ":" + cacheName + ":" + statName
redisKey = strings.Replace(redisKey, ":bandwidth", ":kbps", 1)
statValue := statData[0].Value
//fmt.Printf("%s ->%s\n", redisKey, statValue)
statCount++
statFloatValue, err := strconv.ParseFloat(statValue, 64)
if err != nil {
statFloatValue = 0.0
}
statTotals[cdnName+":all:"+cacheGroupMap[cacheName]+":all:"+statName] += statFloatValue
statTotals[cdnName+":all:all:all:"+statName] += statFloatValue
if statName == "maxKbps" {
r := redisClient.Cmd("zadd", redisKey, sampleTime, statValue) // only care for the last val here.
errHndlr(r.Err, ERROR)
} else {
r := redisClient.Cmd("rpush", redisKey, statValue)
errHndlr(r.Err, ERROR)
}
}
}
for totalKey, totalVal := range statTotals {
totalKey = strings.Replace(totalKey, ":bandwidth", ":kbps", 1)
if strings.Contains(totalKey, "maxKbps") {
r := redisClient.Cmd("zadd", totalKey, sampleTime, strconv.FormatFloat(totalVal, 'f', 2, 64))
errHndlr(r.Err, ERROR)
} else {
r := redisClient.Cmd("rpush", totalKey, strconv.FormatFloat(totalVal, 'f', 2, 64))
errHndlr(r.Err, ERROR)
}
statCount++
}
r := redisClient.Cmd("rpush", cdnName+":tstamp", sampleTime)
errHndlr(r.Err, ERROR)
log.Info("Saved ", statCount, " values for ", cdnName, " @ ", sampleTime)
return nil
}
func someFunc(c *redis.Client) (err error) {
reply := c.Cmd("set", "mykey", "myvalue")
// what is the recommended error
if reply.Err != nil {
return reply.Err
}
// some code here
return
}
func search(connection *redis.Client, actor string) bool {
searchString := fmt.Sprintf("%s:*", actor)
results := connection.Cmd("KEYS", searchString)
if len(results.Elems) == 1 {
return true
} else {
return false
}
}
func GetDomain(key string, conn *redis.Client) (string, error) {
reply := conn.Cmd("HGET", "$domainkeys", key)
if reply.Type == redis.NilReply {
return "", reply.Err
}
domain, err := reply.Str()
if err != nil {
return "", err
} else {
return domain, nil
}
}
/* the ds json looks like:
{
"deliveryService": {
"linear-gbr-hls-sbr": {
"location.us-ma-woburn.kbps": [{
"index": 520281,
"time": 1398893383605,
"value": "0",
"span": 520024
}],
"location.us-de-newcastle.kbps": [{
"index": 520281,
"time": 1398893383605,
"value": "0",
"span": 517707
}],
}
}
*/
func storeDsValues(rascalData []byte, cdnName string, sampleTime int64, redisClient *redis.Client, dsAggregate map[string]AggregationConfig) error {
type DsStatsJson struct {
Pp string `json:"pp"`
Date string `json:"date"`
DeliveryService map[string]map[string][]struct {
Index uint64 `json:"index"`
Time uint64 `json:"time"`
Value string `json:"value"`
Span uint64 `json:"span"`
} `json:"deliveryService"`
}
var jData DsStatsJson
err := json.Unmarshal(rascalData, &jData)
errHndlr(err, ERROR)
statCount := 0
statTotals := make(map[string]float64)
for dsName, dsData := range jData.DeliveryService {
for dsMetric, dsMetricData := range dsData {
keyPart := strings.Replace(dsMetric, "location.", "", -1)
keyPart = strings.Replace(keyPart, ".kbps", ":all:kbps", -1)
keyPart = strings.Replace(keyPart, ".tps", ":all:tps", -1)
keyPart = strings.Replace(keyPart, ".status", ":all:status", -1)
keyPart = strings.Replace(keyPart, "total:all:", "all:all:", -1) // for consistency all everywhere
redisKey := cdnName + ":" + dsName + ":" + keyPart
statValue := dsMetricData[0].Value
//fmt.Printf("%s ->%s\n", redisKey, statValue)
statCount++
aggConfig, exists := dsAggregate[dsMetric]
if exists {
statFloatValue, err := strconv.ParseFloat(statValue, 64)
if err != nil {
statFloatValue = 0.0
}
statTotals[cdnName+":all:all:all:"+aggConfig.RedisKey] += statFloatValue
}
r := redisClient.Cmd("rpush", redisKey, statValue)
errHndlr(r.Err, ERROR)
}
}
for totalKey, totalVal := range statTotals {
r := redisClient.Cmd("rpush", totalKey, strconv.FormatFloat(totalVal, 'f', 2, 64))
errHndlr(r.Err, ERROR)
statCount++
}
log.Info("Saved ", statCount, " ds values for ", cdnName, " @ ", sampleTime)
return nil
}
// benchmark benchmarks the given command with the given parameters
// and displays the given test name.
func benchmark(c *redis.Client, testname string, command string, params ...interface{}) {
fmt.Printf("===== %s =====\n", testname)
start := time.Now()
for i := 0; i < *requests; i++ {
c.Cmd(command, params...)
}
duration := time.Now().Sub(start)
rps := float64(*requests) / duration.Seconds()
fmt.Println("Requests per second:", rps)
fmt.Printf("Duration: %v\n\n", duration.Seconds())
}
func (u *Universe) InitUniverse(redis_client *redis.Client) {
live_asteroids := redis_client.Cmd("KEYS", "asteroid-*")
//load asteroids from db
u.Asteroids = make([]*asteroid.Asteroid, 0, len(live_asteroids.Elems))
for key := range live_asteroids.Elems {
asteroid := &asteroid.Asteroid{}
temp_asteroid := redis_client.Cmd("GET", live_asteroids.Elems[key])
if err := json.Unmarshal([]byte(temp_asteroid.String()), &asteroid); err != nil {
}
u.Asteroids = append(u.Asteroids, asteroid)
}
}
func addToList(connection *redis.Client, list string, actor string, reason string, ttl int) {
actorString := fmt.Sprintf("%s:repsheet:ip:%sed", actor, list)
if reason == "" {
reason = "true"
}
if ttl < 0 {
connection.Cmd("SET", actorString, reason)
} else {
connection.Cmd("SETEX", actorString, ttl, reason)
}
}
func (s *Server) GetIdentIndex(db *redis.Client, start int, stop int) ([]Ident, error) {
zrange, err := db.Cmd("zrange", "nodes", start, stop).List()
if err != nil {
return []Ident{}, err
}
idents := make([]Ident, len(zrange))
for index, signature := range zrange {
ident, err := s.GetIdent(db, signature)
if err != nil {
return []Ident{}, err
}
idents[index] = ident
}
return idents, nil
}
func (u *Universe) Update(delta_time float64, redis_client *redis.Client) {
//here we run the game simulation
//check redis for api spawned things
//bullets
//should we spawn more asteroids?
u.SpawnAsteroid(redis_client)
//asteroid updates a.Update()
for _, a := range u.Asteroids {
a.Update(delta_time)
//write to redis
redis_client.Cmd("SET", fmt.Sprintf("asteroid-%v", a.Id), a.Dump())
}
}
func (s *Server) PutIdent(db *redis.Client, identity Ident) (Ident, error) {
data, err := identity.ToJson()
if err != nil {
return identity, err
}
zadd := db.Cmd("zadd", "nodes", time.Now().Unix(), identity.Signature)
if zadd.Err != nil {
return identity, zadd.Err
}
set := db.Cmd("set", node_key(identity.Signature), data)
if set.Err != nil {
return identity, set.Err
}
return identity, nil
}
func (s *Server) GetIdent(db *redis.Client, signature string) (Ident, error) {
ident := Ident{}
get := db.Cmd("get", node_key(signature))
if get.Err != nil {
return ident, get.Err
}
if get.Type == redis.NilReply {
return ident, errors.New(fmt.Sprintf("Ident %s does not exist.", signature))
}
str, err := get.Str()
if err != nil {
panic(err)
}
return NewIdentFromString(str)
}
func slugExists(redis *redis.Client, slug string) bool {
exists, err := redis.Cmd("HEXISTS", UrlStore, slug).Bool()
if err != nil {
fmt.Println("[!ERR] slugExists - ", err)
// TODO: Better to return true, but it could make infinite loop. Make better.
return false
}
return exists
}
func (self *OutputConfig) redisSend(client *redis.Client, key string, raw []byte) (err error) {
var (
res *redis.Reply
)
switch self.DataType {
case "list":
res = client.Cmd("rpush", key, raw)
err = res.Err
case "channel":
res = client.Cmd("publish", key, raw)
err = res.Err
default:
err = errors.New("unknown DataType: " + self.DataType)
}
return
}
func (s *Server) DeleteIdent(db *redis.Client, signature string) (Ident, error) {
ident := Ident{}
get := db.Cmd("get", node_key(signature))
if get.Err != nil {
return ident, get.Err
}
if get.Type == redis.NilReply {
return ident, errors.New(fmt.Sprintf("Ident %s does not exist.", signature))
}
str, err := get.Str()
if err != nil {
panic(err)
}
del := db.Cmd("del", node_key(signature))
if del.Err != nil {
return ident, del.Err
}
zrem := db.Cmd("zrem", "nodes", signature)
if zrem.Err != nil {
return ident, zrem.Err
}
if num_deleted, _ := del.Int(); num_deleted == 0 {
return ident, errors.New(fmt.Sprintf("Deleting %s failed.", signature))
}
return NewIdentFromString(str)
}
func fetchPercentageOfClients(c *redis.Client, stat map[string]float64) error {
r := c.Cmd("CONFIG", "GET", "maxclients")
if r.Err != nil {
logger.Errorf("Failed to run `CONFIG GET maxclients` command. %s", r.Err)
return r.Err
}
res, err := r.Hash()
if err != nil {
logger.Errorf("Failed to fetch maxclients. %s", err)
return err
}
maxsize, err := strconv.ParseFloat(res["maxclients"], 64)
if err != nil {
logger.Errorf("Failed to parse maxclients. %s", err)
return err
}
stat["percentage_of_clients"] = 100.0 * stat["connected_clients"] / maxsize
return nil
}
func getStats(client *redis.Client) (*RedisStats, error) {
info, err := client.Cmd("INFO").Str()
errHndlr(err)
maxmem_slice, err := client.Cmd("CONFIG", "GET", "maxmemory").List()
errHndlr(err)
usedmem_regex, _ := regexp.Compile(`used_memory:(\d+)`)
clientconn_regex, _ := regexp.Compile(`connected_clients:(\d+)`)
iops_regex, _ := regexp.Compile(`instantaneous_ops_per_sec:(\d+)`)
evictedkeys_regex, _ := regexp.Compile(`evicted_keys:(\d+)`)
db_regex, _ := regexp.Compile(`(db\d+):keys=(\d+),expires=(\d+)`)
usedmem, err := strconv.ParseInt(usedmem_regex.FindStringSubmatch(info)[1], 10, 64)
maxmem, err := strconv.ParseInt(maxmem_slice[len(maxmem_slice)-1], 10, 64)
freemem := maxmem - usedmem
evictedkeys, err := strconv.ParseInt(evictedkeys_regex.FindStringSubmatch(info)[1], 10, 64)
iops, err := strconv.ParseInt(iops_regex.FindStringSubmatch(info)[1], 10, 64)
clientconn, err := strconv.ParseInt(clientconn_regex.FindStringSubmatch(info)[1], 10, 64)
dbs := make(map[string]*DBKeyspace)
for _, db := range db_regex.FindAllStringSubmatch(info, -1) {
keys, _ := strconv.ParseInt(db[2], 10, 64)
expires, _ := strconv.ParseInt(db[3], 10, 64)
dbs[db[1]] = &DBKeyspace{
Keys: keys,
Expires: expires,
}
}
stats := &RedisStats{
FreeMemory: freemem,
EvictedKeys: evictedkeys,
InstantaneousOpsPerSec: iops,
ClientConnections: clientconn,
DBs: dbs,
}
return stats, err
}
func status(connection *redis.Client, actor string) *Status {
blacklistedString := fmt.Sprintf("%s:repsheet:ip:blacklisted", actor)
whitelistedString := fmt.Sprintf("%s:repsheet:ip:whitelisted", actor)
markedString := fmt.Sprintf("%s:repsheet:ip:marked", actor)
connection.Cmd("MULTI")
connection.Cmd("GET", whitelistedString)
connection.Cmd("GET", blacklistedString)
connection.Cmd("GET", markedString)
reply := connection.Cmd("EXEC")
if reply.Elems[0].Type != redis.NilReply {
str, _ := reply.Elems[0].Str()
return &Status{Type: Whitelisted, reason: str, actor: actor}
} else if reply.Elems[1].Type != redis.NilReply {
str, _ := reply.Elems[1].Str()
return &Status{Type: Blacklisted, reason: str, actor: actor}
} else if reply.Elems[2].Type != redis.NilReply {
str, _ := reply.Elems[1].Str()
return &Status{Type: Marked, reason: str, actor: actor}
} else {
return &Status{Type: OK, reason: "", actor: actor}
}
}
func (dispatcher *Dispatcher) redisSetExpiringValue(redisClient *redis.Client, dbIndex int, key string, value string, expirySeconds int) {
response := redisClient.Cmd("select", dbIndex)
if response.Err != nil {
log.Printf("redis select command failed: %s", response.Err)
return
}
response = redisClient.Cmd("set", key, value)
if response.Err != nil {
log.Printf("redis set command failed: %s", response.Err)
return
}
response = redisClient.Cmd("expire", key, expirySeconds)
if response.Err != nil {
log.Printf("redis expire command failed: %s", response.Err)
return
}
}
func cleanup(conn *redis.Client) {
conn.Cmd("FLUSHDB")
conn.Close()
}
func SetDomain(key string, domain string, conn *redis.Client) error {
return conn.Cmd("HSET", "$domainkeys", key, domain).Err
}
func IncrementRedisValue(redisClient *redis.Client, key string) {
var reply *redis.Reply = redisClient.Cmd("INCR", key)
if reply.Err != nil {
panic(reply.Err)
}
}
