func New(opts util.JsMap, log *util.HekaLogger) *Storage {
config = opts
var ok bool
if _, ok = config["memcache.server"]; !ok {
config["memcache.server"] = "127.0.0.1:11211"
}
if _, ok = config["db.timeout_live"]; !ok {
config["db.timeout_live"] = "259200"
}
if _, ok = config["db.timeout_reg"]; !ok {
config["db.timeout_reg"] = "10800"
}
if _, ok = config["db.timeout_del"]; !ok {
config["db.timeout_del"] = "86400"
}
if _, ok = config["shard.defaultHost"]; !ok {
config["shard.defaultHost"] = "localhost"
}
if _, ok = config["shard.currentHost"]; !ok {
config["shard.currentHost"] = config["shard.defaultHost"]
}
if _, ok = config["shard.prefix"]; !ok {
config["shard.prefix"] = "_h-"
}
log.Info("storage", "Creating new memcache handler", nil)
return &Storage{mc: memcache.New(config["memcache.server"].(string)),
config: config,
log: log}
}
// Generate a new Memcache Client
func newMC(servers []string, config util.JsMap, logger *util.HekaLogger) (mc gomc.Client) {
var err error
/* if mcsPoolSize >= mcsMaxPoolSize {
return nil
}
*/
mc, err = gomc.NewClient(servers, 1, gomc.ENCODING_GOB)
if err != nil {
logger.Critical("storage", "CRITICAL HIT!",
util.Fields{"error": err.Error()})
}
// internally hash key using MD5 (for key distribution)
mc.SetBehavior(gomc.BEHAVIOR_KETAMA_HASH, 1)
// Use the binary protocol, which allows us faster data xfer
// and better data storage (can use full UTF-8 char space)
mc.SetBehavior(gomc.BEHAVIOR_BINARY_PROTOCOL, 1)
//mc.SetBehavior(gomc.BEHAVIOR_NO_BLOCK, 1)
// NOTE! do NOT set BEHAVIOR_NOREPLY + Binary. This will cause
// libmemcache to drop into an infinite loop.
if v, ok := config["memcache.recv_timeout"]; ok {
d, err := time.ParseDuration(v.(string))
if err == nil {
mc.SetBehavior(gomc.BEHAVIOR_SND_TIMEOUT,
uint64(d.Nanoseconds()*1000))
}
}
if v, ok := config["memcache.send_timeout"]; ok {
d, err := time.ParseDuration(v.(string))
if err == nil {
mc.SetBehavior(gomc.BEHAVIOR_RCV_TIMEOUT,
uint64(d.Nanoseconds()*1000))
}
}
if v, ok := config["memcache.poll_timeout"]; ok {
d, err := time.ParseDuration(v.(string))
if err == nil {
mc.SetBehavior(gomc.BEHAVIOR_POLL_TIMEOUT,
uint64(d.Nanoseconds()*1000))
}
}
if v, ok := config["memcache.retry_timeout"]; ok {
d, err := time.ParseDuration(v.(string))
if err == nil {
mc.SetBehavior(gomc.BEHAVIOR_RETRY_TIMEOUT,
uint64(d.Nanoseconds()*1000))
}
}
atomic.AddInt32(&mcsPoolSize, 1)
return mc
}
// Handle a routed update.
func updater(update *router.Update, logger *util.HekaLogger) (err error) {
//log.Printf("UPDATE::: %s", update)
simplepush.MetricIncrement("updates.routed.incoming")
pk, _ := storage.GenPK(update.Uaid, update.Chid)
err = store.UpdateChannel(pk, update.Vers)
if client, ok := simplepush.Clients[update.Uaid]; ok {
simplepush.Flush(client, update.Chid, int64(update.Vers))
duration := strconv.FormatInt(time.Now().Sub(update.Time).Nanoseconds(), 10)
if logger != nil {
logger.Info("timer", "Routed flush to client completed",
util.Fields{
"uaid": update.Uaid,
"chid": update.Chid,
"duration": duration})
} else {
log.Printf("Routed flush complete: %s", duration)
}
}
return nil
}
// -- REST
func UpdateHandler(resp http.ResponseWriter, req *http.Request, config util.JsMap, logger *util.HekaLogger) {
// Handle the version updates.
var err error
timer := time.Now()
logger.Debug("main", fmt.Sprintf("Handling Update %s", req.URL.Path), nil)
if req.Method != "PUT" {
http.Error(resp, "", http.StatusMethodNotAllowed)
return
}
vers := fmt.Sprintf("%d", time.Now().UTC().Unix())
elements := strings.Split(req.URL.Path, "/")
pk := elements[len(elements)-1]
if len(pk) == 0 {
logger.Error("main", "No token, rejecting request", nil)
http.Error(resp, "Token not found", http.StatusNotFound)
return
}
store := storage.New(config, logger)
if token, ok := config["token_key"]; ok {
var err error
bpk, err := Decode(token.([]byte),
pk)
if err != nil {
logger.Error("main",
fmt.Sprintf("Could not decode token %s, %s", pk, err),
nil)
http.Error(resp, "", http.StatusNotFound)
return
}
pk = strings.TrimSpace(string(bpk))
}
uaid, appid, err := storage.ResolvePK(pk)
if err != nil {
logger.Error("main",
fmt.Sprintf("Could not resolve PK %s, %s", pk, err), nil)
return
}
currentHost := "localhost"
if val, ok := config["shard.currentHost"]; ok {
currentHost = val.(string)
}
host, err := store.GetUAIDHost(uaid)
if err != nil {
logger.Error("main",
fmt.Sprintf("Could not discover host for %s, %s (using default)",
uaid, err), nil)
if val, ok := config["shard.defaultHost"]; ok {
host = val.(string)
} else {
val = "localhost"
}
}
if host != currentHost || host != "localhost" {
logger.Info("main", fmt.Sprintf("Proxying request to %s", host), nil)
err = proxyNotification(host, req.URL.Path)
if err != nil {
logger.Error("main",
fmt.Sprintf("Proxy to %s failed: %s", host, err),
nil)
}
return
}
defer func(uaid, appid, path string, timer time.Time) {
logger.Info("timer", "Client Update complete",
util.JsMap{
"uaid": uaid,
"path": req.URL.Path,
"channelID": appid,
"duration": time.Now().Sub(timer).Nanoseconds()})
}(uaid, appid, req.URL.Path, timer)
logger.Info("main",
fmt.Sprintf("setting version for %s.%s to %s", uaid, appid, vers),
nil)
err = store.UpdateChannel(pk, vers)
if err != nil {
errstr := fmt.Sprintf("Could not update channel %s.%s :: %s", uaid, appid, err)
logger.Warn("main", errstr, nil)
status := sperrors.ErrToStatus(err)
http.Error(resp, errstr, status)
return
}
resp.Header().Set("Content-Type", "application/json")
resp.Write([]byte("{}"))
logger.Info("timer", "Client Update complete",
util.JsMap{"uaid": uaid,
"channelID": appid,
"duration": time.Now().Sub(timer).Nanoseconds()})
// Ping the appropriate server
if client, ok := Clients[uaid]; ok {
Flush(client)
}
return
}
func New(opts util.JsMap, logger *util.HekaLogger) *Storage {
config = opts
var ok bool
var err error
if configEndpoint, ok := config["elasticache.config_endpoint"]; ok {
memcacheEndpoint, err := getElastiCacheEndpointsTimeout(configEndpoint.(string), 2)
if err == nil {
config["memcache.server"] = memcacheEndpoint
} else {
fmt.Println(err)
if logger != nil {
logger.Error("storage", "Elastisearch error.",
util.Fields{"error": err.Error()})
}
}
}
if _, ok = config["memcache.server"]; !ok {
config["memcache.server"] = "127.0.0.1:11211"
}
timeout, err := time.ParseDuration(util.MzGet(config, "db.handle_timeout", "5s"))
if err != nil {
if logger != nil {
logger.Error("storage", "Could not parse db.handle_timeout",
util.Fields{"err": err.Error()})
}
timeout = 10 * time.Second
}
if _, ok = config["memcache.pool_size"]; !ok {
config["memcache.pool_size"] = "100"
}
if config["memcache.pool_size"], err =
strconv.ParseInt(config["memcache.pool_size"].(string), 0, 0); err != nil {
config["memcache.pool_size"] = 100
}
poolSize := int(config["memcache.pool_size"].(int64))
if _, ok = config["db.timeout_live"]; !ok {
config["db.timeout_live"] = "259200"
}
if _, ok = config["db.timeout_reg"]; !ok {
config["db.timeout_reg"] = "10800"
}
if _, ok = config["db.timeout_del"]; !ok {
config["db.timeout_del"] = "86400"
}
if _, ok = config["shard.default_host"]; !ok {
config["shard.default_host"] = "localhost"
}
if _, ok = config["shard.current_host"]; !ok {
config["shard.current_host"] = config["shard.default_host"]
}
if _, ok = config["shard.prefix"]; !ok {
config["shard.prefix"] = "_h-"
}
/*
i, err := strconv.ParseInt(util.MzGet(config,
"memcache.max_pool_size", "400"), 0, 10)
if err != nil {
if logger != nil {
logger.Error("storage", "Could not parse memcache.max_pool_size",
util.Fields{"error": err.Error()})
}
mcsMaxPoolSize = 400
}
mcsMaxPoolSize = int32(i)
*/
if logger != nil {
logger.Info("storage", "Creating new gomc handler", nil)
}
// do NOT include any spaces
servers := strings.Split(
no_whitespace.Replace(config["memcache.server"].(string)),
",")
mcs := make(chan gomc.Client, poolSize)
for i := 0; i < poolSize; i++ {
mcs <- newMC(servers, config, logger)
}
return &Storage{
mcs: mcs,
config: config,
logger: logger,
mc_timeout: timeout,
servers: servers,
}
}