func New(opts util.JsMap, log *util.HekaLogger) *Storage { config = opts var ok bool if _, ok = config["memcache.server"]; !ok { config["memcache.server"] = "127.0.0.1:11211" } if _, ok = config["db.timeout_live"]; !ok { config["db.timeout_live"] = "259200" } if _, ok = config["db.timeout_reg"]; !ok { config["db.timeout_reg"] = "10800" } if _, ok = config["db.timeout_del"]; !ok { config["db.timeout_del"] = "86400" } if _, ok = config["shard.defaultHost"]; !ok { config["shard.defaultHost"] = "localhost" } if _, ok = config["shard.currentHost"]; !ok { config["shard.currentHost"] = config["shard.defaultHost"] } if _, ok = config["shard.prefix"]; !ok { config["shard.prefix"] = "_h-" } log.Info("storage", "Creating new memcache handler", nil) return &Storage{mc: memcache.New(config["memcache.server"].(string)), config: config, log: log} }
// Generate a new Memcache Client func newMC(servers []string, config util.JsMap, logger *util.HekaLogger) (mc gomc.Client) { var err error /* if mcsPoolSize >= mcsMaxPoolSize { return nil } */ mc, err = gomc.NewClient(servers, 1, gomc.ENCODING_GOB) if err != nil { logger.Critical("storage", "CRITICAL HIT!", util.Fields{"error": err.Error()}) } // internally hash key using MD5 (for key distribution) mc.SetBehavior(gomc.BEHAVIOR_KETAMA_HASH, 1) // Use the binary protocol, which allows us faster data xfer // and better data storage (can use full UTF-8 char space) mc.SetBehavior(gomc.BEHAVIOR_BINARY_PROTOCOL, 1) //mc.SetBehavior(gomc.BEHAVIOR_NO_BLOCK, 1) // NOTE! do NOT set BEHAVIOR_NOREPLY + Binary. This will cause // libmemcache to drop into an infinite loop. if v, ok := config["memcache.recv_timeout"]; ok { d, err := time.ParseDuration(v.(string)) if err == nil { mc.SetBehavior(gomc.BEHAVIOR_SND_TIMEOUT, uint64(d.Nanoseconds()*1000)) } } if v, ok := config["memcache.send_timeout"]; ok { d, err := time.ParseDuration(v.(string)) if err == nil { mc.SetBehavior(gomc.BEHAVIOR_RCV_TIMEOUT, uint64(d.Nanoseconds()*1000)) } } if v, ok := config["memcache.poll_timeout"]; ok { d, err := time.ParseDuration(v.(string)) if err == nil { mc.SetBehavior(gomc.BEHAVIOR_POLL_TIMEOUT, uint64(d.Nanoseconds()*1000)) } } if v, ok := config["memcache.retry_timeout"]; ok { d, err := time.ParseDuration(v.(string)) if err == nil { mc.SetBehavior(gomc.BEHAVIOR_RETRY_TIMEOUT, uint64(d.Nanoseconds()*1000)) } } atomic.AddInt32(&mcsPoolSize, 1) return mc }
// Handle a routed update. func updater(update *router.Update, logger *util.HekaLogger) (err error) { //log.Printf("UPDATE::: %s", update) simplepush.MetricIncrement("updates.routed.incoming") pk, _ := storage.GenPK(update.Uaid, update.Chid) err = store.UpdateChannel(pk, update.Vers) if client, ok := simplepush.Clients[update.Uaid]; ok { simplepush.Flush(client, update.Chid, int64(update.Vers)) duration := strconv.FormatInt(time.Now().Sub(update.Time).Nanoseconds(), 10) if logger != nil { logger.Info("timer", "Routed flush to client completed", util.Fields{ "uaid": update.Uaid, "chid": update.Chid, "duration": duration}) } else { log.Printf("Routed flush complete: %s", duration) } } return nil }
// -- REST func UpdateHandler(resp http.ResponseWriter, req *http.Request, config util.JsMap, logger *util.HekaLogger) { // Handle the version updates. var err error timer := time.Now() logger.Debug("main", fmt.Sprintf("Handling Update %s", req.URL.Path), nil) if req.Method != "PUT" { http.Error(resp, "", http.StatusMethodNotAllowed) return } vers := fmt.Sprintf("%d", time.Now().UTC().Unix()) elements := strings.Split(req.URL.Path, "/") pk := elements[len(elements)-1] if len(pk) == 0 { logger.Error("main", "No token, rejecting request", nil) http.Error(resp, "Token not found", http.StatusNotFound) return } store := storage.New(config, logger) if token, ok := config["token_key"]; ok { var err error bpk, err := Decode(token.([]byte), pk) if err != nil { logger.Error("main", fmt.Sprintf("Could not decode token %s, %s", pk, err), nil) http.Error(resp, "", http.StatusNotFound) return } pk = strings.TrimSpace(string(bpk)) } uaid, appid, err := storage.ResolvePK(pk) if err != nil { logger.Error("main", fmt.Sprintf("Could not resolve PK %s, %s", pk, err), nil) return } currentHost := "localhost" if val, ok := config["shard.currentHost"]; ok { currentHost = val.(string) } host, err := store.GetUAIDHost(uaid) if err != nil { logger.Error("main", fmt.Sprintf("Could not discover host for %s, %s (using default)", uaid, err), nil) if val, ok := config["shard.defaultHost"]; ok { host = val.(string) } else { val = "localhost" } } if host != currentHost || host != "localhost" { logger.Info("main", fmt.Sprintf("Proxying request to %s", host), nil) err = proxyNotification(host, req.URL.Path) if err != nil { logger.Error("main", fmt.Sprintf("Proxy to %s failed: %s", host, err), nil) } return } defer func(uaid, appid, path string, timer time.Time) { logger.Info("timer", "Client Update complete", util.JsMap{ "uaid": uaid, "path": req.URL.Path, "channelID": appid, "duration": time.Now().Sub(timer).Nanoseconds()}) }(uaid, appid, req.URL.Path, timer) logger.Info("main", fmt.Sprintf("setting version for %s.%s to %s", uaid, appid, vers), nil) err = store.UpdateChannel(pk, vers) if err != nil { errstr := fmt.Sprintf("Could not update channel %s.%s :: %s", uaid, appid, err) logger.Warn("main", errstr, nil) status := sperrors.ErrToStatus(err) http.Error(resp, errstr, status) return } resp.Header().Set("Content-Type", "application/json") resp.Write([]byte("{}")) logger.Info("timer", "Client Update complete", util.JsMap{"uaid": uaid, "channelID": appid, "duration": time.Now().Sub(timer).Nanoseconds()}) // Ping the appropriate server if client, ok := Clients[uaid]; ok { Flush(client) } return }
func New(opts util.JsMap, logger *util.HekaLogger) *Storage { config = opts var ok bool var err error if configEndpoint, ok := config["elasticache.config_endpoint"]; ok { memcacheEndpoint, err := getElastiCacheEndpointsTimeout(configEndpoint.(string), 2) if err == nil { config["memcache.server"] = memcacheEndpoint } else { fmt.Println(err) if logger != nil { logger.Error("storage", "Elastisearch error.", util.Fields{"error": err.Error()}) } } } if _, ok = config["memcache.server"]; !ok { config["memcache.server"] = "127.0.0.1:11211" } timeout, err := time.ParseDuration(util.MzGet(config, "db.handle_timeout", "5s")) if err != nil { if logger != nil { logger.Error("storage", "Could not parse db.handle_timeout", util.Fields{"err": err.Error()}) } timeout = 10 * time.Second } if _, ok = config["memcache.pool_size"]; !ok { config["memcache.pool_size"] = "100" } if config["memcache.pool_size"], err = strconv.ParseInt(config["memcache.pool_size"].(string), 0, 0); err != nil { config["memcache.pool_size"] = 100 } poolSize := int(config["memcache.pool_size"].(int64)) if _, ok = config["db.timeout_live"]; !ok { config["db.timeout_live"] = "259200" } if _, ok = config["db.timeout_reg"]; !ok { config["db.timeout_reg"] = "10800" } if _, ok = config["db.timeout_del"]; !ok { config["db.timeout_del"] = "86400" } if _, ok = config["shard.default_host"]; !ok { config["shard.default_host"] = "localhost" } if _, ok = config["shard.current_host"]; !ok { config["shard.current_host"] = config["shard.default_host"] } if _, ok = config["shard.prefix"]; !ok { config["shard.prefix"] = "_h-" } /* i, err := strconv.ParseInt(util.MzGet(config, "memcache.max_pool_size", "400"), 0, 10) if err != nil { if logger != nil { logger.Error("storage", "Could not parse memcache.max_pool_size", util.Fields{"error": err.Error()}) } mcsMaxPoolSize = 400 } mcsMaxPoolSize = int32(i) */ if logger != nil { logger.Info("storage", "Creating new gomc handler", nil) } // do NOT include any spaces servers := strings.Split( no_whitespace.Replace(config["memcache.server"].(string)), ",") mcs := make(chan gomc.Client, poolSize) for i := 0; i < poolSize; i++ { mcs <- newMC(servers, config, logger) } return &Storage{ mcs: mcs, config: config, logger: logger, mc_timeout: timeout, servers: servers, } }