func getCaches(u utils.Utils, caches *sliceCache) error {
if caches == nil || len(*caches) < 1 {
return nil
}
items := make([]utils.MemoryItem, len(*caches))
for index, cache := range *caches {
items[index] = utils.MemoryItem{Key: getCacheKey(cache)}
}
if err := u.MemoryGet(&items); err != nil {
u.Errorf("u.MemoryGet(%d) error %v", len(items), err)
return err
}
for index, item := range items {
parts := strings.Split(item.Value, ";")
if count, err := strconv.ParseInt(parts[0], 10, 64); err == nil {
(*caches)[index].Count = count
}
if len(parts) >= 2 {
// value with expire timestamp
if exp, expErr := strconv.ParseInt(parts[1], 10, 64); expErr == nil {
(*caches)[index].TtlLeft = exp - time.Now().Unix()
}
}
utils.Verbosef(u, "services.getCaches caches[%d] = %v", index, &(*caches)[index])
}
return nil
}
func setCaches(u utils.Utils, caches *sliceCache) {
if caches == nil || len(*caches) < 1 {
return
}
items := make([]utils.MemoryItem, len(*caches))
ttlMemory := utils.ConfigGetIntWithDefault(u, "TTL_MEMORY", 86400)
ttlDefault := utils.ConfigGetTtlDefault(u)
ttlRestricted := utils.ConfigGetIntWithDefault(u, "TTL_RESTRICTED", 60)
for index, cache := range *caches {
valueTtl := ttlDefault
if cache.Count < 1 {
valueTtl = ttlRestricted
}
items[index] = utils.MemoryItem{
Key: getCacheKey(cache),
Value: fmt.Sprintf("%d;%d", cache.Count, time.Now().Unix()+valueTtl),
Ttl: ttlMemory,
}
}
if err := u.MemorySet(&items); err != nil {
u.Errorf("u.MemorySet(%d) error %v", len(items), err)
}
}
func getAdsAsJson(u utils.Utils) string {
json, err := json.Marshal(u.ConfigGet("ADS"))
if err != nil {
return "''"
}
return string(json)
}
func DataJson(u utils.Utils, w http.ResponseWriter, r *http.Request, oneUrl bool) {
_, countsJson, err := getCountsJson(u, r, oneUrl)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
u.Errorf("web.DataJson: getCountsJson error %v", err)
return
}
writeJson(u, w, r, countsJson)
}
func prepareFbGraphUrl(u utils.Utils, ids string) string {
scheme := "http"
accessToken := ""
if appId := u.ConfigGet("FACEBOOK_APP_ID"); appId != "" {
if appSecret := u.ConfigGet("FACEBOOK_APP_SECRET"); appSecret != "" {
scheme = "https"
accessToken = fmt.Sprintf("&access_token=%s|%s", appId, appSecret)
}
}
return fmt.Sprintf("%s://graph.facebook.com/?ids=%s&fields=share%s", scheme, neturl.QueryEscape(ids), accessToken)
}
func scheduleRefreshIfNeeded(u utils.Utils) {
if dataNeedRefresh == nil {
return
}
if dataNeedRefreshCount < utils.ConfigGetIntWithDefault(u, "REFRESH_BATCH_SIZE", 20) {
return
}
u.Delay(DELAY_HANDLER_NAME_REFRESH, *dataNeedRefresh)
dataNeedRefresh = nil
dataNeedRefreshCount = 0
}
func googleLegacy(u utils.Utils, url string) result {
start := time.Now()
var res result
urlJson, err := json.Marshal(url)
if err != nil {
res.Error = err
return res
}
// http://bradsknutson.com/blog/get-google-share-count-url/
body := `[{"method":"pos.plusones.get","id":"p","params":{"nolog":true,"id":` + string(urlJson) + `,"source":"widget","userId":"@viewer","groupId":"@self"},"jsonrpc":"2.0","key":"p","apiVersion":"v1"}]`
req, err := http.NewRequest("POST", "https://clients6.google.com/rpc", bytes.NewBufferString(body))
if err != nil {
res.Error = err
return res
}
utils.Verbosef(u, "Calling http.Client.Do(body=%s)", body)
req.Header.Set("Content-Type", "application/json")
resp, err := u.HttpClient().Do(req)
if err != nil {
res.Error = err
return res
}
defer resp.Body.Close()
respBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
res.Error = err
return res
}
res.Response = respBody
u.Debugf("googleLegacy(url=%s) took %s", url, time.Since(start))
jsonparser.ArrayEach(respBody, func(element []byte, _ jsonparser.ValueType, _ int, err error) {
if err != nil {
res.Error = err
return
}
if count, err := jsonparser.GetFloat(element, "result", "metadata", "globalCounts", "count"); err != nil {
res.Error = err
return
} else {
res.Count = int64(count)
}
})
return res
}
func JqueryPluginJs(u utils.Utils, w http.ResponseWriter, r *http.Request) {
jsData, err := ioutil.ReadFile("private/js/jquery.plugin.js")
if err != nil {
w.WriteHeader(http.StatusNotFound)
u.Errorf("web.JqueryPluginJs: ReadFile error %v", err)
return
}
jsonUrl := fmt.Sprintf("//%s/js/data.json", r.Host)
js := strings.Replace(string(jsData), "{jsonUrl}", jsonUrl, 1)
js = strings.Replace(js, "{ads}", getAdsAsJson(u), 1)
writeJs(u, w, r, js)
}
func getBlacklist(u utils.Utils, refresh bool) *regexp.Regexp {
if !blacklistPrepared || refresh {
if value := u.ConfigGet("BLACKLIST"); value != "" {
compiled, err := regexp.Compile(value)
if err != nil {
u.Errorf("web.getBlacklist error on %s: %v", value, err)
}
blacklist = compiled
}
blacklistPrepared = true
}
return blacklist
}
func getWhitelist(u utils.Utils, refresh bool) *regexp.Regexp {
if !whitelistPrepared || refresh {
if value := u.ConfigGet("WHITELIST"); value != "" {
compiled, err := regexp.Compile(value)
if err != nil {
u.Errorf("web.getWhitelist error on %s: %v", value, err)
}
whitelist = compiled
}
whitelistPrepared = true
}
return whitelist
}
func buildRequests(u utils.Utils, data *MapUrlServiceCount, handleNoValueOnly bool) MapServiceRequest {
utils.Verbosef(u, "services.buildRequests(%s, %s)", data, handleNoValueOnly)
requests := make(MapServiceRequest)
caches := make(sliceCache, 0)
for url, services := range *data {
for service, count := range services {
if handleNoValueOnly && count != COUNT_NO_VALUE {
continue
}
worker, ok := workers[service]
if !ok {
u.Errorf("services.buildRequests: Unrecognized service %s", service)
continue
}
// temporary mark the cached count as fresh to avoid other process
// also trying to refresh it, we will take care of it later
temporaryCount := count
if temporaryCount == COUNT_NO_VALUE {
temporaryCount = COUNT_INITIAL_VALUE
}
caches = append(caches, cache{Service: service, Url: url, Count: temporaryCount})
if req, ok := requests[service]; ok {
req.Urls = append(req.Urls, url)
requests[service] = req
} else {
var newReq request
newReq.Service = service
newReq.Worker = worker
newReq.Urls = []string{url}
newReq.Results = make(MapUrlResult)
requests[service] = newReq
}
}
}
setCaches(u, &caches)
return requests
}
func facebookWorker(u utils.Utils, req *request) {
start := time.Now()
urls := strings.Join(req.Urls, ",")
url := prepareFbGraphUrl(u, urls)
utils.Verbosef(u, "Calling http.Client.Get(%s)", url)
resp, err := u.HttpClient().Get(url)
if err != nil {
req.Error = err
return
}
defer resp.Body.Close()
respBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
req.Error = err
return
}
req.Response = respBody
u.Debugf("facebookWorker(urls=%s) took %s", urls, time.Since(start))
for _, url := range req.Urls {
var res result
if respUrl, _, _, err := jsonparser.Get(respBody, url); err != nil {
res.Error = err
res.Response = respBody
} else {
res.Response = respUrl
if shareCount, err := jsonparser.GetInt(respUrl, "share", "share_count"); err != nil {
// it's alright, for new urls Facebook does not return share.share_count at all
res.Count = COUNT_INITIAL_VALUE
} else {
res.Count = shareCount
}
}
req.Results[url] = res
}
return
}
func getCountsJson(u utils.Utils, r *http.Request, oneUrl bool) (string, string, error) {
requestedUrls := parseUrls(r)
requestedServices := parseServices(r)
data := services.DataSetup()
if len(requestedUrls) < 1 {
return "", "{}", nil
}
url := ""
if oneUrl {
url = requestedUrls[0]
requestedUrls = []string{url}
}
for _, requestedUrl := range requestedUrls {
if !RulesAllowUrl(u, requestedUrl) {
u.Errorf("Url not allowed %s", requestedUrl)
continue
}
for _, requestedService := range requestedServices {
services.DataAdd(&data, requestedService, requestedUrl)
}
}
services.FillData(u, &data)
var dataByte []byte
var dataErr error
if oneUrl {
dataByte, dataErr = json.Marshal(data[url])
} else {
dataByte, dataErr = json.Marshal(data)
}
if dataErr != nil {
return url, "{}", dataErr
}
return url, string(dataByte), nil
}
func ConfigPost(u utils.Utils, w http.ResponseWriter, r *http.Request) {
specifiedSecret := parseSecret(r)
configSecret := os.Getenv("CONFIG_SECRET")
if configSecret == "" || specifiedSecret != configSecret {
w.WriteHeader(http.StatusForbidden)
u.Errorf("admin.ConfigGet: wrong secret %s", specifiedSecret)
return
}
r.ParseForm()
if keys, ok := r.PostForm["key"]; ok {
for _, key := range keys {
if len(key) < 1 {
continue
}
if values, ok := r.PostForm[key]; ok {
for _, value := range values {
if err := u.ConfigSet(key, value); err != nil {
w.WriteHeader(http.StatusInternalServerError)
u.Errorf("admin.ConfigSet: u.ConfigSet(%q, %q) error %v", key, value, err)
return
}
}
}
}
}
RulesRefresh(u)
w.WriteHeader(http.StatusAccepted)
}
func ConfigGet(u utils.Utils, w http.ResponseWriter, r *http.Request) {
specifiedSecret := parseSecret(r)
configSecret := os.Getenv("CONFIG_SECRET")
if configSecret == "" || specifiedSecret != configSecret {
w.WriteHeader(http.StatusForbidden)
writeJson(u, w, r, "{}")
u.Errorf("admin.ConfigGet: wrong secret %s", specifiedSecret)
return
}
keys := parseKeys(r)
values := make(map[string]string)
for _, key := range keys {
values[key] = u.ConfigGet(key)
}
valuesJson, err := json.Marshal(values)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
writeJson(u, w, r, "{}")
u.Errorf("admin.ConfigGet: json.Marshal(values) error %v", err)
return
}
writeJson(u, w, r, string(valuesJson))
}
func fillDataFromCache(u utils.Utils, data *MapUrlServiceCount, handleNoValueOnly bool) {
caches := make(sliceCache, 0)
for url, services := range *data {
for service, count := range services {
if handleNoValueOnly && count != COUNT_NO_VALUE {
continue
}
caches = append(caches, cache{Service: service, Url: url, Count: COUNT_NO_VALUE})
}
}
if err := getCaches(u, &caches); err != nil {
u.Errorf("services.getCaches(%d) error %v", len(caches), err)
} else {
for _, cache := range caches {
(*data)[cache.Url][cache.Service] = cache.Count
}
checkCachesForRefresh(u, &caches)
}
}
func twitterLegacy(u utils.Utils, url string) result {
var res result
start := time.Now()
oscUrl := "http://opensharecount.com/count.json?url=" + neturl.QueryEscape(url)
utils.Verbosef(u, "Calling http.Client.Get(%s)", oscUrl)
resp, err := u.HttpClient().Get(oscUrl)
if err != nil {
res.Error = err
return res
}
defer resp.Body.Close()
respBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
res.Error = err
return res
}
res.Response = respBody
u.Debugf("twitterLegacy(url=%s) took %s", url, time.Since(start))
if count, err := jsonparser.GetInt(respBody, "count"); err != nil {
res.Error = err
return res
} else {
if count == 0 {
if oscError, err := jsonparser.GetString(respBody, "error"); err != nil {
res.Error = errors.New(oscError)
return res
}
}
res.Count = count
}
return res
}
func AllJs(u utils.Utils, w http.ResponseWriter, r *http.Request) {
url, countsJson, err := getCountsJson(u, r, true)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
u.Errorf("web.AllJs: getCountsJson error %v", err)
return
}
urlByte, err := json.Marshal(url)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
u.Errorf("web.AllJs: json.Marshal(url) error %v", err)
return
}
jsData, err := ioutil.ReadFile("private/js/all.js")
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
u.Errorf("web.AllJs: ReadFile error %v", err)
return
}
js := string(jsData)
js = strings.Replace(js, "{url}", string(urlByte), 1)
js = strings.Replace(js, "{now}", fmt.Sprintf("%v", time.Now()), 1)
// keep using css.MinifyFromFile because it does the data uri inline for us
// TODO: drop this github.com/daohoangson/go-minify/css dependency
css := css.MinifyFromFile("public/css/main.css")
css = MinifyCss(css)
js = strings.Replace(js, "{css}", css, 1)
js = strings.Replace(js, "{facebooksvg}", readSvgAsJson("private/img/facebook.svg"), 1)
js = strings.Replace(js, "{twittersvg}", readSvgAsJson("private/img/twitter.svg"), 1)
js = strings.Replace(js, "{googlesvg}", readSvgAsJson("private/img/google.svg"), 1)
js = strings.Replace(js, "{ads}", getAdsAsJson(u), 1)
js = strings.Replace(js, "{counts}", string(countsJson), 1)
js = strings.Replace(js, "{shorten}", parseShortenAsBool(r), 1)
js = strings.Replace(js, "{target}", parseTargetAsJson(r), 1)
writeJs(u, w, r, js)
}
func HistoryJson(u utils.Utils, w http.ResponseWriter, r *http.Request) {
requestedUrls := parseUrls(r)
url := requestedUrls[0]
records, err := u.HistoryLoad(url)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
u.Errorf("web.HistoryJson: HistoryLoad error %v", err)
return
}
data := make(map[string]HistorySlot)
slotSize := int64(300) // each slot lasts 5 minutes
for _, record := range records {
slotInt64 := record.Time.Unix() / slotSize * slotSize
slotString := fmt.Sprintf("%d", slotInt64)
if slot, ok := data[slotString]; ok {
if _, ok := slot.Counts[record.Service]; !ok {
slot.Counts[record.Service] = record.Count
slot.Total += record.Count
data[slotString] = slot
}
} else {
data[slotString] = HistorySlot{
Time: time.Unix(slotInt64, 0).Format(time.RFC1123),
Counts: map[string]int64{record.Service: record.Count},
Total: record.Count,
}
}
}
historyJson, err := json.Marshal(data)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
u.Errorf("web.HistoryJson: json.Marshal error %v", err)
return
}
writeJson(u, w, r, string(historyJson))
}
func RulesAllowUrl(u utils.Utils, url string) bool {
if !getBasic().MatchString(url) {
u.Debugf("web.RulesAllowUrl: %s is not a valid url", url)
return false
}
if wl := getWhitelist(u, false); wl != nil {
if !wl.MatchString(url) {
u.Debugf("web.RulesAllowUrl: %s is not whitelisted", url)
return false
}
}
if bl := getBlacklist(u, false); bl != nil {
if bl.MatchString(url) {
u.Debugf("web.RulesAllowUrl: %s is blacklisted", url)
return false
}
}
return true
}
func executeRequests(u utils.Utils, requests *MapServiceRequest, data *MapUrlServiceCount) {
utils.Verbosef(u, "services.executeRequests(%s)", requests)
if len(*requests) < 1 {
return
}
var wg sync.WaitGroup
var cacheWg sync.WaitGroup
cacheCh := make(chan cache, 1)
caches := make(sliceCache, 0)
var historyWg sync.WaitGroup
historyTime := time.Now()
historyCh := make(chan utils.HistoryRecord, 1)
histories := make([]utils.HistoryRecord, 0)
wg.Add(len(*requests))
for _, req := range *requests {
go func(req request) {
defer wg.Done()
req.Worker(u, &req)
if req.Error != nil {
u.Errorf("services.%s: %v", req.Service, req.Error)
}
for url, res := range req.Results {
oldCount, _ := (*data)[url][req.Service]
(*data)[url][req.Service] = res.Count
if res.Error != nil {
u.Errorf("services.%s: %s error %v response %s", req.Service, url, res.Error, res.Response)
} else {
if res.Count > COUNT_INITIAL_VALUE && res.Count > oldCount {
cacheWg.Add(1)
cacheCh <- cache{Service: req.Service, Url: url, Count: res.Count}
historyWg.Add(1)
historyCh <- utils.HistoryRecord{Service: req.Service, Url: url, Count: res.Count, Time: historyTime}
}
}
}
}(req)
}
go func() {
for cache := range cacheCh {
caches = append(caches, cache)
cacheWg.Done()
}
}()
go func() {
for history := range historyCh {
histories = append(histories, history)
historyWg.Done()
}
}()
wg.Wait()
cacheWg.Wait()
historyWg.Wait()
setCaches(u, &caches)
if err := u.HistorySave(&histories); err != nil {
u.Errorf("u.HistorySave error %v", err)
}
}
func RulesRefresh(u utils.Utils) {
getWhitelist(u, true)
getBlacklist(u, true)
u.Debugf("web.RulesReset ok")
}