Example #1
0
func getCaches(u utils.Utils, caches *sliceCache) error {
	if caches == nil || len(*caches) < 1 {
		return nil
	}

	items := make([]utils.MemoryItem, len(*caches))

	for index, cache := range *caches {
		items[index] = utils.MemoryItem{Key: getCacheKey(cache)}
	}

	if err := u.MemoryGet(&items); err != nil {
		u.Errorf("u.MemoryGet(%d) error %v", len(items), err)
		return err
	}

	for index, item := range items {
		parts := strings.Split(item.Value, ";")

		if count, err := strconv.ParseInt(parts[0], 10, 64); err == nil {
			(*caches)[index].Count = count
		}

		if len(parts) >= 2 {
			// value with expire timestamp
			if exp, expErr := strconv.ParseInt(parts[1], 10, 64); expErr == nil {
				(*caches)[index].TtlLeft = exp - time.Now().Unix()
			}
		}

		utils.Verbosef(u, "services.getCaches caches[%d] = %v", index, &(*caches)[index])
	}

	return nil
}
Example #2
0
func googleLegacy(u utils.Utils, url string) result {
	start := time.Now()
	var res result

	urlJson, err := json.Marshal(url)
	if err != nil {
		res.Error = err
		return res
	}

	// http://bradsknutson.com/blog/get-google-share-count-url/
	body := `[{"method":"pos.plusones.get","id":"p","params":{"nolog":true,"id":` + string(urlJson) + `,"source":"widget","userId":"@viewer","groupId":"@self"},"jsonrpc":"2.0","key":"p","apiVersion":"v1"}]`
	req, err := http.NewRequest("POST", "https://clients6.google.com/rpc", bytes.NewBufferString(body))
	if err != nil {
		res.Error = err
		return res
	}

	utils.Verbosef(u, "Calling http.Client.Do(body=%s)", body)
	req.Header.Set("Content-Type", "application/json")
	resp, err := u.HttpClient().Do(req)
	if err != nil {
		res.Error = err
		return res
	}

	defer resp.Body.Close()
	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		res.Error = err
		return res
	}
	res.Response = respBody
	u.Debugf("googleLegacy(url=%s) took %s", url, time.Since(start))

	jsonparser.ArrayEach(respBody, func(element []byte, _ jsonparser.ValueType, _ int, err error) {
		if err != nil {
			res.Error = err
			return
		}

		if count, err := jsonparser.GetFloat(element, "result", "metadata", "globalCounts", "count"); err != nil {
			res.Error = err
			return
		} else {
			res.Count = int64(count)
		}
	})

	return res
}
Example #3
0
func buildRequests(u utils.Utils, data *MapUrlServiceCount, handleNoValueOnly bool) MapServiceRequest {
	utils.Verbosef(u, "services.buildRequests(%s, %s)", data, handleNoValueOnly)

	requests := make(MapServiceRequest)
	caches := make(sliceCache, 0)

	for url, services := range *data {
		for service, count := range services {
			if handleNoValueOnly && count != COUNT_NO_VALUE {
				continue
			}

			worker, ok := workers[service]
			if !ok {
				u.Errorf("services.buildRequests: Unrecognized service %s", service)
				continue
			}

			// temporary mark the cached count as fresh to avoid other process
			// also trying to refresh it, we will take care of it later
			temporaryCount := count
			if temporaryCount == COUNT_NO_VALUE {
				temporaryCount = COUNT_INITIAL_VALUE
			}
			caches = append(caches, cache{Service: service, Url: url, Count: temporaryCount})

			if req, ok := requests[service]; ok {
				req.Urls = append(req.Urls, url)
				requests[service] = req
			} else {
				var newReq request
				newReq.Service = service
				newReq.Worker = worker
				newReq.Urls = []string{url}
				newReq.Results = make(MapUrlResult)

				requests[service] = newReq
			}
		}
	}

	setCaches(u, &caches)

	return requests
}
func facebookWorker(u utils.Utils, req *request) {
	start := time.Now()
	urls := strings.Join(req.Urls, ",")
	url := prepareFbGraphUrl(u, urls)
	utils.Verbosef(u, "Calling http.Client.Get(%s)", url)

	resp, err := u.HttpClient().Get(url)
	if err != nil {
		req.Error = err
		return
	}

	defer resp.Body.Close()
	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		req.Error = err
		return
	}
	req.Response = respBody
	u.Debugf("facebookWorker(urls=%s) took %s", urls, time.Since(start))

	for _, url := range req.Urls {
		var res result

		if respUrl, _, _, err := jsonparser.Get(respBody, url); err != nil {
			res.Error = err
			res.Response = respBody
		} else {
			res.Response = respUrl
			if shareCount, err := jsonparser.GetInt(respUrl, "share", "share_count"); err != nil {
				// it's alright, for new urls Facebook does not return share.share_count at all
				res.Count = COUNT_INITIAL_VALUE
			} else {
				res.Count = shareCount
			}
		}

		req.Results[url] = res
	}

	return
}
Example #5
0
func twitterLegacy(u utils.Utils, url string) result {
	var res result
	start := time.Now()
	oscUrl := "http://opensharecount.com/count.json?url=" + neturl.QueryEscape(url)
	utils.Verbosef(u, "Calling http.Client.Get(%s)", oscUrl)

	resp, err := u.HttpClient().Get(oscUrl)
	if err != nil {
		res.Error = err
		return res
	}

	defer resp.Body.Close()
	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		res.Error = err
		return res
	}
	res.Response = respBody
	u.Debugf("twitterLegacy(url=%s) took %s", url, time.Since(start))

	if count, err := jsonparser.GetInt(respBody, "count"); err != nil {
		res.Error = err
		return res
	} else {
		if count == 0 {
			if oscError, err := jsonparser.GetString(respBody, "error"); err != nil {
				res.Error = errors.New(oscError)
				return res
			}
		}

		res.Count = count
	}

	return res
}
Example #6
0
func executeRequests(u utils.Utils, requests *MapServiceRequest, data *MapUrlServiceCount) {
	utils.Verbosef(u, "services.executeRequests(%s)", requests)
	if len(*requests) < 1 {
		return
	}

	var wg sync.WaitGroup
	var cacheWg sync.WaitGroup
	cacheCh := make(chan cache, 1)
	caches := make(sliceCache, 0)
	var historyWg sync.WaitGroup
	historyTime := time.Now()
	historyCh := make(chan utils.HistoryRecord, 1)
	histories := make([]utils.HistoryRecord, 0)

	wg.Add(len(*requests))
	for _, req := range *requests {
		go func(req request) {
			defer wg.Done()

			req.Worker(u, &req)
			if req.Error != nil {
				u.Errorf("services.%s: %v", req.Service, req.Error)
			}

			for url, res := range req.Results {
				oldCount, _ := (*data)[url][req.Service]
				(*data)[url][req.Service] = res.Count

				if res.Error != nil {
					u.Errorf("services.%s: %s error %v response %s", req.Service, url, res.Error, res.Response)
				} else {
					if res.Count > COUNT_INITIAL_VALUE && res.Count > oldCount {
						cacheWg.Add(1)
						cacheCh <- cache{Service: req.Service, Url: url, Count: res.Count}

						historyWg.Add(1)
						historyCh <- utils.HistoryRecord{Service: req.Service, Url: url, Count: res.Count, Time: historyTime}
					}
				}
			}
		}(req)
	}

	go func() {
		for cache := range cacheCh {
			caches = append(caches, cache)
			cacheWg.Done()
		}
	}()

	go func() {
		for history := range historyCh {
			histories = append(histories, history)
			historyWg.Done()
		}
	}()

	wg.Wait()
	cacheWg.Wait()
	historyWg.Wait()

	setCaches(u, &caches)
	if err := u.HistorySave(&histories); err != nil {
		u.Errorf("u.HistorySave error %v", err)
	}
}