func TestGetAllTimestamps(t *testing.T) {
filePathFormat := "testdata/distributed_stars_%d.json"
expectedTimestamps := 16
maxPage := 4
batch := 5
// Create worker queue
maxQueue := 4
maxWorker := 4
dispatch := service.NewDispatcher(maxWorker, maxQueue)
dispatch.Run()
var serverURL string
handler := func(w http.ResponseWriter, r *http.Request) {
var page int
pageString := r.FormValue("page")
if pageString != "" {
var err error
page, err = strconv.Atoi(pageString)
if err != nil {
t.Errorf("Parsing error of %s: %v\n", pageString, err)
w.WriteHeader(http.StatusInternalServerError)
return
}
}
filePath := fmt.Sprintf(filePathFormat, page)
body, err := ioutil.ReadFile(filePath)
if err != nil {
t.Errorf("Reading error of %s: %v\n", filePath, err)
w.WriteHeader(http.StatusInternalServerError)
return
}
linkFormat := github.BuildLinksFormat(serverURL)
w.Header().Add("Link", fmt.Sprintf(linkFormat, page, maxPage))
w.Write(body)
}
server := httptest.NewServer(http.HandlerFunc(handler))
serverURL = server.URL + "?per_page=" + strconv.Itoa(batch)
repoInfo := mockRepoInfo{
count: expectedTimestamps,
url: serverURL,
}
timestamps, err := GetAllTimestamps(dispatch.JobQueue, batch, "token", repoInfo)
if err != nil {
dispatch.Stop()
t.Fatalf("An error occured in GetAllTimestamps %v\n", err)
}
timestampCount := len(timestamps)
if timestampCount != expectedTimestamps {
dispatch.Stop()
t.Fatalf("Expected %d timestamps and got %d", expectedTimestamps, timestampCount)
}
dispatch.Stop()
}
// GetTimestamps gets the timestamps of the stars from the Github API
// count is the number of pages to query
// url is the Github API url of the repository you want to crawl
// token is the Github API token
func GetTimestamps(perPage int, url, token string) (timestamps []int64, err error) {
if perPage > 0 {
url = url + "?per_page=" + strconv.Itoa(perPage)
}
linkFormat := github.BuildLinksFormat(url)
getParam := "?page="
if strings.Contains(url, "?") {
getParam = "&page="
}
var i = 1
var last = 2
var next int
for {
pageURL := url + getParam + strconv.Itoa(i)
var stargazers []github.Stargazer
var linkHeader string
stargazers, linkHeader, err = github.GetStargazers(pageURL, token)
if err != nil {
return
}
var timestamp int64
for _, star := range stargazers {
timestamp, err = star.GetTimestamp()
if err != nil {
return
}
timestamps = append(timestamps, timestamp)
}
// If the header is empty, it is the only page
if linkHeader == "" {
break
}
// This is a little check because the last call will return a Link header
// that doesn't have the same format
if i < last {
_, err = fmt.Sscanf(linkHeader, linkFormat, &next, &last)
if err != nil {
err = fmt.Errorf("An error occured while parsing the header: %v, parser is %s, link header is %s", err, linkFormat, linkHeader)
return
}
}
if i == last {
break
}
i++
}
return timestamps, nil
}
func TestGetTimestampsMulPages(t *testing.T) {
filePathFormat := "testdata/mul_stars_%d.json"
expectedTimestamps := []int64{1446285600, 1446289200, 1446292800}
maxPage := 3
batch := 3
serverUrl := ""
handler := func(w http.ResponseWriter, r *http.Request) {
page := 1
pageString := r.FormValue("page")
if pageString != "" {
var err error
page, err = strconv.Atoi(pageString)
if err != nil {
t.Errorf("Parsing error of %s: %v\n", pageString, err)
w.WriteHeader(http.StatusInternalServerError)
return
}
}
filePath := fmt.Sprintf(filePathFormat, page)
body, err := ioutil.ReadFile(filePath)
if err != nil {
t.Errorf("Reading error of %s: %v\n", filePath, err)
w.WriteHeader(http.StatusInternalServerError)
return
}
linkFormat := github.BuildLinksFormat(serverUrl)
w.Header().Add("Link", fmt.Sprintf(linkFormat, page, maxPage))
w.Write(body)
}
server := httptest.NewServer(http.HandlerFunc(handler))
serverUrl = server.URL + "?per_page=" + strconv.Itoa(batch)
timestamps, err := GetTimestamps(batch, server.URL, "token")
if err != nil {
t.Fatalf("An error occured while requesting the timestamps: %v\n", err)
}
if len(timestamps) != len(expectedTimestamps) {
t.Fatalf("The expected timestamps %v and the actual ones %v"+
" don't have the same size\n", expectedTimestamps, timestamps)
}
equals := true
for i, v := range timestamps {
if expectedTimestamps[i] != v {
equals = false
break
}
}
if !equals {
t.Fatalf("The expected timestamps %v and the actual ones %v"+
" don't have the same values\n", expectedTimestamps, timestamps)
}
}
func TestGetTimestampsSimple(t *testing.T) {
filePath := "testdata/simple_stars.json"
expectedTimestamps := []int64{1446285600, 1446289200, 1446292800}
body, err := ioutil.ReadFile(filePath)
batch := 1
if err != nil {
t.Fatalf("An error occured while reading the file %s: %v\n", filePath, err)
}
serverUrl := ""
handler := func(w http.ResponseWriter, r *http.Request) {
linkFormat := github.BuildLinksFormat(serverUrl)
w.Header().Add("Link", fmt.Sprintf(linkFormat, 1, 1))
w.WriteHeader(http.StatusOK)
w.Write(body)
}
server := httptest.NewServer(http.HandlerFunc(handler))
serverUrl = server.URL + "?per_page=" + strconv.Itoa(batch)
timestamps, err := GetTimestamps(1, server.URL, "token")
if err != nil {
t.Fatalf("An error occured while requesting the timestamps: %v\n", err)
}
if len(timestamps) != len(expectedTimestamps) {
t.Fatalf("The expected timestamps %v and the actual ones %v"+
" don't have the same size\n", expectedTimestamps, timestamps)
}
equals := true
for i, v := range timestamps {
if expectedTimestamps[i] != v {
equals = false
break
}
}
if !equals {
t.Fatalf("The expected timestamps %v and the actual ones %v"+
" don't have the same values\n", expectedTimestamps, timestamps)
}
}