func TestGetAllTimestamps(t *testing.T) { filePathFormat := "testdata/distributed_stars_%d.json" expectedTimestamps := 16 maxPage := 4 batch := 5 // Create worker queue maxQueue := 4 maxWorker := 4 dispatch := service.NewDispatcher(maxWorker, maxQueue) dispatch.Run() var serverURL string handler := func(w http.ResponseWriter, r *http.Request) { var page int pageString := r.FormValue("page") if pageString != "" { var err error page, err = strconv.Atoi(pageString) if err != nil { t.Errorf("Parsing error of %s: %v\n", pageString, err) w.WriteHeader(http.StatusInternalServerError) return } } filePath := fmt.Sprintf(filePathFormat, page) body, err := ioutil.ReadFile(filePath) if err != nil { t.Errorf("Reading error of %s: %v\n", filePath, err) w.WriteHeader(http.StatusInternalServerError) return } linkFormat := github.BuildLinksFormat(serverURL) w.Header().Add("Link", fmt.Sprintf(linkFormat, page, maxPage)) w.Write(body) } server := httptest.NewServer(http.HandlerFunc(handler)) serverURL = server.URL + "?per_page=" + strconv.Itoa(batch) repoInfo := mockRepoInfo{ count: expectedTimestamps, url: serverURL, } timestamps, err := GetAllTimestamps(dispatch.JobQueue, batch, "token", repoInfo) if err != nil { dispatch.Stop() t.Fatalf("An error occured in GetAllTimestamps %v\n", err) } timestampCount := len(timestamps) if timestampCount != expectedTimestamps { dispatch.Stop() t.Fatalf("Expected %d timestamps and got %d", expectedTimestamps, timestampCount) } dispatch.Stop() }
// GetTimestamps gets the timestamps of the stars from the Github API // count is the number of pages to query // url is the Github API url of the repository you want to crawl // token is the Github API token func GetTimestamps(perPage int, url, token string) (timestamps []int64, err error) { if perPage > 0 { url = url + "?per_page=" + strconv.Itoa(perPage) } linkFormat := github.BuildLinksFormat(url) getParam := "?page=" if strings.Contains(url, "?") { getParam = "&page=" } var i = 1 var last = 2 var next int for { pageURL := url + getParam + strconv.Itoa(i) var stargazers []github.Stargazer var linkHeader string stargazers, linkHeader, err = github.GetStargazers(pageURL, token) if err != nil { return } var timestamp int64 for _, star := range stargazers { timestamp, err = star.GetTimestamp() if err != nil { return } timestamps = append(timestamps, timestamp) } // If the header is empty, it is the only page if linkHeader == "" { break } // This is a little check because the last call will return a Link header // that doesn't have the same format if i < last { _, err = fmt.Sscanf(linkHeader, linkFormat, &next, &last) if err != nil { err = fmt.Errorf("An error occured while parsing the header: %v, parser is %s, link header is %s", err, linkFormat, linkHeader) return } } if i == last { break } i++ } return timestamps, nil }
func TestGetTimestampsMulPages(t *testing.T) { filePathFormat := "testdata/mul_stars_%d.json" expectedTimestamps := []int64{1446285600, 1446289200, 1446292800} maxPage := 3 batch := 3 serverUrl := "" handler := func(w http.ResponseWriter, r *http.Request) { page := 1 pageString := r.FormValue("page") if pageString != "" { var err error page, err = strconv.Atoi(pageString) if err != nil { t.Errorf("Parsing error of %s: %v\n", pageString, err) w.WriteHeader(http.StatusInternalServerError) return } } filePath := fmt.Sprintf(filePathFormat, page) body, err := ioutil.ReadFile(filePath) if err != nil { t.Errorf("Reading error of %s: %v\n", filePath, err) w.WriteHeader(http.StatusInternalServerError) return } linkFormat := github.BuildLinksFormat(serverUrl) w.Header().Add("Link", fmt.Sprintf(linkFormat, page, maxPage)) w.Write(body) } server := httptest.NewServer(http.HandlerFunc(handler)) serverUrl = server.URL + "?per_page=" + strconv.Itoa(batch) timestamps, err := GetTimestamps(batch, server.URL, "token") if err != nil { t.Fatalf("An error occured while requesting the timestamps: %v\n", err) } if len(timestamps) != len(expectedTimestamps) { t.Fatalf("The expected timestamps %v and the actual ones %v"+ " don't have the same size\n", expectedTimestamps, timestamps) } equals := true for i, v := range timestamps { if expectedTimestamps[i] != v { equals = false break } } if !equals { t.Fatalf("The expected timestamps %v and the actual ones %v"+ " don't have the same values\n", expectedTimestamps, timestamps) } }
func TestGetTimestampsSimple(t *testing.T) { filePath := "testdata/simple_stars.json" expectedTimestamps := []int64{1446285600, 1446289200, 1446292800} body, err := ioutil.ReadFile(filePath) batch := 1 if err != nil { t.Fatalf("An error occured while reading the file %s: %v\n", filePath, err) } serverUrl := "" handler := func(w http.ResponseWriter, r *http.Request) { linkFormat := github.BuildLinksFormat(serverUrl) w.Header().Add("Link", fmt.Sprintf(linkFormat, 1, 1)) w.WriteHeader(http.StatusOK) w.Write(body) } server := httptest.NewServer(http.HandlerFunc(handler)) serverUrl = server.URL + "?per_page=" + strconv.Itoa(batch) timestamps, err := GetTimestamps(1, server.URL, "token") if err != nil { t.Fatalf("An error occured while requesting the timestamps: %v\n", err) } if len(timestamps) != len(expectedTimestamps) { t.Fatalf("The expected timestamps %v and the actual ones %v"+ " don't have the same size\n", expectedTimestamps, timestamps) } equals := true for i, v := range timestamps { if expectedTimestamps[i] != v { equals = false break } } if !equals { t.Fatalf("The expected timestamps %v and the actual ones %v"+ " don't have the same values\n", expectedTimestamps, timestamps) } }