Example #1
0
func TestGetAllTimestamps(t *testing.T) {
	filePathFormat := "testdata/distributed_stars_%d.json"
	expectedTimestamps := 16
	maxPage := 4
	batch := 5

	// Create worker queue
	maxQueue := 4
	maxWorker := 4
	dispatch := service.NewDispatcher(maxWorker, maxQueue)
	dispatch.Run()

	var serverURL string
	handler := func(w http.ResponseWriter, r *http.Request) {
		var page int
		pageString := r.FormValue("page")
		if pageString != "" {
			var err error
			page, err = strconv.Atoi(pageString)
			if err != nil {
				t.Errorf("Parsing error of %s: %v\n", pageString, err)
				w.WriteHeader(http.StatusInternalServerError)
				return
			}
		}
		filePath := fmt.Sprintf(filePathFormat, page)
		body, err := ioutil.ReadFile(filePath)
		if err != nil {
			t.Errorf("Reading error of %s: %v\n", filePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		linkFormat := github.BuildLinksFormat(serverURL)
		w.Header().Add("Link", fmt.Sprintf(linkFormat, page, maxPage))
		w.Write(body)
	}

	server := httptest.NewServer(http.HandlerFunc(handler))
	serverURL = server.URL + "?per_page=" + strconv.Itoa(batch)
	repoInfo := mockRepoInfo{
		count: expectedTimestamps,
		url:   serverURL,
	}
	timestamps, err := GetAllTimestamps(dispatch.JobQueue, batch, "token", repoInfo)

	if err != nil {
		dispatch.Stop()
		t.Fatalf("An error occured in GetAllTimestamps %v\n", err)
	}

	timestampCount := len(timestamps)
	if timestampCount != expectedTimestamps {
		dispatch.Stop()
		t.Fatalf("Expected %d timestamps and got %d", expectedTimestamps, timestampCount)
	}
	dispatch.Stop()
}
Example #2
0
// GetTimestamps gets the timestamps of the stars from the Github API
// count is the number of pages to query
// url is the Github API url of the repository you want to crawl
// token is the Github API token
func GetTimestamps(perPage int, url, token string) (timestamps []int64, err error) {
	if perPage > 0 {
		url = url + "?per_page=" + strconv.Itoa(perPage)
	}
	linkFormat := github.BuildLinksFormat(url)

	getParam := "?page="
	if strings.Contains(url, "?") {
		getParam = "&page="
	}
	var i = 1
	var last = 2
	var next int
	for {
		pageURL := url + getParam + strconv.Itoa(i)
		var stargazers []github.Stargazer
		var linkHeader string
		stargazers, linkHeader, err = github.GetStargazers(pageURL, token)
		if err != nil {
			return
		}

		var timestamp int64
		for _, star := range stargazers {
			timestamp, err = star.GetTimestamp()
			if err != nil {
				return
			}
			timestamps = append(timestamps, timestamp)
		}

		// If the header is empty, it is the only page
		if linkHeader == "" {
			break
		}
		// This is a little check because the last call will return a Link header
		// that doesn't have the same format
		if i < last {
			_, err = fmt.Sscanf(linkHeader, linkFormat, &next, &last)
			if err != nil {
				err = fmt.Errorf("An error occured while parsing the header: %v, parser is %s, link header is %s", err, linkFormat, linkHeader)
				return
			}
		}

		if i == last {
			break
		}
		i++
	}
	return timestamps, nil
}
Example #3
0
func TestGetTimestampsMulPages(t *testing.T) {
	filePathFormat := "testdata/mul_stars_%d.json"
	expectedTimestamps := []int64{1446285600, 1446289200, 1446292800}
	maxPage := 3
	batch := 3
	serverUrl := ""
	handler := func(w http.ResponseWriter, r *http.Request) {
		page := 1
		pageString := r.FormValue("page")
		if pageString != "" {
			var err error
			page, err = strconv.Atoi(pageString)
			if err != nil {
				t.Errorf("Parsing error of %s: %v\n", pageString, err)
				w.WriteHeader(http.StatusInternalServerError)
				return
			}
		}
		filePath := fmt.Sprintf(filePathFormat, page)
		body, err := ioutil.ReadFile(filePath)
		if err != nil {
			t.Errorf("Reading error of %s: %v\n", filePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		linkFormat := github.BuildLinksFormat(serverUrl)
		w.Header().Add("Link", fmt.Sprintf(linkFormat, page, maxPage))
		w.Write(body)
	}
	server := httptest.NewServer(http.HandlerFunc(handler))
	serverUrl = server.URL + "?per_page=" + strconv.Itoa(batch)
	timestamps, err := GetTimestamps(batch, server.URL, "token")
	if err != nil {
		t.Fatalf("An error occured while requesting the timestamps: %v\n", err)
	}
	if len(timestamps) != len(expectedTimestamps) {
		t.Fatalf("The expected timestamps %v and the actual ones %v"+
			" don't have the same size\n", expectedTimestamps, timestamps)
	}
	equals := true
	for i, v := range timestamps {
		if expectedTimestamps[i] != v {
			equals = false
			break
		}
	}
	if !equals {
		t.Fatalf("The expected timestamps %v and the actual ones %v"+
			" don't have the same values\n", expectedTimestamps, timestamps)
	}
}
Example #4
0
func TestGetTimestampsSimple(t *testing.T) {
	filePath := "testdata/simple_stars.json"
	expectedTimestamps := []int64{1446285600, 1446289200, 1446292800}
	body, err := ioutil.ReadFile(filePath)
	batch := 1
	if err != nil {
		t.Fatalf("An error occured while reading the file %s: %v\n", filePath, err)
	}

	serverUrl := ""
	handler := func(w http.ResponseWriter, r *http.Request) {
		linkFormat := github.BuildLinksFormat(serverUrl)
		w.Header().Add("Link", fmt.Sprintf(linkFormat, 1, 1))
		w.WriteHeader(http.StatusOK)
		w.Write(body)
	}
	server := httptest.NewServer(http.HandlerFunc(handler))
	serverUrl = server.URL + "?per_page=" + strconv.Itoa(batch)

	timestamps, err := GetTimestamps(1, server.URL, "token")
	if err != nil {
		t.Fatalf("An error occured while requesting the timestamps: %v\n", err)
	}
	if len(timestamps) != len(expectedTimestamps) {
		t.Fatalf("The expected timestamps %v and the actual ones %v"+
			" don't have the same size\n", expectedTimestamps, timestamps)
	}
	equals := true
	for i, v := range timestamps {
		if expectedTimestamps[i] != v {
			equals = false
			break
		}
	}
	if !equals {
		t.Fatalf("The expected timestamps %v and the actual ones %v"+
			" don't have the same values\n", expectedTimestamps, timestamps)
	}
}