Exemplo n.º 1
0
func TestGetUserId(t *testing.T) {
	im := &imp{}
	ctx, cancel := context.WithCancel(context.WithValue(context.TODO(), ctxutil.HTTPClient, &http.Client{
		Transport: httputil.NewFakeTransport(map[string]func() *http.Response{
			"https://api.foursquare.com/v2/users/self?oauth_token=footoken&v=20140225": httputil.FileResponder("testdata/users-me-res.json"),
		}),
	}))
	defer cancel()
	inf, err := im.getUserInfo(ctx, "footoken")
	if err != nil {
		t.Fatal(err)
	}
	want := user{
		Id:        "13674",
		FirstName: "Brad",
		LastName:  "Fitzpatrick",
		Photo: photoItem{
			Prefix: "https://irs0.4sqi.net/img/user/",
			Suffix: "/CKG5FOF2WMCMPD3E.jpg",
		},
	}
	if inf != want {
		t.Errorf("user info = %+v; want %+v", inf, want)
	}
}
Exemplo n.º 2
0
func (im *imp) MakeTestData() http.RoundTripper {

	const (
		apiURL        = "https://picasaweb.google.com/data/feed/api"
		nAlbums       = 10 // Arbitrary number of albums generated.
		nEntries      = 3  // number of albums or photos returned in the feed at each call.
		defaultUserId = "default"
	)

	albumsListCached := make(map[int]string)
	okHeader := `HTTP/1.1 200 OK
Content-Type: application/json; charset=UTF-8

`

	responses := make(map[string]func() *http.Response)

	// register the get albums list calls
	for i := 1; i < nAlbums+1; i += nEntries {
		url := fmt.Sprintf("%s/user/%s?start-index=%d", apiURL, defaultUserId, i)
		response := okHeader + fakeAlbumsList(i, nAlbums, nEntries, albumsListCached)
		responses[url] = httputil.StaticResponder(response)
	}

	// register the get album calls
	for i := 1; i < nAlbums+1; i++ {
		albumId := blob.RefFromString(fmt.Sprintf("Album %d", i)).DigestPrefix(10)
		for j := 1; j < i+1; j += nEntries {
			url := fmt.Sprintf("%s/user/%s/albumid/%s?imgmax=d&start-index=%d", apiURL, defaultUserId, albumId, j)
			// Using i as nTotal argument means album N will have N photos in it.
			response := okHeader + fakePhotosList(j, i, nEntries)
			responses[url] = httputil.StaticResponder(response)
		}
	}

	// register the photo download calls
	pudgyPic := fakePhoto()
	photoURL1 := "https://camlistore.org/pic/pudgy1.png"
	photoURL2 := "https://camlistore.org/pic/pudgy2.png"
	responses[photoURL1] = httputil.FileResponder(pudgyPic)
	responses[photoURL2] = httputil.FileResponder(pudgyPic)

	return httputil.NewFakeTransport(responses)
}
Exemplo n.º 3
0
func (im *imp) MakeTestData() http.RoundTripper {
	const (
		fakeMaxId = int64(486450108201201664) // Most recent tweet.
		nTweets   = 300                       // Arbitrary number of tweets generated.
	)
	fakeMinId := fakeMaxId - nTweets // Oldest tweet in our timeline.

	timeLineURL := apiURL + userTimeLineAPIPath
	timeLineCached := make(map[int64]string)
	okHeader := `HTTP/1.1 200 OK
Content-Type: application/json; charset=UTF-8

`
	timeLineResponse := okHeader + fakeTimeLine(fakeMaxId, fakeMinId, timeLineCached)

	fakePic := fakePicture()
	responses := map[string]func() *http.Response{
		timeLineURL: httputil.StaticResponder(timeLineResponse),
		fmt.Sprintf("%s?count=%d&user_id=fakeUserID", timeLineURL, tweetRequestLimit): httputil.StaticResponder(timeLineResponse),
		"https://twitpic.com/show/large/bar":                                          httputil.FileResponder(fakePic),
		"https://i.imgur.com/foo.gif":                                                 httputil.FileResponder(fakePic),
	}

	// register all the user_timeline calls (max_id varies) that should occur,
	responses[fmt.Sprintf("%s?count=%d&max_id=%d&user_id=fakeUserID", timeLineURL, tweetRequestLimit, fakeMaxId-nTweets+1)] = httputil.StaticResponder(okHeader + fakeTimeLine(fakeMaxId-nTweets+1, fakeMinId, timeLineCached))
	if nTweets > tweetRequestLimit {
		// that is, once every tweetRequestLimit-1, going down from fakeMaxId.
		for i := fakeMaxId; i > fakeMinId; i -= tweetRequestLimit - 1 {
			responses[fmt.Sprintf("%s?count=%d&max_id=%d&user_id=fakeUserID", timeLineURL, tweetRequestLimit, i)] = httputil.StaticResponder(okHeader + fakeTimeLine(i, fakeMinId, timeLineCached))
		}
	}

	// register all the possible combinations of media twimg
	for _, scheme := range []string{"http://", "https://"} {
		for _, picsize := range []string{"thumb", "small", "medium", "large"} {
			responses[fmt.Sprintf("%spbs.twimg.com/media/foo.jpg:%s", scheme, picsize)] = httputil.FileResponder(fakePic)
			responses[fmt.Sprintf("%spbs.twimg.com/media/bar.png:%s", scheme, picsize)] = httputil.FileResponder(fakePic)
		}
	}

	return httputil.NewFakeTransport(responses)
}
Exemplo n.º 4
0
func TestGetUserID(t *testing.T) {
	ctx := context.New(context.WithHTTPClient(&http.Client{
		Transport: httputil.NewFakeTransport(map[string]func() *http.Response{
			apiURL + userInfoAPIPath: httputil.FileResponder(filepath.FromSlash("testdata/verify_credentials-res.json")),
		}),
	}))
	defer ctx.Cancel()
	inf, err := getUserInfo(importer.OAuthContext{ctx, &oauth.Client{}, &oauth.Credentials{}})
	if err != nil {
		t.Fatal(err)
	}
	want := userInfo{
		ID:         "2325935334",
		ScreenName: "lejatorn",
		Name:       "Mathieu Lonjaret",
	}
	if inf != want {
		t.Errorf("user info = %+v; want %+v", inf, want)
	}
}
Exemplo n.º 5
0
func TestGetUserId(t *testing.T) {
	im := &imp{}
	ctx := context.New(context.WithHTTPClient(&http.Client{
		Transport: httputil.NewFakeTransport(map[string]func() *http.Response{
			"https://api.foursquare.com/v2/users/self?oauth_token=footoken&v=20140225": httputil.FileResponder("testdata/users-me-res.json"),
		}),
	}))
	defer ctx.Cancel()
	inf, err := im.getUserInfo(ctx, "footoken")
	if err != nil {
		t.Fatal(err)
	}
	want := user{
		Id:        "13674",
		FirstName: "Brad",
		LastName:  "Fitzpatrick",
	}
	if inf != want {
		t.Errorf("user info = %+v; want %+v", inf, want)
	}
}
Exemplo n.º 6
0
func TestGetUserId(t *testing.T) {
	userID := "11047045264"
	responder := httputil.FileResponder("testdata/users-me-res.xml")
	cl := &http.Client{
		Transport: httputil.NewFakeTransport(map[string]func() *http.Response{
			"https://picasaweb.google.com/data/feed/api/user/default/contacts?kind=user":        responder,
			"https://picasaweb.google.com/data/feed/api/user/" + userID + "/contacts?kind=user": responder,
		})}
	inf, err := picago.GetUser(cl, "default")
	if err != nil {
		t.Fatal(err)
	}
	want := picago.User{
		ID:        userID,
		URI:       "https://picasaweb.google.com/" + userID,
		Name:      "Tamás Gulácsi",
		Thumbnail: "https://lh4.googleusercontent.com/-qqove344/AAAAAAAAAAI/AAAAAAABcbg/TXl3f2K9dzI/s64-c/11047045264.jpg",
	}
	if inf != want {
		t.Errorf("user info = %+v; want %+v", inf, want)
	}
}
Exemplo n.º 7
0
func (im *imp) MakeTestData() http.RoundTripper {

	const nCheckins = 150 // Arbitrary number of checkins generated.

	// if you add another venue, make sure the venueCounter reset
	// in fakeCheckinsList allows for that case to happen.
	// We could use global vars instead, but don't want to pollute the
	// fousquare pkg namespace.
	towns := map[int]*venueLocationItem{
		0: {
			Address:    "Baker street",
			City:       "Dublin",
			PostalCode: "0",
			State:      "none",
			Country:    "Ireland",
			Lat:        53.4053427,
			Lng:        -8.3320801,
		},
		1: {
			Address:    "Fish&Ships street",
			City:       "London",
			PostalCode: "1",
			State:      "none",
			Country:    "England",
			Lat:        55.3617609,
			Lng:        -3.4433238,
		},
		2: {
			Address:    "Haggis street",
			City:       "Glasgow",
			PostalCode: "2",
			State:      "none",
			Country:    "Scotland",
			Lat:        57.7394571,
			Lng:        -4.686997,
		},
		3: {
			Address:    "rue du croissant",
			City:       "Grenoble",
			PostalCode: "38000",
			State:      "none",
			Country:    "France",
			Lat:        45.1841655,
			Lng:        5.7155424,
		},
		4: {
			Address:    "burrito street",
			City:       "San Francisco",
			PostalCode: "94114",
			State:      "CA",
			Country:    "US",
			Lat:        37.7593625,
			Lng:        -122.4266995,
		},
	}

	// We need to compute the venueIds in advance, because the venue id is used as a parameter
	// in some of the requests we need to register.
	var venueIds []string
	for _, v := range towns {
		venueIds = append(venueIds, blob.RefFromString(v.City).DigestPrefix(10))
	}

	checkinsURL := apiURL + checkinsAPIPath
	checkinsListCached := make(map[int]string)
	okHeader := `HTTP/1.1 200 OK
Content-Type: application/json; charset=UTF-8

`

	responses := make(map[string]func() *http.Response)

	// register all the checkins calls; offset varies.
	for i := 0; i < nCheckins; i += checkinsRequestLimit {
		url := fmt.Sprintf("%s?limit=%d&oauth_token=fakeAccessToken&offset=%d&v=%s",
			checkinsURL, checkinsRequestLimit, i, apiVersion)
		response := okHeader + fakeCheckinsList(i, nCheckins, towns, checkinsListCached)
		responses[url] = httputil.StaticResponder(response)
	}

	// register all the venue photos calls (venueId varies)
	photosURL := apiURL + "venues"
	photosResponse := okHeader + fakePhotosList()
	for _, id := range venueIds {
		url := fmt.Sprintf("%s/%s/photos?limit=%d&oauth_token=fakeAccessToken&v=%s",
			photosURL, id, photosRequestLimit, apiVersion)
		responses[url] = httputil.StaticResponder(photosResponse)
	}

	// register the photoitem calls
	pudgyPic := fakePhoto()
	photoURL := "https://camlistore.org/pic/pudgy.png"
	originalPhotoURL := "https://camlistore.org/original/pic/pudgy.png"
	iconURL := "https://camlistore.org/bg_88/pic/pudgy.png"
	responses[photoURL] = httputil.FileResponder(pudgyPic)
	responses[originalPhotoURL] = httputil.FileResponder(pudgyPic)
	responses[iconURL] = httputil.FileResponder(pudgyPic)

	return httputil.NewFakeTransport(responses)
}
Exemplo n.º 8
0
// Verify that a batch import of 3 posts works
func TestIntegrationRun(t *testing.T) {
	const importerPrefix = "/importer/"
	const authToken = "gina:foo"
	const attrKey = "key"
	const attrValue = "value"

	w := test.GetWorld(t)
	defer w.Stop()
	baseURL := w.ServerBaseURL()

	// TODO(mpl): add a utility in integration package to provide a client that
	// just works with World.
	cl, err := setupClient(w)
	if err != nil {
		t.Fatal(err)
	}
	signer, err := cl.Signer()
	if err != nil {
		t.Fatal(err)
	}
	clientId := map[string]string{
		"pinboard": "fakeStaticClientId",
	}
	clientSecret := map[string]string{
		"pinboard": "fakeStaticClientSecret",
	}

	responder := httputil.FileResponder("testdata/batchresponse.json")
	transport, err := httputil.NewRegexpFakeTransport([]*httputil.Matcher{
		&httputil.Matcher{`^https\://api\.pinboard\.in/v1/posts/all\?auth_token=gina:foo&format=json&results=10000&todt=\d\d\d\d.*`, responder},
	})
	if err != nil {
		t.Fatal(err)
	}
	httpClient := &http.Client{
		Transport: transport,
	}

	hc := importer.HostConfig{
		BaseURL:      baseURL,
		Prefix:       importerPrefix,
		Target:       cl,
		BlobSource:   cl,
		Signer:       signer,
		Search:       cl,
		ClientId:     clientId,
		ClientSecret: clientSecret,
		HTTPClient:   httpClient,
	}

	host, err := importer.NewHost(hc)
	if err != nil {
		t.Fatal(err)
	}
	rc, err := importer.CreateAccount(host, "pinboard")
	if err != nil {
		t.Fatal(err)
	}
	err = rc.AccountNode().SetAttrs(attrAuthToken, authToken)
	if err != nil {
		t.Fatal(err)
	}

	testee := imp{}
	if err := testee.Run(rc); err != nil {
		t.Fatal(err)
	}

	postsNode, err := getRequiredChildPathObj(rc.RootNode(), "posts")
	if err != nil {
		t.Fatal(err)
	}

	childRefs, err := findChildRefs(postsNode)
	if err != nil {
		t.Fatal(err)
	}

	expectedPosts := map[string]string{
		`https://wiki.archlinux.org/index.php/xorg#Display_size_and_DPI`:                   "Xorg - ArchWiki",
		`http://www.harihareswara.net/sumana/2014/08/17/0`:                                 "One Way Confidence Will Look",
		`http://www.wikiart.org/en/marcus-larson/fishing-near-the-fjord-by-moonlight-1862`: "Fishing Near The Fjord By Moonlight - Marcus Larson - WikiArt.org",
	}

	if len(childRefs) != len(expectedPosts) {
		t.Fatalf("After import, found %d child refs, want %d: %v", len(childRefs), len(expectedPosts), childRefs)
	}

	for _, ref := range childRefs {
		childNode, err := host.ObjectFromRef(ref)
		if err != nil {
			t.Fatal(err)
		}
		foundURL := childNode.Attr(nodeattr.URL)
		expectedTitle, ok := expectedPosts[foundURL]
		if !ok {
			t.Fatalf("Found unexpected child node %v with url %q", childNode, foundURL)
		}
		foundTitle := childNode.Attr(nodeattr.Title)
		if foundTitle != expectedTitle {
			t.Fatalf("Found unexpected child node %v with title %q when we want %q", childNode, foundTitle, expectedTitle)
		}
		delete(expectedPosts, foundURL)
	}
	if len(expectedPosts) != 0 {
		t.Fatalf("The following entries were expected but not found: %#v", expectedPosts)
	}
}
Exemplo n.º 9
0
func (im imp) MakeTestData() http.RoundTripper {
	const (
		nPhotosets = 5 // Arbitrary number of sets.
		perPage    = 3 // number of photos per page (both when getting sets and when getting photos).
		fakeUserId = "fakeUserId"
	)
	// Photoset N has N photos, so we've got 15 ( = 5 + 4 + 3 + 2 + 1) photos in total.
	var nPhotos int
	for i := 1; i <= nPhotosets; i++ {
		nPhotos += i
	}
	nPhotosPages := nPhotos / perPage
	if nPhotos%perPage != 0 {
		nPhotosPages++
	}

	okHeader := `HTTP/1.1 200 OK
Content-Type: application/json; charset=UTF-8

`

	// TODO(mpl): this scheme does not take into account that we could have the same photo
	// in different albums. These two photos will end up with a different photoId.
	buildPhotoIds := func(nsets, perPage int) []string {
		var ids []string
		for i := 1; i <= nsets; i++ {
			photosetId := blob.RefFromString(fmt.Sprintf("Photoset %d", i)).DigestPrefix(10)
			page := 1
			// Photoset N has N photos.
			indexOnPage := 1
			for j := 1; j <= i; j++ {
				photoId := blob.RefFromString(fmt.Sprintf("Photo %d on page %d of photoset %s", indexOnPage, page, photosetId)).DigestPrefix(10)
				ids = append(ids, photoId)
				indexOnPage++
				if indexOnPage > perPage {
					page++
					indexOnPage = 1
				}
			}
		}
		return ids
	}
	photoIds := buildPhotoIds(nPhotosets, perPage)

	responses := make(map[string]func() *http.Response)
	// Initial photo sets list
	photosetsURL := fmt.Sprintf("%s?format=json&method=%s&nojsoncallback=1&user_id=%s", apiURL, photosetsAPIPath, fakeUserId)
	response := fmt.Sprintf("%s%s", okHeader, fakePhotosetsList(nPhotosets))
	responses[photosetsURL] = httputil.StaticResponder(response)

	// All the photoset calls. One call for each page of each photoset.
	// Each page as perPage photos, or maybe less if end of the photoset.
	{
		pageStart := 0
		albumEnd, pageEnd, albumNum, pages, page := 1, 1, 1, 1, 1
		photosetId := blob.RefFromString(fmt.Sprintf("Photoset %d", albumNum)).DigestPrefix(10)
		photosURL := fmt.Sprintf("%s?extras=original_format&format=json&method=%s&nojsoncallback=1&page=%d&photoset_id=%s&user_id=%s",
			apiURL, photosetAPIPath, page, photosetId, fakeUserId)
		response := fmt.Sprintf("%s%s", okHeader, fakePhotoset(photosetId, page, pages, photoIds[pageStart:pageEnd]))
		responses[photosURL] = httputil.StaticResponder(response)
		for k, _ := range photoIds {
			if k < pageEnd {
				continue
			}
			page++
			pageStart = k
			pageEnd = k + perPage
			if page > pages {
				albumNum++
				page = 1
				pages = albumNum / perPage
				if albumNum%perPage != 0 {
					pages++
				}
				albumEnd = pageStart + albumNum
				photosetId = blob.RefFromString(fmt.Sprintf("Photoset %d", albumNum)).DigestPrefix(10)
			}
			if pageEnd > albumEnd {
				pageEnd = albumEnd
			}
			photosURL := fmt.Sprintf("%s?extras=original_format&format=json&method=%s&nojsoncallback=1&page=%d&photoset_id=%s&user_id=%s",
				apiURL, photosetAPIPath, page, photosetId, fakeUserId)
			response := fmt.Sprintf("%s%s", okHeader, fakePhotoset(photosetId, page, pages, photoIds[pageStart:pageEnd]))
			responses[photosURL] = httputil.StaticResponder(response)
		}
	}

	// All the photo page calls (to get the photos info).
	// Each page has perPage photos, until end of photos.
	for i := 1; i <= nPhotosPages; i++ {
		photosURL := fmt.Sprintf("%s?extras=", apiURL) +
			url.QueryEscape("description,date_upload,date_taken,original_format,last_update,geo,tags,machine_tags,views,media,url_o") +
			fmt.Sprintf("&format=json&method=%s&nojsoncallback=1&page=%d&user_id=%s", photosAPIPath, i, fakeUserId)
		response := fmt.Sprintf("%s%s", okHeader, fakePhotosPage(i, nPhotosPages, perPage, photoIds))
		responses[photosURL] = httputil.StaticResponder(response)
	}

	// Actual photo(s) URL.
	pudgyPic := fakePicture()
	for _, v := range photoIds {
		photoURL := fmt.Sprintf("https://farm3.staticflickr.com/2897/14198397111_%s_o.jpg?user_id=%s", v, fakeUserId)
		responses[photoURL] = httputil.FileResponder(pudgyPic)
	}

	return httputil.NewFakeTransport(responses)
}
Exemplo n.º 10
0
// Verify that a batch import of 3 posts works
func TestIntegrationRun(t *testing.T) {
	const authToken = "gina:foo"
	const attrKey = "key"
	const attrValue = "value"

	responder := httputil.FileResponder("testdata/batchresponse.json")
	transport, err := httputil.NewRegexpFakeTransport([]*httputil.Matcher{
		&httputil.Matcher{`^https\://api\.pinboard\.in/v1/posts/all\?auth_token=gina:foo&format=json&results=10000&todt=\d\d\d\d.*`, responder},
	})
	if err != nil {
		t.Fatal(err)
	}

	imptest.ImporterTest(t, "pinboard", transport, func(rc *importer.RunContext) {
		err = rc.AccountNode().SetAttrs(attrAuthToken, authToken)
		if err != nil {
			t.Fatal(err)
		}

		testee := imp{}
		if err := testee.Run(rc); err != nil {
			t.Fatal(err)
		}

		postsNode, err := imptest.GetRequiredChildPathObj(rc.RootNode(), "posts")
		if err != nil {
			t.Fatal(err)
		}

		childRefs, err := imptest.FindChildRefs(postsNode)
		if err != nil {
			t.Fatal(err)
		}

		expectedPosts := map[string]string{
			`https://wiki.archlinux.org/index.php/xorg#Display_size_and_DPI`:                   "Xorg - ArchWiki",
			`http://www.harihareswara.net/sumana/2014/08/17/0`:                                 "One Way Confidence Will Look",
			`http://www.wikiart.org/en/marcus-larson/fishing-near-the-fjord-by-moonlight-1862`: "Fishing Near The Fjord By Moonlight - Marcus Larson - WikiArt.org",
		}

		if len(childRefs) != len(expectedPosts) {
			t.Fatalf("After import, found %d child refs, want %d: %v", len(childRefs), len(expectedPosts), childRefs)
		}

		for _, ref := range childRefs {
			childNode, err := rc.Host.ObjectFromRef(ref)
			if err != nil {
				t.Fatal(err)
			}
			foundURL := childNode.Attr(nodeattr.URL)
			expectedTitle, ok := expectedPosts[foundURL]
			if !ok {
				t.Fatalf("Found unexpected child node %v with url %q", childNode, foundURL)
			}
			foundTitle := childNode.Attr(nodeattr.Title)
			if foundTitle != expectedTitle {
				t.Fatalf("Found unexpected child node %v with title %q when we want %q", childNode, foundTitle, expectedTitle)
			}
			delete(expectedPosts, foundURL)
		}
		if len(expectedPosts) != 0 {
			t.Fatalf("The following entries were expected but not found: %#v", expectedPosts)
		}
	})
}
Exemplo n.º 11
0
// TestIntegrationRun tests both the twitter API and zip file import paths.
func TestIntegrationRun(t *testing.T) {
	const accessToken = "foo"
	const accessSecret = "bar"
	const userID = "camlistore_test"
	const attrKey = "key"
	const attrValue = "value"

	responder := httputil.FileResponder("testdata/user_timeline.json")
	transport, err := httputil.NewRegexpFakeTransport([]*httputil.Matcher{
		&httputil.Matcher{`^https\://api\.twitter\.com/1.1/statuses/user_timeline.json\?`, responder},
	})
	if err != nil {
		t.Fatal(err)
	}

	imptest.ImporterTest(t, "twitter", transport, func(rc *importer.RunContext) {

		err = rc.AccountNode().SetAttrs(importer.AcctAttrAccessToken, accessToken, importer.AcctAttrAccessTokenSecret, accessSecret, importer.AcctAttrUserID, userID)
		if err != nil {
			t.Fatal(err)
		}

		// First, run without the zip.
		testee := imp{}
		if err := testee.Run(rc); err != nil {
			t.Fatal(err)
		}

		// Tests that special characters are decoded properly, #476.
		jsonTweets := map[string]string{
			"727366997390946304": "I am a test account. Boop beep.",
			"727613700438265858": "foo and bar",
			"727613616149565440": `More beeping and booping & <> . $ % ^ * && /\/\()!`,
		}

		checkTweets(t, rc, jsonTweets)

		zipFile, err := os.Open("testdata/camlistore_test.zip")
		if err != nil {
			t.Fatal(err)
		}
		defer zipFile.Close()

		zipRef, err := schema.WriteFileFromReader(rc.Host.Target(), "camlistore_test.zip", zipFile)
		if err != nil {
			t.Fatal(err)
		}
		err = rc.AccountNode().SetAttrs(acctAttrTweetZip, zipRef.String())
		if err != nil {
			t.Fatal(err)
		}

		// Now run with the zip.
		if err := testee.Run(rc); err != nil {
			t.Fatal(err)
		}

		zipTweets := map[string]string{
			// Different text from JSON version for this item. Tests that importer prefers JSON.
			// Included here just to explain the test.
			"727366997390946304": "I am a test account. Beep boop.",
			"727367542772133888": `& <> . $ % ^ * && /\/\()! @Camlistore camlistore. https://t.co/Ld5gT3wjyq`,
		}
		checkTweets(t, rc, zipTweets, jsonTweets)
	})
}