func TestGetUserId(t *testing.T) {
im := &imp{}
ctx, cancel := context.WithCancel(context.WithValue(context.TODO(), ctxutil.HTTPClient, &http.Client{
Transport: httputil.NewFakeTransport(map[string]func() *http.Response{
"https://api.foursquare.com/v2/users/self?oauth_token=footoken&v=20140225": httputil.FileResponder("testdata/users-me-res.json"),
}),
}))
defer cancel()
inf, err := im.getUserInfo(ctx, "footoken")
if err != nil {
t.Fatal(err)
}
want := user{
Id: "13674",
FirstName: "Brad",
LastName: "Fitzpatrick",
Photo: photoItem{
Prefix: "https://irs0.4sqi.net/img/user/",
Suffix: "/CKG5FOF2WMCMPD3E.jpg",
},
}
if inf != want {
t.Errorf("user info = %+v; want %+v", inf, want)
}
}
func (im *imp) MakeTestData() http.RoundTripper {
const (
apiURL = "https://picasaweb.google.com/data/feed/api"
nAlbums = 10 // Arbitrary number of albums generated.
nEntries = 3 // number of albums or photos returned in the feed at each call.
defaultUserId = "default"
)
albumsListCached := make(map[int]string)
okHeader := `HTTP/1.1 200 OK
Content-Type: application/json; charset=UTF-8
`
responses := make(map[string]func() *http.Response)
// register the get albums list calls
for i := 1; i < nAlbums+1; i += nEntries {
url := fmt.Sprintf("%s/user/%s?start-index=%d", apiURL, defaultUserId, i)
response := okHeader + fakeAlbumsList(i, nAlbums, nEntries, albumsListCached)
responses[url] = httputil.StaticResponder(response)
}
// register the get album calls
for i := 1; i < nAlbums+1; i++ {
albumId := blob.RefFromString(fmt.Sprintf("Album %d", i)).DigestPrefix(10)
for j := 1; j < i+1; j += nEntries {
url := fmt.Sprintf("%s/user/%s/albumid/%s?imgmax=d&start-index=%d", apiURL, defaultUserId, albumId, j)
// Using i as nTotal argument means album N will have N photos in it.
response := okHeader + fakePhotosList(j, i, nEntries)
responses[url] = httputil.StaticResponder(response)
}
}
// register the photo download calls
pudgyPic := fakePhoto()
photoURL1 := "https://camlistore.org/pic/pudgy1.png"
photoURL2 := "https://camlistore.org/pic/pudgy2.png"
responses[photoURL1] = httputil.FileResponder(pudgyPic)
responses[photoURL2] = httputil.FileResponder(pudgyPic)
return httputil.NewFakeTransport(responses)
}
func (im *imp) MakeTestData() http.RoundTripper {
const (
fakeMaxId = int64(486450108201201664) // Most recent tweet.
nTweets = 300 // Arbitrary number of tweets generated.
)
fakeMinId := fakeMaxId - nTweets // Oldest tweet in our timeline.
timeLineURL := apiURL + userTimeLineAPIPath
timeLineCached := make(map[int64]string)
okHeader := `HTTP/1.1 200 OK
Content-Type: application/json; charset=UTF-8
`
timeLineResponse := okHeader + fakeTimeLine(fakeMaxId, fakeMinId, timeLineCached)
fakePic := fakePicture()
responses := map[string]func() *http.Response{
timeLineURL: httputil.StaticResponder(timeLineResponse),
fmt.Sprintf("%s?count=%d&user_id=fakeUserID", timeLineURL, tweetRequestLimit): httputil.StaticResponder(timeLineResponse),
"https://twitpic.com/show/large/bar": httputil.FileResponder(fakePic),
"https://i.imgur.com/foo.gif": httputil.FileResponder(fakePic),
}
// register all the user_timeline calls (max_id varies) that should occur,
responses[fmt.Sprintf("%s?count=%d&max_id=%d&user_id=fakeUserID", timeLineURL, tweetRequestLimit, fakeMaxId-nTweets+1)] = httputil.StaticResponder(okHeader + fakeTimeLine(fakeMaxId-nTweets+1, fakeMinId, timeLineCached))
if nTweets > tweetRequestLimit {
// that is, once every tweetRequestLimit-1, going down from fakeMaxId.
for i := fakeMaxId; i > fakeMinId; i -= tweetRequestLimit - 1 {
responses[fmt.Sprintf("%s?count=%d&max_id=%d&user_id=fakeUserID", timeLineURL, tweetRequestLimit, i)] = httputil.StaticResponder(okHeader + fakeTimeLine(i, fakeMinId, timeLineCached))
}
}
// register all the possible combinations of media twimg
for _, scheme := range []string{"http://", "https://"} {
for _, picsize := range []string{"thumb", "small", "medium", "large"} {
responses[fmt.Sprintf("%spbs.twimg.com/media/foo.jpg:%s", scheme, picsize)] = httputil.FileResponder(fakePic)
responses[fmt.Sprintf("%spbs.twimg.com/media/bar.png:%s", scheme, picsize)] = httputil.FileResponder(fakePic)
}
}
return httputil.NewFakeTransport(responses)
}
func TestGetUserID(t *testing.T) {
ctx := context.New(context.WithHTTPClient(&http.Client{
Transport: httputil.NewFakeTransport(map[string]func() *http.Response{
apiURL + userInfoAPIPath: httputil.FileResponder(filepath.FromSlash("testdata/verify_credentials-res.json")),
}),
}))
defer ctx.Cancel()
inf, err := getUserInfo(importer.OAuthContext{ctx, &oauth.Client{}, &oauth.Credentials{}})
if err != nil {
t.Fatal(err)
}
want := userInfo{
ID: "2325935334",
ScreenName: "lejatorn",
Name: "Mathieu Lonjaret",
}
if inf != want {
t.Errorf("user info = %+v; want %+v", inf, want)
}
}
func TestGetUserId(t *testing.T) {
im := &imp{}
ctx := context.New(context.WithHTTPClient(&http.Client{
Transport: httputil.NewFakeTransport(map[string]func() *http.Response{
"https://api.foursquare.com/v2/users/self?oauth_token=footoken&v=20140225": httputil.FileResponder("testdata/users-me-res.json"),
}),
}))
defer ctx.Cancel()
inf, err := im.getUserInfo(ctx, "footoken")
if err != nil {
t.Fatal(err)
}
want := user{
Id: "13674",
FirstName: "Brad",
LastName: "Fitzpatrick",
}
if inf != want {
t.Errorf("user info = %+v; want %+v", inf, want)
}
}
func TestGetUserId(t *testing.T) {
userID := "11047045264"
responder := httputil.FileResponder("testdata/users-me-res.xml")
cl := &http.Client{
Transport: httputil.NewFakeTransport(map[string]func() *http.Response{
"https://picasaweb.google.com/data/feed/api/user/default/contacts?kind=user": responder,
"https://picasaweb.google.com/data/feed/api/user/" + userID + "/contacts?kind=user": responder,
})}
inf, err := picago.GetUser(cl, "default")
if err != nil {
t.Fatal(err)
}
want := picago.User{
ID: userID,
URI: "https://picasaweb.google.com/" + userID,
Name: "Tamás Gulácsi",
Thumbnail: "https://lh4.googleusercontent.com/-qqove344/AAAAAAAAAAI/AAAAAAABcbg/TXl3f2K9dzI/s64-c/11047045264.jpg",
}
if inf != want {
t.Errorf("user info = %+v; want %+v", inf, want)
}
}
func (im *imp) MakeTestData() http.RoundTripper {
const nCheckins = 150 // Arbitrary number of checkins generated.
// if you add another venue, make sure the venueCounter reset
// in fakeCheckinsList allows for that case to happen.
// We could use global vars instead, but don't want to pollute the
// fousquare pkg namespace.
towns := map[int]*venueLocationItem{
0: {
Address: "Baker street",
City: "Dublin",
PostalCode: "0",
State: "none",
Country: "Ireland",
Lat: 53.4053427,
Lng: -8.3320801,
},
1: {
Address: "Fish&Ships street",
City: "London",
PostalCode: "1",
State: "none",
Country: "England",
Lat: 55.3617609,
Lng: -3.4433238,
},
2: {
Address: "Haggis street",
City: "Glasgow",
PostalCode: "2",
State: "none",
Country: "Scotland",
Lat: 57.7394571,
Lng: -4.686997,
},
3: {
Address: "rue du croissant",
City: "Grenoble",
PostalCode: "38000",
State: "none",
Country: "France",
Lat: 45.1841655,
Lng: 5.7155424,
},
4: {
Address: "burrito street",
City: "San Francisco",
PostalCode: "94114",
State: "CA",
Country: "US",
Lat: 37.7593625,
Lng: -122.4266995,
},
}
// We need to compute the venueIds in advance, because the venue id is used as a parameter
// in some of the requests we need to register.
var venueIds []string
for _, v := range towns {
venueIds = append(venueIds, blob.RefFromString(v.City).DigestPrefix(10))
}
checkinsURL := apiURL + checkinsAPIPath
checkinsListCached := make(map[int]string)
okHeader := `HTTP/1.1 200 OK
Content-Type: application/json; charset=UTF-8
`
responses := make(map[string]func() *http.Response)
// register all the checkins calls; offset varies.
for i := 0; i < nCheckins; i += checkinsRequestLimit {
url := fmt.Sprintf("%s?limit=%d&oauth_token=fakeAccessToken&offset=%d&v=%s",
checkinsURL, checkinsRequestLimit, i, apiVersion)
response := okHeader + fakeCheckinsList(i, nCheckins, towns, checkinsListCached)
responses[url] = httputil.StaticResponder(response)
}
// register all the venue photos calls (venueId varies)
photosURL := apiURL + "venues"
photosResponse := okHeader + fakePhotosList()
for _, id := range venueIds {
url := fmt.Sprintf("%s/%s/photos?limit=%d&oauth_token=fakeAccessToken&v=%s",
photosURL, id, photosRequestLimit, apiVersion)
responses[url] = httputil.StaticResponder(photosResponse)
}
// register the photoitem calls
pudgyPic := fakePhoto()
photoURL := "https://camlistore.org/pic/pudgy.png"
originalPhotoURL := "https://camlistore.org/original/pic/pudgy.png"
iconURL := "https://camlistore.org/bg_88/pic/pudgy.png"
responses[photoURL] = httputil.FileResponder(pudgyPic)
responses[originalPhotoURL] = httputil.FileResponder(pudgyPic)
responses[iconURL] = httputil.FileResponder(pudgyPic)
return httputil.NewFakeTransport(responses)
}
// Verify that a batch import of 3 posts works
func TestIntegrationRun(t *testing.T) {
const importerPrefix = "/importer/"
const authToken = "gina:foo"
const attrKey = "key"
const attrValue = "value"
w := test.GetWorld(t)
defer w.Stop()
baseURL := w.ServerBaseURL()
// TODO(mpl): add a utility in integration package to provide a client that
// just works with World.
cl, err := setupClient(w)
if err != nil {
t.Fatal(err)
}
signer, err := cl.Signer()
if err != nil {
t.Fatal(err)
}
clientId := map[string]string{
"pinboard": "fakeStaticClientId",
}
clientSecret := map[string]string{
"pinboard": "fakeStaticClientSecret",
}
responder := httputil.FileResponder("testdata/batchresponse.json")
transport, err := httputil.NewRegexpFakeTransport([]*httputil.Matcher{
&httputil.Matcher{`^https\://api\.pinboard\.in/v1/posts/all\?auth_token=gina:foo&format=json&results=10000&todt=\d\d\d\d.*`, responder},
})
if err != nil {
t.Fatal(err)
}
httpClient := &http.Client{
Transport: transport,
}
hc := importer.HostConfig{
BaseURL: baseURL,
Prefix: importerPrefix,
Target: cl,
BlobSource: cl,
Signer: signer,
Search: cl,
ClientId: clientId,
ClientSecret: clientSecret,
HTTPClient: httpClient,
}
host, err := importer.NewHost(hc)
if err != nil {
t.Fatal(err)
}
rc, err := importer.CreateAccount(host, "pinboard")
if err != nil {
t.Fatal(err)
}
err = rc.AccountNode().SetAttrs(attrAuthToken, authToken)
if err != nil {
t.Fatal(err)
}
testee := imp{}
if err := testee.Run(rc); err != nil {
t.Fatal(err)
}
postsNode, err := getRequiredChildPathObj(rc.RootNode(), "posts")
if err != nil {
t.Fatal(err)
}
childRefs, err := findChildRefs(postsNode)
if err != nil {
t.Fatal(err)
}
expectedPosts := map[string]string{
`https://wiki.archlinux.org/index.php/xorg#Display_size_and_DPI`: "Xorg - ArchWiki",
`http://www.harihareswara.net/sumana/2014/08/17/0`: "One Way Confidence Will Look",
`http://www.wikiart.org/en/marcus-larson/fishing-near-the-fjord-by-moonlight-1862`: "Fishing Near The Fjord By Moonlight - Marcus Larson - WikiArt.org",
}
if len(childRefs) != len(expectedPosts) {
t.Fatalf("After import, found %d child refs, want %d: %v", len(childRefs), len(expectedPosts), childRefs)
}
for _, ref := range childRefs {
childNode, err := host.ObjectFromRef(ref)
if err != nil {
t.Fatal(err)
}
foundURL := childNode.Attr(nodeattr.URL)
expectedTitle, ok := expectedPosts[foundURL]
if !ok {
t.Fatalf("Found unexpected child node %v with url %q", childNode, foundURL)
}
foundTitle := childNode.Attr(nodeattr.Title)
if foundTitle != expectedTitle {
t.Fatalf("Found unexpected child node %v with title %q when we want %q", childNode, foundTitle, expectedTitle)
}
delete(expectedPosts, foundURL)
}
if len(expectedPosts) != 0 {
t.Fatalf("The following entries were expected but not found: %#v", expectedPosts)
}
}
func (im imp) MakeTestData() http.RoundTripper {
const (
nPhotosets = 5 // Arbitrary number of sets.
perPage = 3 // number of photos per page (both when getting sets and when getting photos).
fakeUserId = "fakeUserId"
)
// Photoset N has N photos, so we've got 15 ( = 5 + 4 + 3 + 2 + 1) photos in total.
var nPhotos int
for i := 1; i <= nPhotosets; i++ {
nPhotos += i
}
nPhotosPages := nPhotos / perPage
if nPhotos%perPage != 0 {
nPhotosPages++
}
okHeader := `HTTP/1.1 200 OK
Content-Type: application/json; charset=UTF-8
`
// TODO(mpl): this scheme does not take into account that we could have the same photo
// in different albums. These two photos will end up with a different photoId.
buildPhotoIds := func(nsets, perPage int) []string {
var ids []string
for i := 1; i <= nsets; i++ {
photosetId := blob.RefFromString(fmt.Sprintf("Photoset %d", i)).DigestPrefix(10)
page := 1
// Photoset N has N photos.
indexOnPage := 1
for j := 1; j <= i; j++ {
photoId := blob.RefFromString(fmt.Sprintf("Photo %d on page %d of photoset %s", indexOnPage, page, photosetId)).DigestPrefix(10)
ids = append(ids, photoId)
indexOnPage++
if indexOnPage > perPage {
page++
indexOnPage = 1
}
}
}
return ids
}
photoIds := buildPhotoIds(nPhotosets, perPage)
responses := make(map[string]func() *http.Response)
// Initial photo sets list
photosetsURL := fmt.Sprintf("%s?format=json&method=%s&nojsoncallback=1&user_id=%s", apiURL, photosetsAPIPath, fakeUserId)
response := fmt.Sprintf("%s%s", okHeader, fakePhotosetsList(nPhotosets))
responses[photosetsURL] = httputil.StaticResponder(response)
// All the photoset calls. One call for each page of each photoset.
// Each page as perPage photos, or maybe less if end of the photoset.
{
pageStart := 0
albumEnd, pageEnd, albumNum, pages, page := 1, 1, 1, 1, 1
photosetId := blob.RefFromString(fmt.Sprintf("Photoset %d", albumNum)).DigestPrefix(10)
photosURL := fmt.Sprintf("%s?extras=original_format&format=json&method=%s&nojsoncallback=1&page=%d&photoset_id=%s&user_id=%s",
apiURL, photosetAPIPath, page, photosetId, fakeUserId)
response := fmt.Sprintf("%s%s", okHeader, fakePhotoset(photosetId, page, pages, photoIds[pageStart:pageEnd]))
responses[photosURL] = httputil.StaticResponder(response)
for k, _ := range photoIds {
if k < pageEnd {
continue
}
page++
pageStart = k
pageEnd = k + perPage
if page > pages {
albumNum++
page = 1
pages = albumNum / perPage
if albumNum%perPage != 0 {
pages++
}
albumEnd = pageStart + albumNum
photosetId = blob.RefFromString(fmt.Sprintf("Photoset %d", albumNum)).DigestPrefix(10)
}
if pageEnd > albumEnd {
pageEnd = albumEnd
}
photosURL := fmt.Sprintf("%s?extras=original_format&format=json&method=%s&nojsoncallback=1&page=%d&photoset_id=%s&user_id=%s",
apiURL, photosetAPIPath, page, photosetId, fakeUserId)
response := fmt.Sprintf("%s%s", okHeader, fakePhotoset(photosetId, page, pages, photoIds[pageStart:pageEnd]))
responses[photosURL] = httputil.StaticResponder(response)
}
}
// All the photo page calls (to get the photos info).
// Each page has perPage photos, until end of photos.
for i := 1; i <= nPhotosPages; i++ {
photosURL := fmt.Sprintf("%s?extras=", apiURL) +
url.QueryEscape("description,date_upload,date_taken,original_format,last_update,geo,tags,machine_tags,views,media,url_o") +
fmt.Sprintf("&format=json&method=%s&nojsoncallback=1&page=%d&user_id=%s", photosAPIPath, i, fakeUserId)
response := fmt.Sprintf("%s%s", okHeader, fakePhotosPage(i, nPhotosPages, perPage, photoIds))
responses[photosURL] = httputil.StaticResponder(response)
}
// Actual photo(s) URL.
pudgyPic := fakePicture()
for _, v := range photoIds {
photoURL := fmt.Sprintf("https://farm3.staticflickr.com/2897/14198397111_%s_o.jpg?user_id=%s", v, fakeUserId)
responses[photoURL] = httputil.FileResponder(pudgyPic)
}
return httputil.NewFakeTransport(responses)
}
// Verify that a batch import of 3 posts works
func TestIntegrationRun(t *testing.T) {
const authToken = "gina:foo"
const attrKey = "key"
const attrValue = "value"
responder := httputil.FileResponder("testdata/batchresponse.json")
transport, err := httputil.NewRegexpFakeTransport([]*httputil.Matcher{
&httputil.Matcher{`^https\://api\.pinboard\.in/v1/posts/all\?auth_token=gina:foo&format=json&results=10000&todt=\d\d\d\d.*`, responder},
})
if err != nil {
t.Fatal(err)
}
imptest.ImporterTest(t, "pinboard", transport, func(rc *importer.RunContext) {
err = rc.AccountNode().SetAttrs(attrAuthToken, authToken)
if err != nil {
t.Fatal(err)
}
testee := imp{}
if err := testee.Run(rc); err != nil {
t.Fatal(err)
}
postsNode, err := imptest.GetRequiredChildPathObj(rc.RootNode(), "posts")
if err != nil {
t.Fatal(err)
}
childRefs, err := imptest.FindChildRefs(postsNode)
if err != nil {
t.Fatal(err)
}
expectedPosts := map[string]string{
`https://wiki.archlinux.org/index.php/xorg#Display_size_and_DPI`: "Xorg - ArchWiki",
`http://www.harihareswara.net/sumana/2014/08/17/0`: "One Way Confidence Will Look",
`http://www.wikiart.org/en/marcus-larson/fishing-near-the-fjord-by-moonlight-1862`: "Fishing Near The Fjord By Moonlight - Marcus Larson - WikiArt.org",
}
if len(childRefs) != len(expectedPosts) {
t.Fatalf("After import, found %d child refs, want %d: %v", len(childRefs), len(expectedPosts), childRefs)
}
for _, ref := range childRefs {
childNode, err := rc.Host.ObjectFromRef(ref)
if err != nil {
t.Fatal(err)
}
foundURL := childNode.Attr(nodeattr.URL)
expectedTitle, ok := expectedPosts[foundURL]
if !ok {
t.Fatalf("Found unexpected child node %v with url %q", childNode, foundURL)
}
foundTitle := childNode.Attr(nodeattr.Title)
if foundTitle != expectedTitle {
t.Fatalf("Found unexpected child node %v with title %q when we want %q", childNode, foundTitle, expectedTitle)
}
delete(expectedPosts, foundURL)
}
if len(expectedPosts) != 0 {
t.Fatalf("The following entries were expected but not found: %#v", expectedPosts)
}
})
}
// TestIntegrationRun tests both the twitter API and zip file import paths.
func TestIntegrationRun(t *testing.T) {
const accessToken = "foo"
const accessSecret = "bar"
const userID = "camlistore_test"
const attrKey = "key"
const attrValue = "value"
responder := httputil.FileResponder("testdata/user_timeline.json")
transport, err := httputil.NewRegexpFakeTransport([]*httputil.Matcher{
&httputil.Matcher{`^https\://api\.twitter\.com/1.1/statuses/user_timeline.json\?`, responder},
})
if err != nil {
t.Fatal(err)
}
imptest.ImporterTest(t, "twitter", transport, func(rc *importer.RunContext) {
err = rc.AccountNode().SetAttrs(importer.AcctAttrAccessToken, accessToken, importer.AcctAttrAccessTokenSecret, accessSecret, importer.AcctAttrUserID, userID)
if err != nil {
t.Fatal(err)
}
// First, run without the zip.
testee := imp{}
if err := testee.Run(rc); err != nil {
t.Fatal(err)
}
// Tests that special characters are decoded properly, #476.
jsonTweets := map[string]string{
"727366997390946304": "I am a test account. Boop beep.",
"727613700438265858": "foo and bar",
"727613616149565440": `More beeping and booping & <> . $ % ^ * && /\/\()!`,
}
checkTweets(t, rc, jsonTweets)
zipFile, err := os.Open("testdata/camlistore_test.zip")
if err != nil {
t.Fatal(err)
}
defer zipFile.Close()
zipRef, err := schema.WriteFileFromReader(rc.Host.Target(), "camlistore_test.zip", zipFile)
if err != nil {
t.Fatal(err)
}
err = rc.AccountNode().SetAttrs(acctAttrTweetZip, zipRef.String())
if err != nil {
t.Fatal(err)
}
// Now run with the zip.
if err := testee.Run(rc); err != nil {
t.Fatal(err)
}
zipTweets := map[string]string{
// Different text from JSON version for this item. Tests that importer prefers JSON.
// Included here just to explain the test.
"727366997390946304": "I am a test account. Beep boop.",
"727367542772133888": `& <> . $ % ^ * && /\/\()! @Camlistore camlistore. https://t.co/Ld5gT3wjyq`,
}
checkTweets(t, rc, zipTweets, jsonTweets)
})
}