// Verify that a batch import of 3 posts works
func TestIntegrationRun(t *testing.T) {
const authToken = "gina:foo"
const attrKey = "key"
const attrValue = "value"
responder := httputil.FileResponder("testdata/batchresponse.json")
transport, err := httputil.NewRegexpFakeTransport([]*httputil.Matcher{
&httputil.Matcher{`^https\://api\.pinboard\.in/v1/posts/all\?auth_token=gina:foo&format=json&results=10000&todt=\d\d\d\d.*`, responder},
})
if err != nil {
t.Fatal(err)
}
imptest.ImporterTest(t, "pinboard", transport, func(rc *importer.RunContext) {
err = rc.AccountNode().SetAttrs(attrAuthToken, authToken)
if err != nil {
t.Fatal(err)
}
testee := imp{}
if err := testee.Run(rc); err != nil {
t.Fatal(err)
}
postsNode, err := imptest.GetRequiredChildPathObj(rc.RootNode(), "posts")
if err != nil {
t.Fatal(err)
}
childRefs, err := imptest.FindChildRefs(postsNode)
if err != nil {
t.Fatal(err)
}
expectedPosts := map[string]string{
`https://wiki.archlinux.org/index.php/xorg#Display_size_and_DPI`: "Xorg - ArchWiki",
`http://www.harihareswara.net/sumana/2014/08/17/0`: "One Way Confidence Will Look",
`http://www.wikiart.org/en/marcus-larson/fishing-near-the-fjord-by-moonlight-1862`: "Fishing Near The Fjord By Moonlight - Marcus Larson - WikiArt.org",
}
if len(childRefs) != len(expectedPosts) {
t.Fatalf("After import, found %d child refs, want %d: %v", len(childRefs), len(expectedPosts), childRefs)
}
for _, ref := range childRefs {
childNode, err := rc.Host.ObjectFromRef(ref)
if err != nil {
t.Fatal(err)
}
foundURL := childNode.Attr(nodeattr.URL)
expectedTitle, ok := expectedPosts[foundURL]
if !ok {
t.Fatalf("Found unexpected child node %v with url %q", childNode, foundURL)
}
foundTitle := childNode.Attr(nodeattr.Title)
if foundTitle != expectedTitle {
t.Fatalf("Found unexpected child node %v with title %q when we want %q", childNode, foundTitle, expectedTitle)
}
delete(expectedPosts, foundURL)
}
if len(expectedPosts) != 0 {
t.Fatalf("The following entries were expected but not found: %#v", expectedPosts)
}
})
}
// Verify that a batch import of 3 posts works
func TestIntegrationRun(t *testing.T) {
const importerPrefix = "/importer/"
const authToken = "gina:foo"
const attrKey = "key"
const attrValue = "value"
w := test.GetWorld(t)
defer w.Stop()
baseURL := w.ServerBaseURL()
// TODO(mpl): add a utility in integration package to provide a client that
// just works with World.
cl, err := setupClient(w)
if err != nil {
t.Fatal(err)
}
signer, err := cl.Signer()
if err != nil {
t.Fatal(err)
}
clientId := map[string]string{
"pinboard": "fakeStaticClientId",
}
clientSecret := map[string]string{
"pinboard": "fakeStaticClientSecret",
}
responder := httputil.FileResponder("testdata/batchresponse.json")
transport, err := httputil.NewRegexpFakeTransport([]*httputil.Matcher{
&httputil.Matcher{`^https\://api\.pinboard\.in/v1/posts/all\?auth_token=gina:foo&format=json&results=10000&todt=\d\d\d\d.*`, responder},
})
if err != nil {
t.Fatal(err)
}
httpClient := &http.Client{
Transport: transport,
}
hc := importer.HostConfig{
BaseURL: baseURL,
Prefix: importerPrefix,
Target: cl,
BlobSource: cl,
Signer: signer,
Search: cl,
ClientId: clientId,
ClientSecret: clientSecret,
HTTPClient: httpClient,
}
host, err := importer.NewHost(hc)
if err != nil {
t.Fatal(err)
}
rc, err := importer.CreateAccount(host, "pinboard")
if err != nil {
t.Fatal(err)
}
err = rc.AccountNode().SetAttrs(attrAuthToken, authToken)
if err != nil {
t.Fatal(err)
}
testee := imp{}
if err := testee.Run(rc); err != nil {
t.Fatal(err)
}
postsNode, err := getRequiredChildPathObj(rc.RootNode(), "posts")
if err != nil {
t.Fatal(err)
}
childRefs, err := findChildRefs(postsNode)
if err != nil {
t.Fatal(err)
}
expectedPosts := map[string]string{
`https://wiki.archlinux.org/index.php/xorg#Display_size_and_DPI`: "Xorg - ArchWiki",
`http://www.harihareswara.net/sumana/2014/08/17/0`: "One Way Confidence Will Look",
`http://www.wikiart.org/en/marcus-larson/fishing-near-the-fjord-by-moonlight-1862`: "Fishing Near The Fjord By Moonlight - Marcus Larson - WikiArt.org",
}
if len(childRefs) != len(expectedPosts) {
t.Fatalf("After import, found %d child refs, want %d: %v", len(childRefs), len(expectedPosts), childRefs)
}
for _, ref := range childRefs {
childNode, err := host.ObjectFromRef(ref)
if err != nil {
t.Fatal(err)
}
foundURL := childNode.Attr(nodeattr.URL)
expectedTitle, ok := expectedPosts[foundURL]
if !ok {
t.Fatalf("Found unexpected child node %v with url %q", childNode, foundURL)
}
foundTitle := childNode.Attr(nodeattr.Title)
if foundTitle != expectedTitle {
t.Fatalf("Found unexpected child node %v with title %q when we want %q", childNode, foundTitle, expectedTitle)
}
delete(expectedPosts, foundURL)
}
if len(expectedPosts) != 0 {
t.Fatalf("The following entries were expected but not found: %#v", expectedPosts)
}
}
// TestIntegrationRun tests both the twitter API and zip file import paths.
func TestIntegrationRun(t *testing.T) {
const accessToken = "foo"
const accessSecret = "bar"
const userID = "camlistore_test"
const attrKey = "key"
const attrValue = "value"
responder := httputil.FileResponder("testdata/user_timeline.json")
transport, err := httputil.NewRegexpFakeTransport([]*httputil.Matcher{
&httputil.Matcher{`^https\://api\.twitter\.com/1.1/statuses/user_timeline.json\?`, responder},
})
if err != nil {
t.Fatal(err)
}
imptest.ImporterTest(t, "twitter", transport, func(rc *importer.RunContext) {
err = rc.AccountNode().SetAttrs(importer.AcctAttrAccessToken, accessToken, importer.AcctAttrAccessTokenSecret, accessSecret, importer.AcctAttrUserID, userID)
if err != nil {
t.Fatal(err)
}
// First, run without the zip.
testee := imp{}
if err := testee.Run(rc); err != nil {
t.Fatal(err)
}
// Tests that special characters are decoded properly, #476.
jsonTweets := map[string]string{
"727366997390946304": "I am a test account. Boop beep.",
"727613700438265858": "foo and bar",
"727613616149565440": `More beeping and booping & <> . $ % ^ * && /\/\()!`,
}
checkTweets(t, rc, jsonTweets)
zipFile, err := os.Open("testdata/camlistore_test.zip")
if err != nil {
t.Fatal(err)
}
defer zipFile.Close()
zipRef, err := schema.WriteFileFromReader(rc.Host.Target(), "camlistore_test.zip", zipFile)
if err != nil {
t.Fatal(err)
}
err = rc.AccountNode().SetAttrs(acctAttrTweetZip, zipRef.String())
if err != nil {
t.Fatal(err)
}
// Now run with the zip.
if err := testee.Run(rc); err != nil {
t.Fatal(err)
}
zipTweets := map[string]string{
// Different text from JSON version for this item. Tests that importer prefers JSON.
// Included here just to explain the test.
"727366997390946304": "I am a test account. Beep boop.",
"727367542772133888": `& <> . $ % ^ * && /\/\()! @Camlistore camlistore. https://t.co/Ld5gT3wjyq`,
}
checkTweets(t, rc, zipTweets, jsonTweets)
})
}
