Example #1
0
// Verify that a batch import of 3 posts works
func TestIntegrationRun(t *testing.T) {
	const authToken = "gina:foo"
	const attrKey = "key"
	const attrValue = "value"

	responder := httputil.FileResponder("testdata/batchresponse.json")
	transport, err := httputil.NewRegexpFakeTransport([]*httputil.Matcher{
		&httputil.Matcher{`^https\://api\.pinboard\.in/v1/posts/all\?auth_token=gina:foo&format=json&results=10000&todt=\d\d\d\d.*`, responder},
	})
	if err != nil {
		t.Fatal(err)
	}

	imptest.ImporterTest(t, "pinboard", transport, func(rc *importer.RunContext) {
		err = rc.AccountNode().SetAttrs(attrAuthToken, authToken)
		if err != nil {
			t.Fatal(err)
		}

		testee := imp{}
		if err := testee.Run(rc); err != nil {
			t.Fatal(err)
		}

		postsNode, err := imptest.GetRequiredChildPathObj(rc.RootNode(), "posts")
		if err != nil {
			t.Fatal(err)
		}

		childRefs, err := imptest.FindChildRefs(postsNode)
		if err != nil {
			t.Fatal(err)
		}

		expectedPosts := map[string]string{
			`https://wiki.archlinux.org/index.php/xorg#Display_size_and_DPI`:                   "Xorg - ArchWiki",
			`http://www.harihareswara.net/sumana/2014/08/17/0`:                                 "One Way Confidence Will Look",
			`http://www.wikiart.org/en/marcus-larson/fishing-near-the-fjord-by-moonlight-1862`: "Fishing Near The Fjord By Moonlight - Marcus Larson - WikiArt.org",
		}

		if len(childRefs) != len(expectedPosts) {
			t.Fatalf("After import, found %d child refs, want %d: %v", len(childRefs), len(expectedPosts), childRefs)
		}

		for _, ref := range childRefs {
			childNode, err := rc.Host.ObjectFromRef(ref)
			if err != nil {
				t.Fatal(err)
			}
			foundURL := childNode.Attr(nodeattr.URL)
			expectedTitle, ok := expectedPosts[foundURL]
			if !ok {
				t.Fatalf("Found unexpected child node %v with url %q", childNode, foundURL)
			}
			foundTitle := childNode.Attr(nodeattr.Title)
			if foundTitle != expectedTitle {
				t.Fatalf("Found unexpected child node %v with title %q when we want %q", childNode, foundTitle, expectedTitle)
			}
			delete(expectedPosts, foundURL)
		}
		if len(expectedPosts) != 0 {
			t.Fatalf("The following entries were expected but not found: %#v", expectedPosts)
		}
	})
}
Example #2
0
// Verify that a batch import of 3 posts works
func TestIntegrationRun(t *testing.T) {
	const importerPrefix = "/importer/"
	const authToken = "gina:foo"
	const attrKey = "key"
	const attrValue = "value"

	w := test.GetWorld(t)
	defer w.Stop()
	baseURL := w.ServerBaseURL()

	// TODO(mpl): add a utility in integration package to provide a client that
	// just works with World.
	cl, err := setupClient(w)
	if err != nil {
		t.Fatal(err)
	}
	signer, err := cl.Signer()
	if err != nil {
		t.Fatal(err)
	}
	clientId := map[string]string{
		"pinboard": "fakeStaticClientId",
	}
	clientSecret := map[string]string{
		"pinboard": "fakeStaticClientSecret",
	}

	responder := httputil.FileResponder("testdata/batchresponse.json")
	transport, err := httputil.NewRegexpFakeTransport([]*httputil.Matcher{
		&httputil.Matcher{`^https\://api\.pinboard\.in/v1/posts/all\?auth_token=gina:foo&format=json&results=10000&todt=\d\d\d\d.*`, responder},
	})
	if err != nil {
		t.Fatal(err)
	}
	httpClient := &http.Client{
		Transport: transport,
	}

	hc := importer.HostConfig{
		BaseURL:      baseURL,
		Prefix:       importerPrefix,
		Target:       cl,
		BlobSource:   cl,
		Signer:       signer,
		Search:       cl,
		ClientId:     clientId,
		ClientSecret: clientSecret,
		HTTPClient:   httpClient,
	}

	host, err := importer.NewHost(hc)
	if err != nil {
		t.Fatal(err)
	}
	rc, err := importer.CreateAccount(host, "pinboard")
	if err != nil {
		t.Fatal(err)
	}
	err = rc.AccountNode().SetAttrs(attrAuthToken, authToken)
	if err != nil {
		t.Fatal(err)
	}

	testee := imp{}
	if err := testee.Run(rc); err != nil {
		t.Fatal(err)
	}

	postsNode, err := getRequiredChildPathObj(rc.RootNode(), "posts")
	if err != nil {
		t.Fatal(err)
	}

	childRefs, err := findChildRefs(postsNode)
	if err != nil {
		t.Fatal(err)
	}

	expectedPosts := map[string]string{
		`https://wiki.archlinux.org/index.php/xorg#Display_size_and_DPI`:                   "Xorg - ArchWiki",
		`http://www.harihareswara.net/sumana/2014/08/17/0`:                                 "One Way Confidence Will Look",
		`http://www.wikiart.org/en/marcus-larson/fishing-near-the-fjord-by-moonlight-1862`: "Fishing Near The Fjord By Moonlight - Marcus Larson - WikiArt.org",
	}

	if len(childRefs) != len(expectedPosts) {
		t.Fatalf("After import, found %d child refs, want %d: %v", len(childRefs), len(expectedPosts), childRefs)
	}

	for _, ref := range childRefs {
		childNode, err := host.ObjectFromRef(ref)
		if err != nil {
			t.Fatal(err)
		}
		foundURL := childNode.Attr(nodeattr.URL)
		expectedTitle, ok := expectedPosts[foundURL]
		if !ok {
			t.Fatalf("Found unexpected child node %v with url %q", childNode, foundURL)
		}
		foundTitle := childNode.Attr(nodeattr.Title)
		if foundTitle != expectedTitle {
			t.Fatalf("Found unexpected child node %v with title %q when we want %q", childNode, foundTitle, expectedTitle)
		}
		delete(expectedPosts, foundURL)
	}
	if len(expectedPosts) != 0 {
		t.Fatalf("The following entries were expected but not found: %#v", expectedPosts)
	}
}
Example #3
0
// TestIntegrationRun tests both the twitter API and zip file import paths.
func TestIntegrationRun(t *testing.T) {
	const accessToken = "foo"
	const accessSecret = "bar"
	const userID = "camlistore_test"
	const attrKey = "key"
	const attrValue = "value"

	responder := httputil.FileResponder("testdata/user_timeline.json")
	transport, err := httputil.NewRegexpFakeTransport([]*httputil.Matcher{
		&httputil.Matcher{`^https\://api\.twitter\.com/1.1/statuses/user_timeline.json\?`, responder},
	})
	if err != nil {
		t.Fatal(err)
	}

	imptest.ImporterTest(t, "twitter", transport, func(rc *importer.RunContext) {

		err = rc.AccountNode().SetAttrs(importer.AcctAttrAccessToken, accessToken, importer.AcctAttrAccessTokenSecret, accessSecret, importer.AcctAttrUserID, userID)
		if err != nil {
			t.Fatal(err)
		}

		// First, run without the zip.
		testee := imp{}
		if err := testee.Run(rc); err != nil {
			t.Fatal(err)
		}

		// Tests that special characters are decoded properly, #476.
		jsonTweets := map[string]string{
			"727366997390946304": "I am a test account. Boop beep.",
			"727613700438265858": "foo and bar",
			"727613616149565440": `More beeping and booping & <> . $ % ^ * && /\/\()!`,
		}

		checkTweets(t, rc, jsonTweets)

		zipFile, err := os.Open("testdata/camlistore_test.zip")
		if err != nil {
			t.Fatal(err)
		}
		defer zipFile.Close()

		zipRef, err := schema.WriteFileFromReader(rc.Host.Target(), "camlistore_test.zip", zipFile)
		if err != nil {
			t.Fatal(err)
		}
		err = rc.AccountNode().SetAttrs(acctAttrTweetZip, zipRef.String())
		if err != nil {
			t.Fatal(err)
		}

		// Now run with the zip.
		if err := testee.Run(rc); err != nil {
			t.Fatal(err)
		}

		zipTweets := map[string]string{
			// Different text from JSON version for this item. Tests that importer prefers JSON.
			// Included here just to explain the test.
			"727366997390946304": "I am a test account. Beep boop.",
			"727367542772133888": `& <> . $ % ^ * && /\/\()! @Camlistore camlistore. https://t.co/Ld5gT3wjyq`,
		}
		checkTweets(t, rc, zipTweets, jsonTweets)
	})
}