// ImporterTest sets up the environment for an importer integration test.
func ImporterTest(t *testing.T, importerName string, transport http.RoundTripper, fn func(*importer.RunContext)) {
const importerPrefix = "/importer/"
w := test.GetWorld(t)
defer w.Stop()
baseURL := w.ServerBaseURL()
// TODO(mpl): add a utility in integration package to provide a client that
// just works with World.
cl, err := setupClient(w)
if err != nil {
t.Fatal(err)
}
signer, err := cl.Signer()
if err != nil {
t.Fatal(err)
}
clientId := map[string]string{
importerName: "fakeStaticClientId",
}
clientSecret := map[string]string{
importerName: "fakeStaticClientSecret",
}
httpClient := &http.Client{
Transport: transport,
}
hc := importer.HostConfig{
BaseURL: baseURL,
Prefix: importerPrefix,
Target: cl,
BlobSource: cl,
Signer: signer,
Search: cl,
ClientId: clientId,
ClientSecret: clientSecret,
HTTPClient: httpClient,
}
host, err := importer.NewHost(hc)
if err != nil {
t.Fatal(err)
}
rc, err := importer.CreateAccount(host, importerName)
if err != nil {
t.Fatal(err)
}
fn(rc)
}
func (c *serverCmd) makeThings() error {
const importerPrefix = "/importer/"
// check that "/importer/" prefix is in config, just in case it ever changes.
configFile := filepath.Join(camliSrcRoot, "config", "dev-server-config.json")
config, err := ioutil.ReadFile(configFile)
if err != nil {
return fmt.Errorf("could not read config file %v: %v", configFile, err)
}
if !bytes.Contains(config, []byte(importerPrefix)) {
return fmt.Errorf("%s prefix not found in dev config. Did it change?", importerPrefix)
}
if err := netutil.AwaitReachable("localhost:"+c.port, time.Minute); err != nil {
return err
}
osutil.AddSecretRingFlag()
setCamdevVars()
baseURL := c.env.m["CAMLI_BASEURL"]
if baseURL == "" {
return errors.New("CAMLI_BASEURL is not set")
}
cl := client.New(baseURL)
signer, err := cl.Signer()
if err != nil {
return err
}
ClientId := make(map[string]string)
ClientSecret := make(map[string]string)
for name := range importer.All() {
ClientId[name] = "fakeStaticClientId"
ClientSecret[name] = "fakeStaticClientSecret"
}
hc := importer.HostConfig{
BaseURL: baseURL,
Prefix: importerPrefix,
Target: cl,
BlobSource: cl,
Signer: signer,
Search: cl,
ClientId: ClientId,
ClientSecret: ClientSecret,
}
for name, imp := range importer.All() {
mk, ok := imp.(importer.TestDataMaker)
if !ok {
continue
}
tr := mk.MakeTestData()
hc.HTTPClient = &http.Client{Transport: tr}
host, err := importer.NewHost(hc)
if err != nil {
return fmt.Errorf("could not obtain Host: %v", err)
}
rc, err := importer.CreateAccount(host, name)
if err != nil {
return err
}
if err := mk.SetTestAccount(rc.AccountNode()); err != nil {
return fmt.Errorf("could not set fake account node for importer %v: %v", name, err)
}
if err := imp.Run(rc); err != nil {
return err
}
}
return nil
}
// Verify that a batch import of 3 posts works
func TestIntegrationRun(t *testing.T) {
const importerPrefix = "/importer/"
const authToken = "gina:foo"
const attrKey = "key"
const attrValue = "value"
w := test.GetWorld(t)
defer w.Stop()
baseURL := w.ServerBaseURL()
// TODO(mpl): add a utility in integration package to provide a client that
// just works with World.
cl, err := setupClient(w)
if err != nil {
t.Fatal(err)
}
signer, err := cl.Signer()
if err != nil {
t.Fatal(err)
}
clientId := map[string]string{
"pinboard": "fakeStaticClientId",
}
clientSecret := map[string]string{
"pinboard": "fakeStaticClientSecret",
}
responder := httputil.FileResponder("testdata/batchresponse.json")
transport, err := httputil.NewRegexpFakeTransport([]*httputil.Matcher{
&httputil.Matcher{`^https\://api\.pinboard\.in/v1/posts/all\?auth_token=gina:foo&format=json&results=10000&todt=\d\d\d\d.*`, responder},
})
if err != nil {
t.Fatal(err)
}
httpClient := &http.Client{
Transport: transport,
}
hc := importer.HostConfig{
BaseURL: baseURL,
Prefix: importerPrefix,
Target: cl,
BlobSource: cl,
Signer: signer,
Search: cl,
ClientId: clientId,
ClientSecret: clientSecret,
HTTPClient: httpClient,
}
host, err := importer.NewHost(hc)
if err != nil {
t.Fatal(err)
}
rc, err := importer.CreateAccount(host, "pinboard")
if err != nil {
t.Fatal(err)
}
err = rc.AccountNode().SetAttrs(attrAuthToken, authToken)
if err != nil {
t.Fatal(err)
}
testee := imp{}
if err := testee.Run(rc); err != nil {
t.Fatal(err)
}
postsNode, err := getRequiredChildPathObj(rc.RootNode(), "posts")
if err != nil {
t.Fatal(err)
}
childRefs, err := findChildRefs(postsNode)
if err != nil {
t.Fatal(err)
}
expectedPosts := map[string]string{
`https://wiki.archlinux.org/index.php/xorg#Display_size_and_DPI`: "Xorg - ArchWiki",
`http://www.harihareswara.net/sumana/2014/08/17/0`: "One Way Confidence Will Look",
`http://www.wikiart.org/en/marcus-larson/fishing-near-the-fjord-by-moonlight-1862`: "Fishing Near The Fjord By Moonlight - Marcus Larson - WikiArt.org",
}
if len(childRefs) != len(expectedPosts) {
t.Fatalf("After import, found %d child refs, want %d: %v", len(childRefs), len(expectedPosts), childRefs)
}
for _, ref := range childRefs {
childNode, err := host.ObjectFromRef(ref)
if err != nil {
t.Fatal(err)
}
foundURL := childNode.Attr(nodeattr.URL)
expectedTitle, ok := expectedPosts[foundURL]
if !ok {
t.Fatalf("Found unexpected child node %v with url %q", childNode, foundURL)
}
foundTitle := childNode.Attr(nodeattr.Title)
if foundTitle != expectedTitle {
t.Fatalf("Found unexpected child node %v with title %q when we want %q", childNode, foundTitle, expectedTitle)
}
delete(expectedPosts, foundURL)
}
if len(expectedPosts) != 0 {
t.Fatalf("The following entries were expected but not found: %#v", expectedPosts)
}
}