func (im *imp) Run(ctx *importer.RunContext) error {
r := &run{
RunContext: ctx,
im: im,
}
userID := ctx.AccountNode().Attr(acctAttrUserID)
if userID == "" {
return errors.New("UserID hasn't been set by account setup.")
}
if err := r.importTweets(userID); err != nil {
return err
}
return nil
}
func checkTweets(t *testing.T, rc *importer.RunContext, expectedPostGroups ...map[string]string) {
postsNode, err := imptest.GetRequiredChildPathObj(rc.RootNode(), "tweets")
if err != nil {
t.Fatal(err)
}
childRefs, err := imptest.FindChildRefs(postsNode)
if err != nil {
t.Fatal(err)
}
// Merges groups, last wins
expectedPosts := map[string]string{}
for _, posts := range expectedPostGroups {
for k, v := range posts {
expectedPosts[k] = v
}
}
if len(childRefs) != len(expectedPosts) {
t.Fatalf("After import, found %d child refs, want %d: %v", len(childRefs), len(expectedPosts), childRefs)
}
for _, ref := range childRefs {
childNode, err := rc.Host.ObjectFromRef(ref)
if err != nil {
t.Fatal(err)
}
foundID := childNode.Attr("twitterId")
expectedContent, ok := expectedPosts[foundID]
if !ok {
t.Fatalf("Found unexpected child node %v with id %q", childNode, foundID)
}
foundContent := childNode.Attr(nodeattr.Content)
if foundContent != expectedContent {
t.Fatalf("Found unexpected child node %v with content %q when we want %q", childNode, foundContent, expectedContent)
}
delete(expectedPosts, foundID)
}
if len(expectedPosts) != 0 {
t.Fatalf("The following entries were expected but not found: %#v", expectedPosts)
}
}
func (imp) Run(ctx *importer.RunContext) error {
clientId, secret, err := ctx.Credentials()
if err != nil {
return err
}
acctNode := ctx.AccountNode()
ocfg := baseOAuthConfig
ocfg.ClientId, ocfg.ClientSecret = clientId, secret
token := decodeToken(acctNode.Attr(acctAttrOAuthToken))
transport := &oauth.Transport{
Config: &ocfg,
Token: &token,
Transport: notOAuthTransport(ctx.HTTPClient()),
}
ctx.Context = ctx.Context.New(context.WithHTTPClient(transport.Client()))
root := ctx.RootNode()
if root.Attr(nodeattr.Title) == "" {
if err := root.SetAttr(nodeattr.Title,
fmt.Sprintf("%s %s - Google/Picasa Photos",
acctNode.Attr(importer.AcctAttrGivenName),
acctNode.Attr(importer.AcctAttrFamilyName))); err != nil {
return err
}
}
r := &run{
RunContext: ctx,
incremental: !forceFullImport && acctNode.Attr(importer.AcctAttrCompletedVersion) == runCompleteVersion,
photoGate: syncutil.NewGate(3),
}
if err := r.importAlbums(); err != nil {
return err
}
r.mu.Lock()
anyErr := r.anyErr
r.mu.Unlock()
if !anyErr {
if err := acctNode.SetAttrs(importer.AcctAttrCompletedVersion, runCompleteVersion); err != nil {
return err
}
}
return nil
}
func (im *imp) Run(ctx *importer.RunContext) error {
clientId, secret, err := ctx.Credentials()
if err != nil {
return err
}
r := &run{
RunContext: ctx,
im: im,
oauthConfig: &oauth.Config{
ClientId: clientId,
ClientSecret: secret,
AuthURL: authURL,
TokenURL: tokenURL,
},
}
if err := r.importCheckins(); err != nil {
return err
}
return nil
}
func (im *imp) Run(ctx *importer.RunContext) error {
r := &run{
RunContext: ctx,
im: im,
incremental: ctx.AccountNode().Attr(importer.AcctAttrCompletedVersion) == runCompleteVersion,
}
if err := r.importCheckins(); err != nil {
return err
}
r.mu.Lock()
anyErr := r.anyErr
r.mu.Unlock()
if !anyErr {
if err := r.AccountNode().SetAttrs(importer.AcctAttrCompletedVersion, runCompleteVersion); err != nil {
return err
}
}
return nil
}
func (im *imp) LongPoll(rctx *importer.RunContext) error {
clientId, secret, err := rctx.Credentials()
if err != nil {
return err
}
acctNode := rctx.AccountNode()
accessToken := acctNode.Attr(importer.AcctAttrAccessToken)
accessSecret := acctNode.Attr(importer.AcctAttrAccessTokenSecret)
if accessToken == "" || accessSecret == "" {
return errors.New("access credentials not found")
}
oauthClient := &oauth.Client{
TemporaryCredentialRequestURI: temporaryCredentialRequestURL,
ResourceOwnerAuthorizationURI: resourceOwnerAuthorizationURL,
TokenRequestURI: tokenRequestURL,
Credentials: oauth.Credentials{
Token: clientId,
Secret: secret,
},
}
accessCreds := &oauth.Credentials{
Token: accessToken,
Secret: accessSecret,
}
form := url.Values{"with": {"user"}}
req, _ := http.NewRequest("GET", "https://userstream.twitter.com/1.1/user.json", nil)
req.Header.Set("Authorization", oauthClient.AuthorizationHeader(accessCreds, "GET", req.URL, form))
req.URL.RawQuery = form.Encode()
req.Cancel = rctx.Context().Done()
log.Printf("Beginning twitter long poll...")
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != 200 {
return errors.New(res.Status)
}
bs := bufio.NewScanner(res.Body)
for bs.Scan() {
line := strings.TrimSpace(bs.Text())
if line == "" || strings.HasPrefix(line, `{"friends`) {
continue
}
log.Printf("Twitter long poll saw a tweet: %s", line)
return nil
}
if err := bs.Err(); err != nil {
return err
}
return errors.New("got EOF without a tweet.")
}
func (imp) Run(rctx *importer.RunContext) error {
clientID, secret, err := rctx.Credentials()
if err != nil {
return err
}
acctNode := rctx.AccountNode()
ocfg := &oauth2.Config{
Endpoint: google.Endpoint,
ClientID: clientID,
ClientSecret: secret,
Scopes: []string{scopeURL},
}
token := decodeToken(acctNode.Attr(acctAttrOAuthToken))
baseCtx := rctx.Context()
ctx := context.WithValue(baseCtx, ctxutil.HTTPClient, ocfg.Client(baseCtx, token))
root := rctx.RootNode()
if root.Attr(nodeattr.Title) == "" {
if err := root.SetAttr(
nodeattr.Title,
fmt.Sprintf("%s - Google Photos", acctNode.Attr(importer.AcctAttrName)),
); err != nil {
return err
}
}
r := &run{
RunContext: rctx,
incremental: !forceFullImport && acctNode.Attr(importer.AcctAttrCompletedVersion) == runCompleteVersion,
photoGate: syncutil.NewGate(3),
}
if err := r.importAlbums(ctx); err != nil {
return err
}
if err := acctNode.SetAttrs(importer.AcctAttrCompletedVersion, runCompleteVersion); err != nil {
return err
}
return nil
}
func (im *imp) Run(ctx *importer.RunContext) (err error) {
log.Printf("Running dummy importer.")
defer func() {
log.Printf("Dummy importer returned: %v", err)
}()
root := ctx.RootNode()
fileRef, err := schema.WriteFileFromReader(ctx.Host.Target(), "foo.txt", strings.NewReader("Some file.\n"))
if err != nil {
return err
}
obj, err := root.ChildPathObject("foo.txt")
if err != nil {
return err
}
if err = obj.SetAttr("camliContent", fileRef.String()); err != nil {
return err
}
n, _ := strconv.Atoi(ctx.AccountNode().Attr(acctAttrRunNumber))
n++
ctx.AccountNode().SetAttr(acctAttrRunNumber, fmt.Sprint(n))
// Update the title each time, just to show it working. You
// wouldn't actually do this:
return root.SetAttr("title", fmt.Sprintf("dummy: %s import #%d", ctx.AccountNode().Attr(acctAttrUsername), n))
}
func (imp) Run(ctx *importer.RunContext) error {
clientID, secret, err := ctx.Credentials()
if err != nil {
return fmt.Errorf("no API credentials: %v", err)
}
accountNode := ctx.AccountNode()
accessToken := accountNode.Attr(importer.AcctAttrAccessToken)
accessSecret := accountNode.Attr(importer.AcctAttrAccessTokenSecret)
if accessToken == "" || accessSecret == "" {
return errors.New("access credentials not found")
}
userID := ctx.AccountNode().Attr(importer.AcctAttrUserID)
if userID == "" {
return errors.New("UserID hasn't been set by account setup.")
}
r := &run{
userID: userID,
RunContext: ctx,
oauthClient: &oauth.Client{
TemporaryCredentialRequestURI: temporaryCredentialRequestURL,
ResourceOwnerAuthorizationURI: resourceOwnerAuthorizationURL,
TokenRequestURI: tokenRequestURL,
Credentials: oauth.Credentials{
Token: clientID,
Secret: secret,
},
},
accessCreds: &oauth.Credentials{
Token: accessToken,
Secret: accessSecret,
},
primaryPhoto: make(map[string]string),
}
if err := r.importPhotosets(); err != nil {
return err
}
if err := r.importPhotos(); err != nil {
return err
}
return nil
}
func (im *imp) Run(ctx *importer.RunContext) error {
clientId, secret, err := ctx.Credentials()
if err != nil {
return fmt.Errorf("no API credentials: %v", err)
}
acctNode := ctx.AccountNode()
accessToken := acctNode.Attr(importer.AcctAttrAccessToken)
accessSecret := acctNode.Attr(importer.AcctAttrAccessTokenSecret)
if accessToken == "" || accessSecret == "" {
return errors.New("access credentials not found")
}
r := &run{
RunContext: ctx,
im: im,
incremental: !forceFullImport && acctNode.Attr(importer.AcctAttrCompletedVersion) == runCompleteVersion,
oauthClient: &oauth.Client{
TemporaryCredentialRequestURI: temporaryCredentialRequestURL,
ResourceOwnerAuthorizationURI: resourceOwnerAuthorizationURL,
TokenRequestURI: tokenRequestURL,
Credentials: oauth.Credentials{
Token: clientId,
Secret: secret,
},
},
accessCreds: &oauth.Credentials{
Token: accessToken,
Secret: accessSecret,
},
}
userID := acctNode.Attr(importer.AcctAttrUserID)
if userID == "" {
return errors.New("UserID hasn't been set by account setup.")
}
skipAPITweets, _ := strconv.ParseBool(os.Getenv("CAMLI_TWITTER_SKIP_API_IMPORT"))
if !skipAPITweets {
if err := r.importTweets(userID); err != nil {
return err
}
}
zipRef := acctNode.Attr(acctAttrTweetZip)
zipDoneVal := zipRef + ":" + runCompleteVersion
if zipRef != "" && !(r.incremental && acctNode.Attr(acctAttrZipDoneVersion) == zipDoneVal) {
zipbr, ok := blob.Parse(zipRef)
if !ok {
return fmt.Errorf("invalid zip file blobref %q", zipRef)
}
fr, err := schema.NewFileReader(r.Host.BlobSource(), zipbr)
if err != nil {
return fmt.Errorf("error opening zip %v: %v", zipbr, err)
}
defer fr.Close()
zr, err := zip.NewReader(fr, fr.Size())
if err != nil {
return fmt.Errorf("Error opening twitter zip file %v: %v", zipRef, err)
}
if err := r.importTweetsFromZip(userID, zr); err != nil {
return err
}
if err := acctNode.SetAttrs(acctAttrZipDoneVersion, zipDoneVal); err != nil {
return err
}
}
r.mu.Lock()
anyErr := r.anyErr
r.mu.Unlock()
if !anyErr {
if err := acctNode.SetAttrs(importer.AcctAttrCompletedVersion, runCompleteVersion); err != nil {
return err
}
}
return nil
}