func (im *imp) ServeCallback(w http.ResponseWriter, r *http.Request, ctx *importer.SetupContext) {
tempToken := ctx.AccountNode.Attr(importer.AcctAttrTempToken)
tempSecret := ctx.AccountNode.Attr(importer.AcctAttrTempSecret)
if tempToken == "" || tempSecret == "" {
log.Printf("twitter: no temp creds in callback")
httputil.BadRequestError(w, "no temp creds in callback")
return
}
if tempToken != r.FormValue("oauth_token") {
log.Printf("unexpected oauth_token: got %v, want %v", r.FormValue("oauth_token"), tempToken)
httputil.BadRequestError(w, "unexpected oauth_token")
return
}
oauthClient, err := ctx.NewOAuthClient(oAuthURIs)
if err != nil {
err = fmt.Errorf("error getting OAuth client: %v", err)
httputil.ServeError(w, r, err)
return
}
tokenCred, vals, err := oauthClient.RequestToken(
ctxutil.Client(ctx),
&oauth.Credentials{
Token: tempToken,
Secret: tempSecret,
},
r.FormValue("oauth_verifier"),
)
if err != nil {
httputil.ServeError(w, r, fmt.Errorf("Error getting request token: %v ", err))
return
}
userid := vals.Get("user_id")
if userid == "" {
httputil.ServeError(w, r, fmt.Errorf("Couldn't get user id: %v", err))
return
}
if err := ctx.AccountNode.SetAttrs(
importer.AcctAttrAccessToken, tokenCred.Token,
importer.AcctAttrAccessTokenSecret, tokenCred.Secret,
); err != nil {
httputil.ServeError(w, r, fmt.Errorf("Error setting token attributes: %v", err))
return
}
u, err := getUserInfo(importer.OAuthContext{ctx.Context, oauthClient, tokenCred})
if err != nil {
httputil.ServeError(w, r, fmt.Errorf("Couldn't get user info: %v", err))
return
}
if err := ctx.AccountNode.SetAttrs(
importer.AcctAttrUserID, u.ID,
importer.AcctAttrName, u.Name,
importer.AcctAttrUserName, u.ScreenName,
nodeattr.Title, fmt.Sprintf("%s's Twitter Account", u.ScreenName),
); err != nil {
httputil.ServeError(w, r, fmt.Errorf("Error setting attribute: %v", err))
return
}
http.Redirect(w, r, ctx.AccountURL(), http.StatusFound)
}
// Lookup returns rectangles for the given address. Currently the only
// implementation is the Google geocoding service.
func Lookup(ctx context.Context, address string) ([]Rect, error) {
mu.RLock()
rects, ok := cache[address]
mu.RUnlock()
if ok {
return rects, nil
}
rectsi, err := sf.Do(address, func() (interface{}, error) {
// TODO: static data files from OpenStreetMap, Wikipedia, etc?
urlStr := "https://maps.googleapis.com/maps/api/geocode/json?address=" + url.QueryEscape(address) + "&sensor=false"
res, err := ctxhttp.Get(ctx, ctxutil.Client(ctx), urlStr)
if err != nil {
return nil, err
}
defer res.Body.Close()
rects, err := decodeGoogleResponse(res.Body)
log.Printf("Google geocode lookup (%q) = %#v, %v", address, rects, err)
if err == nil {
mu.Lock()
cache[address] = rects
mu.Unlock()
}
return rects, err
})
if err != nil {
return nil, err
}
return rectsi.([]Rect), nil
}
func (imp) getUserInfo(ctx context.Context) (*userInfo, error) {
u, err := picago.GetUser(ctxutil.Client(ctx), "default")
if err != nil {
return nil, err
}
return &userInfo{ID: u.ID, Name: u.Name}, nil
}
func (im *imp) ServeSetup(w http.ResponseWriter, r *http.Request, ctx *importer.SetupContext) error {
oauthClient, err := ctx.NewOAuthClient(oAuthURIs)
if err != nil {
err = fmt.Errorf("error getting OAuth client: %v", err)
httputil.ServeError(w, r, err)
return err
}
tempCred, err := oauthClient.RequestTemporaryCredentials(ctxutil.Client(ctx), ctx.CallbackURL(), nil)
if err != nil {
err = fmt.Errorf("Error getting temp cred: %v", err)
httputil.ServeError(w, r, err)
return err
}
if err := ctx.AccountNode.SetAttrs(
importer.AcctAttrTempToken, tempCred.Token,
importer.AcctAttrTempSecret, tempCred.Secret,
); err != nil {
err = fmt.Errorf("Error saving temp creds: %v", err)
httputil.ServeError(w, r, err)
return err
}
authURL := oauthClient.AuthorizationURL(tempCred, nil)
http.Redirect(w, r, authURL, 302)
return nil
}
// urlFileRef slurps urlstr from the net, writes to a file and returns its
// fileref or "" on error
func (r *run) urlFileRef(urlstr, filename string) string {
im := r.im
im.mu.Lock()
if br, ok := im.imageFileRef[urlstr]; ok {
im.mu.Unlock()
return br.String()
}
im.mu.Unlock()
res, err := ctxutil.Client(r).Get(urlstr)
if err != nil {
log.Printf("couldn't get image: %v", err)
return ""
}
defer res.Body.Close()
fileRef, err := schema.WriteFileFromReader(r.Host.Target(), filename, res.Body)
if err != nil {
r.errorf("couldn't write file: %v", err)
return ""
}
im.mu.Lock()
defer im.mu.Unlock()
im.imageFileRef[urlstr] = fileRef
return fileRef.String()
}
func (im extendedOAuth2) ServeCallback(w http.ResponseWriter, r *http.Request, ctx *importer.SetupContext) {
if im.getUserInfo == nil {
panic("No getUserInfo is provided, don't use the default ServeCallback!")
}
oauthConfig, err := im.auth(ctx)
if err != nil {
httputil.ServeError(w, r, fmt.Errorf("Error getting oauth config: %v", err))
return
}
if r.Method != "GET" {
http.Error(w, "Expected a GET", 400)
return
}
code := r.FormValue("code")
if code == "" {
http.Error(w, "Expected a code", 400)
return
}
// picago calls take an *http.Client, so we need to provide one which already
// has a transport set up correctly wrt to authentication. In particular, it
// needs to have the access token that is obtained during Exchange.
transport := &oauth.Transport{
Config: oauthConfig,
Transport: notOAuthTransport(ctxutil.Client(ctx)),
}
token, err := transport.Exchange(code)
log.Printf("Token = %#v, error %v", token, err)
if err != nil {
log.Printf("Token Exchange error: %v", err)
httputil.ServeError(w, r, fmt.Errorf("token exchange error: %v", err))
return
}
picagoCtx, cancel := context.WithCancel(context.WithValue(ctx, ctxutil.HTTPClient, transport.Client()))
defer cancel()
userInfo, err := im.getUserInfo(picagoCtx)
if err != nil {
log.Printf("Couldn't get username: %v", err)
httputil.ServeError(w, r, fmt.Errorf("can't get username: %v", err))
return
}
if err := ctx.AccountNode.SetAttrs(
importer.AcctAttrUserID, userInfo.ID,
importer.AcctAttrGivenName, userInfo.FirstName,
importer.AcctAttrFamilyName, userInfo.LastName,
acctAttrOAuthToken, encodeToken(token),
); err != nil {
httputil.ServeError(w, r, fmt.Errorf("Error setting attribute: %v", err))
return
}
http.Redirect(w, r, ctx.AccountURL(), http.StatusFound)
}
func (imp) ServeCallback(w http.ResponseWriter, r *http.Request, ctx *importer.SetupContext) {
tempToken := ctx.AccountNode.Attr(importer.AcctAttrTempToken)
tempSecret := ctx.AccountNode.Attr(importer.AcctAttrTempSecret)
if tempToken == "" || tempSecret == "" {
log.Printf("flicker: no temp creds in callback")
httputil.BadRequestError(w, "no temp creds in callback")
return
}
if tempToken != r.FormValue("oauth_token") {
log.Printf("unexpected oauth_token: got %v, want %v", r.FormValue("oauth_token"), tempToken)
httputil.BadRequestError(w, "unexpected oauth_token")
return
}
oauthClient, err := ctx.NewOAuthClient(oAuthURIs)
if err != nil {
err = fmt.Errorf("error getting OAuth client: %v", err)
httputil.ServeError(w, r, err)
return
}
tokenCred, vals, err := oauthClient.RequestToken(
ctxutil.Client(ctx),
&oauth.Credentials{
Token: tempToken,
Secret: tempSecret,
},
r.FormValue("oauth_verifier"),
)
if err != nil {
httputil.ServeError(w, r, fmt.Errorf("Error getting request token: %v ", err))
return
}
userID := vals.Get("user_nsid")
if userID == "" {
httputil.ServeError(w, r, fmt.Errorf("Couldn't get user id: %v", err))
return
}
username := vals.Get("username")
if username == "" {
httputil.ServeError(w, r, fmt.Errorf("Couldn't get user name: %v", err))
return
}
// TODO(mpl): get a few more bits of info (first name, last name etc) like I did for twitter, if possible.
if err := ctx.AccountNode.SetAttrs(
importer.AcctAttrAccessToken, tokenCred.Token,
importer.AcctAttrAccessTokenSecret, tokenCred.Secret,
importer.AcctAttrUserID, userID,
importer.AcctAttrUserName, username,
); err != nil {
httputil.ServeError(w, r, fmt.Errorf("Error setting basic account attributes: %v", err))
return
}
http.Redirect(w, r, ctx.AccountURL(), http.StatusFound)
}
// Get fetches through octx the resource defined by url and the values in form.
func (octx OAuthContext) Get(url string, form url.Values) (*http.Response, error) {
if octx.Creds == nil {
return nil, errors.New("No OAuth credentials. Not logged in?")
}
if octx.Client == nil {
return nil, errors.New("No OAuth client.")
}
res, err := octx.Client.Get(ctxutil.Client(octx.Ctx), octx.Creds, url, form)
if err != nil {
return nil, fmt.Errorf("Error fetching %s: %v", url, err)
}
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("Get request on %s failed with: %s", url, res.Status)
}
return res, nil
}
func doGet(ctx context.Context, url string, form url.Values) (*http.Response, error) {
requestURL := url + "?" + form.Encode()
req, err := http.NewRequest("GET", requestURL, nil)
if err != nil {
return nil, err
}
res, err := ctxutil.Client(ctx).Do(req)
if err != nil {
log.Printf("Error fetching %s: %v", url, err)
return nil, err
}
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("Get request on %s failed with: %s", requestURL, res.Status)
}
return res, nil
}
func doGet(ctx context.Context, url string) ([]byte, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
res, err := ctxutil.Client(ctx).Do(req)
if err != nil {
log.Printf("Error fetching %s: %v", url, err)
return nil, err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("Get request on %s failed with: %s", url, res.Status)
}
return ioutil.ReadAll(io.LimitReader(res.Body, 8<<20))
}
func (imp) Run(ctx *importer.RunContext) error {
clientId, secret, err := ctx.Credentials()
if err != nil {
return err
}
acctNode := ctx.AccountNode()
ocfg := baseOAuthConfig
ocfg.ClientId, ocfg.ClientSecret = clientId, secret
token := decodeToken(acctNode.Attr(acctAttrOAuthToken))
transport := &oauth.Transport{
Config: &ocfg,
Token: &token,
Transport: notOAuthTransport(ctxutil.Client(ctx)),
}
ctx.Context = context.WithValue(ctx.Context, ctxutil.HTTPClient, transport.Client())
root := ctx.RootNode()
if root.Attr(nodeattr.Title) == "" {
if err := root.SetAttr(nodeattr.Title,
fmt.Sprintf("%s %s - Google/Picasa Photos",
acctNode.Attr(importer.AcctAttrGivenName),
acctNode.Attr(importer.AcctAttrFamilyName))); err != nil {
return err
}
}
r := &run{
RunContext: ctx,
incremental: !forceFullImport && acctNode.Attr(importer.AcctAttrCompletedVersion) == runCompleteVersion,
photoGate: syncutil.NewGate(3),
}
if err := r.importAlbums(); err != nil {
return err
}
r.mu.Lock()
anyErr := r.anyErr
r.mu.Unlock()
if !anyErr {
if err := acctNode.SetAttrs(importer.AcctAttrCompletedVersion, runCompleteVersion); err != nil {
return err
}
}
return nil
}
// EnumerateObjects lists the objects in a bucket.
// This function relies on the ctx oauth2.HTTPClient value being set to an OAuth2
// authorized and authenticated HTTP client.
// If after is non-empty, listing will begin with lexically greater object names.
// If limit is non-zero, the length of the list will be limited to that number.
func EnumerateObjects(ctx context.Context, bucket, after string, limit int) ([]*storage.ObjectAttrs, error) {
// Build url, with query params
var params []string
if after != "" {
params = append(params, "marker="+url.QueryEscape(after))
}
if limit > 0 {
params = append(params, fmt.Sprintf("max-keys=%v", limit))
}
query := ""
if len(params) > 0 {
query = "?" + strings.Join(params, "&")
}
req, err := simpleRequest("GET", gsAccessURL+"/"+bucket+"/"+query)
if err != nil {
return nil, err
}
req.Cancel = ctx.Done()
res, err := ctxutil.Client(ctx).Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("gcsutil: bad enumerate response code: %v", res.Status)
}
var xres struct {
Contents []SizedObject
}
if err = xml.NewDecoder(res.Body).Decode(&xres); err != nil {
return nil, err
}
objAttrs := make([]*storage.ObjectAttrs, len(xres.Contents))
for k, o := range xres.Contents {
objAttrs[k] = &storage.ObjectAttrs{
Name: o.Key,
Size: o.Size,
}
}
return objAttrs, nil
}
func (r *run) importAlbums(ctx context.Context) error {
albums, err := picago.GetAlbums(ctxutil.Client(ctx), "default")
if err != nil {
return fmt.Errorf("importAlbums: error listing albums: %v", err)
}
albumsNode, err := r.getTopLevelNode("albums", "Albums")
for _, album := range albums {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if err := r.importAlbum(ctx, albumsNode, album); err != nil {
return fmt.Errorf("picasa importer: error importing album %s: %v", album, err)
}
}
return nil
}
func init() {
importer.Register("picasa", imp{
newExtendedOAuth2(
baseOAuthConfig,
func(ctx context.Context) (*userInfo, error) {
u, err := picago.GetUser(ctxutil.Client(ctx), "default")
if err != nil {
return nil, err
}
firstName, lastName := u.Name, ""
i := strings.LastIndex(u.Name, " ")
if i >= 0 {
firstName, lastName = u.Name[:i], u.Name[i+1:]
}
return &userInfo{
ID: u.ID,
FirstName: firstName,
LastName: lastName,
}, nil
}),
})
}
// GetPartialObject fetches part of a Google Cloud Storage object.
// This function relies on the ctx ctxutil.HTTPClient value being set to an OAuth2
// authorized and authenticated HTTP client.
// If length is negative, the rest of the object is returned.
// It returns ErrInvalidRange if the server replies with http.StatusRequestedRangeNotSatisfiable.
// The caller must call Close on the returned value.
func GetPartialObject(ctx context.Context, obj Object, offset, length int64) (io.ReadCloser, error) {
if offset < 0 {
return nil, errors.New("invalid negative offset")
}
if err := obj.valid(); err != nil {
return nil, err
}
req, err := simpleRequest("GET", gsAccessURL+"/"+obj.Bucket+"/"+obj.Key)
if err != nil {
return nil, err
}
if length >= 0 {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1))
} else {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
}
req.Cancel = ctx.Done()
res, err := ctxutil.Client(ctx).Do(req)
if err != nil {
return nil, fmt.Errorf("GET (offset=%d, length=%d) failed: %v\n", offset, length, err)
}
if res.StatusCode == http.StatusNotFound {
res.Body.Close()
return nil, os.ErrNotExist
}
if !(res.StatusCode == http.StatusPartialContent || (offset == 0 && res.StatusCode == http.StatusOK)) {
res.Body.Close()
if res.StatusCode == http.StatusRequestedRangeNotSatisfiable {
return nil, ErrInvalidRange
}
return nil, fmt.Errorf("GET (offset=%d, length=%d) got failed status: %v\n", offset, length, res.Status)
}
return res.Body, nil
}
func (r *run) importBatch(authToken string, parent *importer.Object) (keepTrying bool, err error) {
sleepDuration := r.nextAfter.Sub(time.Now())
// block until we either get canceled or until it is time to run
select {
case <-r.Context().Done():
log.Printf("pinboard: Importer interrupted.")
return false, r.Context().Err()
case <-time.After(sleepDuration):
// just proceed
}
start := time.Now()
u := fmt.Sprintf(fetchUrl, authToken, batchLimit, r.nextCursor)
resp, err := ctxutil.Client(r.Context()).Get(u)
if err != nil {
return false, err
}
defer resp.Body.Close()
switch {
case resp.StatusCode == StatusTooManyRequests:
r.lastPause = r.lastPause * 2
r.nextAfter = time.Now().Add(r.lastPause)
return true, nil
case resp.StatusCode != http.StatusOK:
return false, fmt.Errorf("Unexpected status code %v fetching %v", resp.StatusCode, u)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return false, err
}
var postBatch []apiPost
if err = json.Unmarshal(body, &postBatch); err != nil {
return false, err
}
if err != nil {
return false, err
}
postCount := len(postBatch)
if postCount == 0 {
// we are done!
return false, nil
}
log.Printf("pinboard: Importing %d posts...", postCount)
var grp syncutil.Group
for _, post := range postBatch {
select {
case <-r.Context().Done():
log.Printf("pinboard: Importer interrupted")
return false, r.Context().Err()
default:
}
post := post
r.postGate.Start()
grp.Go(func() error {
defer r.postGate.Done()
return r.importPost(&post, parent)
})
}
log.Printf("pinboard: Imported batch of %d posts in %s.", postCount, time.Now().Sub(start))
r.nextCursor = postBatch[postCount-1].Time
r.lastPause = pauseInterval
r.nextAfter = time.Now().Add(pauseInterval)
tryAgain := postCount == batchLimit
return tryAgain, grp.Err()
}
// viaAPI is true if it came via the REST API, or false if it came via a zip file.
func (r *run) importTweet(parent *importer.Object, tweet tweetItem, viaAPI bool) (dup bool, err error) {
select {
case <-r.Context().Done():
r.errorf("Twitter importer: interrupted")
return false, r.Context().Err()
default:
}
id := tweet.ID()
tweetNode, err := parent.ChildPathObject(id)
if err != nil {
return false, err
}
// Because the zip format and the API format differ a bit, and
// might diverge more in the future, never use the zip content
// to overwrite data fetched via the API. If we add new
// support for different fields in the future, we might want
// to revisit this decision. Be wary of flip/flopping data if
// modifying this, though.
if tweetNode.Attr(attrImportMethod) == "api" && !viaAPI {
return true, nil
}
// e.g. "2014-06-12 19:11:51 +0000"
createdTime, err := timeParseFirstFormat(tweet.CreatedAt(), time.RubyDate, "2006-01-02 15:04:05 -0700")
if err != nil {
return false, fmt.Errorf("could not parse time %q: %v", tweet.CreatedAt(), err)
}
url := fmt.Sprintf("https://twitter.com/%s/status/%v",
r.AccountNode().Attr(importer.AcctAttrUserName),
id)
attrs := []string{
"twitterId", id,
nodeattr.Type, "twitter.com:tweet",
nodeattr.StartDate, schema.RFC3339FromTime(createdTime),
nodeattr.Content, tweet.Text(),
nodeattr.URL, url,
}
if lat, long, ok := tweet.LatLong(); ok {
attrs = append(attrs,
nodeattr.Latitude, fmt.Sprint(lat),
nodeattr.Longitude, fmt.Sprint(long),
)
}
if viaAPI {
attrs = append(attrs, attrImportMethod, "api")
} else {
attrs = append(attrs, attrImportMethod, "zip")
}
for i, m := range tweet.Media() {
filename := m.BaseFilename()
if tweetNode.Attr("camliPath:"+filename) != "" && (i > 0 || tweetNode.Attr("camliContentImage") != "") {
// Don't re-import media we've already fetched.
continue
}
tried, gotMedia := 0, false
for _, mediaURL := range m.URLs() {
tried++
res, err := ctxutil.Client(r.Context()).Get(mediaURL)
if err != nil {
return false, fmt.Errorf("Error fetching %s for tweet %s : %v", mediaURL, url, err)
}
if res.StatusCode == http.StatusNotFound {
continue
}
if res.StatusCode != 200 {
return false, fmt.Errorf("HTTP status %d fetching %s for tweet %s", res.StatusCode, mediaURL, url)
}
if !viaAPI {
log.Printf("For zip tweet %s, reading %v", url, mediaURL)
}
fileRef, err := schema.WriteFileFromReader(r.Host.Target(), filename, res.Body)
res.Body.Close()
if err != nil {
return false, fmt.Errorf("Error fetching media %s for tweet %s: %v", mediaURL, url, err)
}
attrs = append(attrs, "camliPath:"+filename, fileRef.String())
if i == 0 {
attrs = append(attrs, "camliContentImage", fileRef.String())
}
log.Printf("Slurped %s as %s for tweet %s (%v)", mediaURL, fileRef.String(), url, tweetNode.PermanodeRef())
gotMedia = true
break
}
if !gotMedia && tried > 0 {
return false, fmt.Errorf("All media URLs 404s for tweet %s", url)
}
}
changes, err := tweetNode.SetAttrs2(attrs...)
if err == nil && changes {
log.Printf("Imported tweet %s", url)
}
return !changes, err
}
func (r *run) updatePhotoInAlbum(ctx context.Context, albumNode *importer.Object, photo picago.Photo) (ret error) {
if photo.ID == "" {
return errors.New("photo has no ID")
}
getMediaBytes := func() (io.ReadCloser, error) {
log.Printf("Importing media from %v", photo.URL)
resp, err := ctxutil.Client(ctx).Get(photo.URL)
if err != nil {
return nil, fmt.Errorf("importing photo %s: %v", photo.ID, err)
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, fmt.Errorf("importing photo %s: status code = %d", photo.ID, resp.StatusCode)
}
return resp.Body, nil
}
var fileRefStr string
idFilename := photo.ID + "-" + photo.Filename
photoNode, err := albumNode.ChildPathObjectOrFunc(idFilename, func() (*importer.Object, error) {
h := blob.NewHash()
rc, err := getMediaBytes()
if err != nil {
return nil, err
}
fileRef, err := schema.WriteFileFromReader(r.Host.Target(), photo.Filename, io.TeeReader(rc, h))
if err != nil {
return nil, err
}
fileRefStr = fileRef.String()
wholeRef := blob.RefFromHash(h)
if pn, err := findExistingPermanode(r.Host.Searcher(), wholeRef); err == nil {
return r.Host.ObjectFromRef(pn)
}
return r.Host.NewObject()
})
if err != nil {
return err
}
const attrMediaURL = "picasaMediaURL"
if fileRefStr == "" {
fileRefStr = photoNode.Attr(nodeattr.CamliContent)
// Only re-download the source photo if its URL has changed.
// Empirically this seems to work: cropping a photo in the
// photos.google.com UI causes its URL to change. And it makes
// sense, looking at the ugliness of the URLs with all their
// encoded/signed state.
if !mediaURLsEqual(photoNode.Attr(attrMediaURL), photo.URL) {
rc, err := getMediaBytes()
if err != nil {
return err
}
fileRef, err := schema.WriteFileFromReader(r.Host.Target(), photo.Filename, rc)
rc.Close()
if err != nil {
return err
}
fileRefStr = fileRef.String()
}
}
title := strings.TrimSpace(photo.Description)
if strings.Contains(title, "\n") {
title = title[:strings.Index(title, "\n")]
}
if title == "" && schema.IsInterestingTitle(photo.Filename) {
title = photo.Filename
}
// TODO(tgulacsi): add more attrs (comments ?)
// for names, see http://schema.org/ImageObject and http://schema.org/CreativeWork
attrs := []string{
nodeattr.CamliContent, fileRefStr,
attrPicasaId, photo.ID,
nodeattr.Title, title,
nodeattr.Description, photo.Description,
nodeattr.LocationText, photo.Location,
nodeattr.DateModified, schema.RFC3339FromTime(photo.Updated),
nodeattr.DatePublished, schema.RFC3339FromTime(photo.Published),
nodeattr.URL, photo.PageURL,
}
if photo.Latitude != 0 || photo.Longitude != 0 {
attrs = append(attrs,
nodeattr.Latitude, fmt.Sprintf("%f", photo.Latitude),
nodeattr.Longitude, fmt.Sprintf("%f", photo.Longitude),
)
}
if err := photoNode.SetAttrs(attrs...); err != nil {
return err
}
if err := photoNode.SetAttrValues("tag", photo.Keywords); err != nil {
return err
}
if photo.Position > 0 {
if err := albumNode.SetAttr(
nodeattr.CamliPathOrderColon+strconv.Itoa(photo.Position-1),
photoNode.PermanodeRef().String()); err != nil {
return err
}
}
// Do this last, after we're sure the "camliContent" attribute
// has been saved successfully, because this is the one that
// causes us to do it again in the future or not.
if err := photoNode.SetAttrs(attrMediaURL, photo.URL); err != nil {
return err
}
return nil
}
func (r *run) importAlbum(ctx context.Context, albumsNode *importer.Object, album picago.Album) (ret error) {
if album.ID == "" {
return errors.New("album has no ID")
}
albumNode, err := albumsNode.ChildPathObject(album.ID)
if err != nil {
return fmt.Errorf("importAlbum: error listing album: %v", err)
}
dateMod := schema.RFC3339FromTime(album.Updated)
// Data reference: https://developers.google.com/picasa-web/docs/2.0/reference
// TODO(tgulacsi): add more album info
changes, err := albumNode.SetAttrs2(
attrPicasaId, album.ID,
nodeattr.Type, "picasaweb.google.com:album",
nodeattr.Title, album.Title,
nodeattr.DatePublished, schema.RFC3339FromTime(album.Published),
nodeattr.LocationText, album.Location,
nodeattr.Description, album.Description,
nodeattr.URL, album.URL,
)
if err != nil {
return fmt.Errorf("error setting album attributes: %v", err)
}
if !changes && r.incremental && albumNode.Attr(nodeattr.DateModified) == dateMod {
return nil
}
defer func() {
// Don't update DateModified on the album node until
// we've successfully imported all the photos.
if ret == nil {
ret = albumNode.SetAttr(nodeattr.DateModified, dateMod)
}
}()
log.Printf("Importing album %v: %v/%v (published %v, updated %v)", album.ID, album.Name, album.Title, album.Published, album.Updated)
// TODO(bradfitz): GetPhotos does multiple HTTP requests to
// return a slice of all photos. My "InstantUpload/Auto
// Backup" album has 6678 photos (and growing) and this
// currently takes like 40 seconds. Fix.
photos, err := picago.GetPhotos(ctxutil.Client(ctx), "default", album.ID)
if err != nil {
return err
}
log.Printf("Importing %d photos from album %q (%s)", len(photos), albumNode.Attr(nodeattr.Title),
albumNode.PermanodeRef())
var grp syncutil.Group
for i := range photos {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
photo := photos[i]
r.photoGate.Start()
grp.Go(func() error {
defer r.photoGate.Done()
return r.updatePhotoInAlbum(ctx, albumNode, photo)
})
}
return grp.Err()
}
