func iterItems(itemch chan<- imageFile, errch chan<- error,
filter filterFunc, client *http.Client, username string) {
defer close(itemch)
albums, err := picago.GetAlbums(client, username)
if err != nil {
errch <- err
return
}
gate := syncutil.NewGate(parallelAlbumRoutines)
for _, album := range albums {
photos, err := picago.GetPhotos(client, username, album.ID)
if err != nil {
select {
case errch <- err:
default:
return
}
continue
}
gate.Start()
go func(albumName, albumTitle string) {
defer gate.Done()
for _, photo := range photos {
img := imageFile{
albumTitle: albumTitle,
albumName: albumName,
fileName: photo.Filename(),
ID: photo.ID,
}
ok, err := filter(img)
if err != nil {
errch <- err
return
}
if !ok {
continue
}
img.r, err = picago.DownloadPhoto(client, photo.URL)
if err != nil {
select {
case errch <- fmt.Errorf("Get(%s): %v", photo.URL, err):
default:
return
}
continue
}
itemch <- img
}
}(album.Name, album.Title)
}
}
func (r *run) importAlbum(albumsNode *importer.Object, album picago.Album) (ret error) {
if album.ID == "" {
return errors.New("album has no ID")
}
albumNode, err := albumsNode.ChildPathObject(album.ID)
if err != nil {
return fmt.Errorf("importAlbum: error listing album: %v", err)
}
dateMod := schema.RFC3339FromTime(album.Updated)
// Data reference: https://developers.google.com/picasa-web/docs/2.0/reference
// TODO(tgulacsi): add more album info
changes, err := albumNode.SetAttrs2(
attrPicasaId, album.ID,
nodeattr.Type, "picasaweb.google.com:album",
nodeattr.Title, album.Title,
nodeattr.DatePublished, schema.RFC3339FromTime(album.Published),
nodeattr.LocationText, album.Location,
nodeattr.Description, album.Description,
nodeattr.URL, album.URL,
)
if err != nil {
return fmt.Errorf("error setting album attributes: %v", err)
}
if !changes && r.incremental && albumNode.Attr(nodeattr.DateModified) == dateMod {
return nil
}
defer func() {
// Don't update DateModified on the album node until
// we've successfully imported all the photos.
if ret == nil {
ret = albumNode.SetAttr(nodeattr.DateModified, dateMod)
}
}()
log.Printf("Importing album %v: %v/%v (published %v, updated %v)", album.ID, album.Name, album.Title, album.Published, album.Updated)
// TODO(bradfitz): GetPhotos does multiple HTTP requests to
// return a slice of all photos. My "InstantUpload/Auto
// Backup" album has 6678 photos (and growing) and this
// currently takes like 40 seconds. Fix.
photos, err := picago.GetPhotos(r.HTTPClient(), "default", album.ID)
if err != nil {
return err
}
log.Printf("Importing %d photos from album %q (%s)", len(photos), albumNode.Attr(nodeattr.Title),
albumNode.PermanodeRef())
var grp syncutil.Group
for i := range photos {
if r.Context.IsCanceled() {
return context.ErrCanceled
}
photo := photos[i]
r.photoGate.Start()
grp.Go(func() error {
defer r.photoGate.Done()
return r.updatePhotoInAlbum(albumNode, photo)
})
}
return grp.Err()
}