Example #1
0
func (r *run) importPlace(parent *importer.Object, place *venueItem) (*importer.Object, error) {
	placeNode, err := parent.ChildPathObject(place.Id)
	if err != nil {
		return nil, err
	}

	catName := ""
	if cat := place.primaryCategory(); cat != nil {
		catName = cat.Name
	}

	icon := place.icon()
	if err := placeNode.SetAttrs(
		"foursquareId", place.Id,
		"camliNodeType", "foursquare.com:venue",
		"camliContentImage", r.urlFileRef(icon, path.Base(icon)),
		"foursquareCategoryName", catName,
		"title", place.Name,
		"streetAddress", place.Location.Address,
		"addressLocality", place.Location.City,
		"postalCode", place.Location.PostalCode,
		"addressRegion", place.Location.State,
		"addressCountry", place.Location.Country,
		"latitude", fmt.Sprint(place.Location.Lat),
		"longitude", fmt.Sprint(place.Location.Lng)); err != nil {
		return nil, err
	}

	return placeNode, nil
}
Example #2
0
func (*imp) IsAccountReady(acct *importer.Object) (ready bool, err error) {
	// This method tells the importer framework whether this account
	// permanode (accessed via the importer.Object) is ready to start
	// an import.  Here you would typically check whether you have the
	// right metadata/tokens on the account.
	return acct.Attr(acctAttrToken) != "" && acct.Attr(acctAttrUsername) != "", nil
}
Example #3
0
func (r *run) importPost(post *apiPost, parent *importer.Object) error {
	postNode, err := parent.ChildPathObject(post.Hash)
	if err != nil {
		return err
	}

	t, err := time.Parse(timeFormat, post.Time)
	if err != nil {
		return err
	}

	attrs := []string{
		"pinboard.in:hash", post.Hash,
		nodeattr.Type, "pinboard.in:post",
		nodeattr.DateCreated, schema.RFC3339FromTime(t),
		nodeattr.Title, post.Description,
		nodeattr.URL, post.Href,
		"pinboard.in:extended", post.Extended,
		"pinboard.in:meta", post.Meta,
		"pinboard.in:shared", post.Shared,
		"pinboard.in:toread", post.ToRead,
	}
	if err = postNode.SetAttrs(attrs...); err != nil {
		return err
	}
	if err = postNode.SetAttrValues("tag", strings.Split(post.Tags, " ")); err != nil {
		return err
	}

	return nil
}
Example #4
0
func (r *run) importPlace(parent *importer.Object, place *venueItem) (*importer.Object, error) {
	placeNode, err := parent.ChildPathObject(place.Id)
	if err != nil {
		return nil, err
	}

	catName := ""
	if cat := place.primaryCategory(); cat != nil {
		catName = cat.Name
	}

	icon := place.icon()
	if err := placeNode.SetAttrs(
		attrFoursquareId, place.Id,
		nodeattr.Type, "foursquare.com:venue",
		nodeattr.CamliContentImage, r.urlFileRef(icon, path.Base(icon)),
		attrFoursquareCategoryName, catName,
		nodeattr.Title, place.Name,
		nodeattr.StreetAddress, place.Location.Address,
		nodeattr.AddressLocality, place.Location.City,
		nodeattr.PostalCode, place.Location.PostalCode,
		nodeattr.AddressRegion, place.Location.State,
		nodeattr.AddressCountry, place.Location.Country,
		nodeattr.Latitude, fmt.Sprint(place.Location.Lat),
		nodeattr.Longitude, fmt.Sprint(place.Location.Lng)); err != nil {
		return nil, err
	}

	return placeNode, nil
}
Example #5
0
// updatePrimaryPhoto uses the camliContent of photoNode to set the
// camliContentImage of any album for which photoNode is the primary photo.
func (r *run) updatePrimaryPhoto(photoNode *importer.Object) error {
	photoId := photoNode.Attr(attrFlickrId)
	for album, photo := range r.primaryPhoto {
		if photoId != photo {
			continue
		}
		setsNode, err := r.getTopLevelNode("sets", "Sets")
		if err != nil {
			return fmt.Errorf("could not set %v as primary photo of %v, no root sets: %v", photoId, album, err)
		}
		setNode, err := setsNode.ChildPathObject(album)
		if err != nil {
			return fmt.Errorf("could not set %v as primary photo of %v, no album: %v", photoId, album, err)
		}
		fileRef := photoNode.Attr(nodeattr.CamliContent)
		if fileRef == "" {
			return fmt.Errorf("could not set %v as primary photo of %v: fileRef of photo is unknown", photoId, album)
		}
		if err := setNode.SetAttr(nodeattr.CamliContentImage, fileRef); err != nil {
			return fmt.Errorf("could not set %v as primary photo of %v: %v", photoId, album, err)
		}
		delete(r.primaryPhoto, album)
	}
	return nil
}
Example #6
0
func (im imp) SummarizeAccount(acct *importer.Object) string {
	ok, err := im.IsAccountReady(acct)
	if err != nil || !ok {
		return ""
	}
	return acct.Attr(importer.AcctAttrUserName)
}
Example #7
0
func (im imp) SetTestAccount(acctNode *importer.Object) error {
	return acctNode.SetAttrs(
		importer.AcctAttrAccessToken, "fakeAccessToken",
		importer.AcctAttrAccessTokenSecret, "fakeAccessSecret",
		importer.AcctAttrUserID, "fakeUserId",
		importer.AcctAttrName, "fakeName",
		importer.AcctAttrUserName, "fakeScreenName",
	)
}
Example #8
0
func (im *imp) SummarizeAccount(acct *importer.Object) string {
	ok, err := im.IsAccountReady(acct)
	if err != nil {
		return "Not configured; error = " + err.Error()
	}
	if !ok {
		return "Not configured"
	}
	return fmt.Sprintf("feed %s", acct.Attr(acctAttrFeedURL))
}
Example #9
0
func (im imp) SummarizeAccount(acct *importer.Object) string {
	ok, err := im.IsAccountReady(acct)
	if err != nil {
		return "Not configured; error = " + err.Error()
	}
	if !ok {
		return "Not configured"
	}
	return fmt.Sprintf("Pinboard account for %s", extractUsername(acct.Attr(attrAuthToken)))
}
Example #10
0
func (im *imp) SetTestAccount(acctNode *importer.Object) error {
	// TODO(mpl): refactor with twitter
	return acctNode.SetAttrs(
		importer.AcctAttrAccessToken, "fakeAccessToken",
		importer.AcctAttrAccessTokenSecret, "fakeAccessSecret",
		importer.AcctAttrUserID, "fakeUserID",
		importer.AcctAttrName, "fakeName",
		importer.AcctAttrUserName, "fakeScreenName",
	)
}
Example #11
0
func (r *run) importPhotoset(parent *importer.Object, photoset *photosetInfo, page int) (int, error) {
	photosetNode, err := parent.ChildPathObject(photoset.Id)
	if err != nil {
		return 0, err
	}

	if err := photosetNode.SetAttrs(
		attrFlickrId, photoset.Id,
		nodeattr.Title, photoset.Title.Content,
		nodeattr.Description, photoset.Description.Content); err != nil {
		return 0, err
	}
	// keep track of primary photo so we can set the fileRef of the photo as CamliContentImage
	// on photosetNode when we eventually know that fileRef.
	r.primaryPhoto[photoset.Id] = photoset.PrimaryPhotoId

	resp := struct {
		Photoset photosetItems
	}{}
	if err := r.flickrAPIRequest(&resp, photosetAPIPath, "user_id", r.userID,
		"page", fmt.Sprintf("%d", page), "photoset_id", photoset.Id, "extras", "original_format"); err != nil {
		return 0, err
	}

	log.Printf("Importing page %d from photoset %s", page, photoset.Id)

	photosNode, err := r.getPhotosNode()
	if err != nil {
		return 0, err
	}

	for _, item := range resp.Photoset.Photo {
		filename := fmt.Sprintf("%s.%s", item.Id, item.OriginalFormat)
		photoNode, err := photosNode.ChildPathObject(filename)
		if err != nil {
			log.Printf("Flickr importer: error finding photo node %s for addition to photoset %s: %s",
				item.Id, photoset.Id, err)
			continue
		}
		if err := photosetNode.SetAttr("camliPath:"+filename, photoNode.PermanodeRef().String()); err != nil {
			log.Printf("Flickr importer: error adding photo %s to photoset %s: %s",
				item.Id, photoset.Id, err)
		}
	}

	if resp.Photoset.Page < resp.Photoset.Pages {
		return page + 1, nil
	} else {
		return 0, nil
	}
}
Example #12
0
func findChildRefs(parent *importer.Object) ([]blob.Ref, error) {
	childRefs := []blob.Ref{}
	var err error
	parent.ForeachAttr(func(key, value string) {
		if strings.HasPrefix(key, "camliPath:") {
			if br, ok := blob.Parse(value); ok {
				childRefs = append(childRefs, br)
				return
			}
			if err == nil {
				err = fmt.Errorf("invalid blobRef for %s attribute of %v: %q", key, parent, value)
			}
		}
	})
	return childRefs, err
}
Example #13
0
func (im *imp) importPhotoset(parent *importer.Object, photoset *photosetsGetListItem, page int) (int, error) {
	photosetNode, err := parent.ChildPathObject(photoset.Id)
	if err != nil {
		return 0, err
	}

	if err := photosetNode.SetAttrs(
		"flickrId", photoset.Title.Content,
		"title", photoset.Title.Content,
		"description", photoset.Description.Content,
		"primaryPhotoId", photoset.PrimaryPhotoId); err != nil {
		return 0, err
	}

	resp := photosetsGetPhotos{}
	if err := im.flickrAPIRequest(&resp, "flickr.photosets.getPhotos",
		"page", fmt.Sprintf("%d", page), "photoset_id", photoset.Id, "extras", "original_format"); err != nil {
		return 0, err
	}

	log.Printf("Importing page %d from photoset %s", page, photoset.Id)

	photosNode, err := im.getPhotosNode()
	if err != nil {
		return 0, err
	}

	for _, item := range resp.Photoset.Photo {
		filename := fmt.Sprintf("%s.%s", item.Id, item.Originalformat)
		photoNode, err := photosNode.ChildPathObject(filename)
		if err != nil {
			log.Printf("Flickr importer: error finding photo node %s for addition to photoset %s: %s",
				item.Id, photoset.Id, err)
			continue
		}
		if err := photosetNode.SetAttr("camliPath:"+filename, photoNode.PermanodeRef().String()); err != nil {
			log.Printf("Flickr importer: error adding photo %s to photoset %s: %s",
				item.Id, photoset.Id, err)
		}
	}

	if resp.Photoset.Page < resp.Photoset.Pages {
		return page + 1, nil
	} else {
		return 0, nil
	}
}
Example #14
0
func (r *run) importCheckin(parent *importer.Object, checkin *checkinItem, placeRef blob.Ref) (checkinNode *importer.Object, dup bool, err error) {
	checkinNode, err = parent.ChildPathObject(checkin.Id)
	if err != nil {
		return
	}

	title := fmt.Sprintf("Checkin at %s", checkin.Venue.Name)
	dup = checkinNode.Attr(nodeattr.StartDate) != ""
	if err := checkinNode.SetAttrs(
		attrFoursquareId, checkin.Id,
		attrFoursquareVenuePermanode, placeRef.String(),
		nodeattr.Type, "foursquare.com:checkin",
		nodeattr.StartDate, schema.RFC3339FromTime(time.Unix(checkin.CreatedAt, 0)),
		nodeattr.Title, title); err != nil {
		return nil, false, err
	}
	return checkinNode, dup, nil
}
Example #15
0
func (r *run) importPhotos(placeNode *importer.Object) error {
	photosNode, err := placeNode.ChildPathObject("photos")
	if err != nil {
		return err
	}

	if err := photosNode.SetAttrs(
		"title", "Photos of "+placeNode.Attr("title"),
		"camliDefVis", "hide"); err != nil {
		return err
	}

	resp := photosList{}
	if err := r.im.doAPI(r.Context, r.token(), &resp, "venues/"+placeNode.Attr("foursquareId")+"/photos", "limit", "10"); err != nil {
		return err
	}

	var need []*photoItem
	for _, photo := range resp.Response.Photos.Items {
		attr := "camliPath:" + photo.Id + filepath.Ext(photo.Suffix)
		if photosNode.Attr(attr) == "" {
			need = append(need, photo)
		}
	}

	if len(need) > 0 {
		log.Printf("foursquare: importing %d photos for venue %s", len(need), placeNode.Attr("title"))
		for _, photo := range need {
			attr := "camliPath:" + photo.Id + filepath.Ext(photo.Suffix)
			url := photo.Prefix + "original" + photo.Suffix
			log.Printf("foursquare: importing photo for venue %s: %s", placeNode.Attr("title"), url)
			ref := r.urlFileRef(url, "")
			if ref == "" {
				log.Printf("Error slurping photo: %s", url)
				continue
			}
			if err := photosNode.SetAttr(attr, ref); err != nil {
				log.Printf("Error adding venue photo: %#v", err)
			}
		}
	}

	return nil
}
Example #16
0
func (r *run) importCheckin(parent *importer.Object, checkin *checkinItem, placeRef blob.Ref) (*importer.Object, error) {
	checkinNode, err := parent.ChildPathObject(checkin.Id)
	if err != nil {
		return nil, err
	}

	title := fmt.Sprintf("Checkin at %s", checkin.Venue.Name)

	if err := checkinNode.SetAttrs(
		"foursquareId", checkin.Id,
		"foursquareVenuePermanode", placeRef.String(),
		"camliNodeType", "foursquare.com:checkin",
		"startDate", schema.RFC3339FromTime(time.Unix(checkin.CreatedAt, 0)),
		"title", title); err != nil {
		return nil, err
	}

	return checkinNode, nil
}
Example #17
0
// TODO(aa):
// * Parallelize: http://golang.org/doc/effective_go.html#concurrency
// * Do more than one "page" worth of results
// * Report progress and errors back through host interface
// * All the rest of the metadata (see photoMeta)
// * Conflicts: For all metadata changes, prefer any non-imported claims
// * Test!
func (im *imp) importPhoto(parent *importer.Object, photo *photosSearchItem) error {
	filename := fmt.Sprintf("%s.%s", photo.Id, photo.Originalformat)
	photoNode, err := parent.ChildPathObject(filename)
	if err != nil {
		return err
	}

	// Import all the metadata. SetAttrs() is a no-op if the value hasn't changed, so there's no cost to doing these on every run.
	// And this way if we add more things to import, they will get picked up.
	if err := photoNode.SetAttrs(
		"flickrId", photo.Id,
		"title", photo.Title,
		"description", photo.Description.Content); err != nil {
		return err
	}

	// Import the photo itself. Since it is expensive to fetch the image, we store its lastupdate and only refetch if it might have changed.
	if photoNode.Attr("flickrLastupdate") == photo.Lastupdate {
		return nil
	}
	res, err := im.flickrRequest(photo.URL, url.Values{})
	if err != nil {
		log.Printf("Flickr importer: Could not fetch %s: %s", photo.URL, err)
		return err
	}
	defer res.Body.Close()

	fileRef, err := schema.WriteFileFromReader(im.host.Target(), filename, res.Body)
	if err != nil {
		return err
	}
	if err := photoNode.SetAttr("camliContent", fileRef.String()); err != nil {
		return err
	}
	// Write lastupdate last, so that if any of the preceding fails, we will try again next time.
	if err := photoNode.SetAttr("flickrLastupdate", photo.Lastupdate); err != nil {
		return err
	}

	return nil
}
Example #18
0
func (r *run) importTweet(parent *importer.Object, tweet *tweetItem) error {
	tweetNode, err := parent.ChildPathObject(tweet.Id)
	if err != nil {
		return err
	}

	title := "Tweet id " + tweet.Id

	createdTime, err := time.Parse(time.RubyDate, tweet.CreatedAt)
	if err != nil {
		return fmt.Errorf("could not parse time %q: %v", tweet.CreatedAt, err)
	}

	// TODO: import photos referenced in tweets
	return tweetNode.SetAttrs(
		"twitterId", tweet.Id,
		"camliNodeType", "twitter.com:tweet",
		"startDate", schema.RFC3339FromTime(createdTime),
		"content", tweet.Text,
		"title", title)
}
Example #19
0
func (r *run) importItem(parent *importer.Object, item *item) error {
	itemNode, err := parent.ChildPathObject(item.ID)
	if err != nil {
		return err
	}
	fileRef, err := schema.WriteFileFromReader(r.Host.Target(), "", bytes.NewBufferString(item.Content))
	if err != nil {
		return err
	}
	if err := itemNode.SetAttrs(
		"feedItemId", item.ID,
		"camliNodeType", "feed:item",
		"title", item.Title,
		"link", item.Link,
		"author", item.Author,
		"camliContent", fileRef.String(),
		"feedMediaContentURL", item.MediaContent,
	); err != nil {
		return err
	}
	return nil
}
Example #20
0
func (r *run) importCompanions(parent *importer.Object, companions []*user) (companionRefs []string, err error) {
	for _, user := range companions {
		personNode, err := parent.ChildPathObject(user.Id)
		if err != nil {
			return nil, err
		}
		attrs := []string{
			attrFoursquareId, user.Id,
			nodeattr.Type, "foursquare.com:person",
			nodeattr.Title, user.FirstName + " " + user.LastName,
			nodeattr.GivenName, user.FirstName,
			nodeattr.FamilyName, user.LastName,
		}
		if icon := user.icon(); icon != "" {
			attrs = append(attrs, nodeattr.CamliContentImage, r.urlFileRef(icon, path.Base(icon)))
		}
		if err := personNode.SetAttrs(attrs...); err != nil {
			return nil, err
		}
		companionRefs = append(companionRefs, personNode.PermanodeRef().String())
	}
	return companionRefs, nil
}
Example #21
0
func (im *imp) SummarizeAccount(acct *importer.Object) string {
	ok, err := im.IsAccountReady(acct)
	if err != nil {
		return "Not configured; error = " + err.Error()
	}
	if !ok {
		return "Not configured"
	}
	s := fmt.Sprintf("@%s (%s), twitter id %s",
		acct.Attr(importer.AcctAttrUserName),
		acct.Attr(importer.AcctAttrName),
		acct.Attr(importer.AcctAttrUserID),
	)
	if acct.Attr(acctAttrTweetZip) != "" {
		s += " + zip file"
	}
	return s
}
Example #22
0
func (r *run) importPhotos(placeNode *importer.Object) error {
	photosNode, err := placeNode.ChildPathObject("photos")
	if err != nil {
		return err
	}

	photosNode.SetAttrs(
		"title", "Photos of "+placeNode.Attr("title"),
		"camliDefVis", "hide")

	resp := photosList{}
	if err := r.im.doAPI(r.Context, r.token(), &resp, "venues/"+placeNode.Attr("foursquareId")+"/photos", "limit", "10"); err != nil {
		return err
	}

	itemcount := len(resp.Response.Photos.Items)
	log.Printf("Importing %d photos for venue %s", itemcount, placeNode.Attr("title"))

	for _, photo := range resp.Response.Photos.Items {
		attr := "camliPath:" + photo.Id + filepath.Ext(photo.Suffix)
		if photosNode.Attr(attr) != "" {
			log.Printf("Skipping photo, we already have it")
			// Assume we have this photo already and don't need to refetch.
			continue
		}
		url := photo.Prefix + "original" + photo.Suffix
		ref := r.urlFileRef(url, "")
		if ref == "" {
			log.Printf("Error slurping photo: %s", url)
			continue
		}
		err = photosNode.SetAttr(attr, ref)
		if err != nil {
			log.Printf("Error adding venue photo: %#v", err)
		}
	}

	return nil
}
Example #23
0
func getRequiredChildPathObj(parent *importer.Object, path string) (*importer.Object, error) {
	return parent.ChildPathObjectOrFunc(path, func() (*importer.Object, error) {
		return nil, fmt.Errorf("Unable to locate child path %s of node %v", path, parent.PermanodeRef())
	})
}
Example #24
0
func (r *run) importAlbum(ctx context.Context, albumsNode *importer.Object, album picago.Album) (ret error) {
	if album.ID == "" {
		return errors.New("album has no ID")
	}
	albumNode, err := albumsNode.ChildPathObject(album.ID)
	if err != nil {
		return fmt.Errorf("importAlbum: error listing album: %v", err)
	}

	dateMod := schema.RFC3339FromTime(album.Updated)

	// Data reference: https://developers.google.com/picasa-web/docs/2.0/reference
	// TODO(tgulacsi): add more album info
	changes, err := albumNode.SetAttrs2(
		attrPicasaId, album.ID,
		nodeattr.Type, "picasaweb.google.com:album",
		nodeattr.Title, album.Title,
		nodeattr.DatePublished, schema.RFC3339FromTime(album.Published),
		nodeattr.LocationText, album.Location,
		nodeattr.Description, album.Description,
		nodeattr.URL, album.URL,
	)
	if err != nil {
		return fmt.Errorf("error setting album attributes: %v", err)
	}
	if !changes && r.incremental && albumNode.Attr(nodeattr.DateModified) == dateMod {
		return nil
	}
	defer func() {
		// Don't update DateModified on the album node until
		// we've successfully imported all the photos.
		if ret == nil {
			ret = albumNode.SetAttr(nodeattr.DateModified, dateMod)
		}
	}()

	log.Printf("Importing album %v: %v/%v (published %v, updated %v)", album.ID, album.Name, album.Title, album.Published, album.Updated)

	// TODO(bradfitz): GetPhotos does multiple HTTP requests to
	// return a slice of all photos. My "InstantUpload/Auto
	// Backup" album has 6678 photos (and growing) and this
	// currently takes like 40 seconds. Fix.
	photos, err := picago.GetPhotos(ctxutil.Client(ctx), "default", album.ID)
	if err != nil {
		return err
	}

	log.Printf("Importing %d photos from album %q (%s)", len(photos), albumNode.Attr(nodeattr.Title),
		albumNode.PermanodeRef())

	var grp syncutil.Group
	for i := range photos {
		select {
		case <-ctx.Done():
			return ctx.Err()
		default:
		}
		photo := photos[i]
		r.photoGate.Start()
		grp.Go(func() error {
			defer r.photoGate.Done()
			return r.updatePhotoInAlbum(ctx, albumNode, photo)
		})
	}
	return grp.Err()
}
Example #25
0
func (r *run) updatePhotoInAlbum(ctx context.Context, albumNode *importer.Object, photo picago.Photo) (ret error) {
	if photo.ID == "" {
		return errors.New("photo has no ID")
	}

	getMediaBytes := func() (io.ReadCloser, error) {
		log.Printf("Importing media from %v", photo.URL)
		resp, err := ctxutil.Client(ctx).Get(photo.URL)
		if err != nil {
			return nil, fmt.Errorf("importing photo %s: %v", photo.ID, err)
		}
		if resp.StatusCode != http.StatusOK {
			resp.Body.Close()
			return nil, fmt.Errorf("importing photo %s: status code = %d", photo.ID, resp.StatusCode)
		}
		return resp.Body, nil
	}

	var fileRefStr string
	idFilename := photo.ID + "-" + photo.Filename
	photoNode, err := albumNode.ChildPathObjectOrFunc(idFilename, func() (*importer.Object, error) {
		h := blob.NewHash()
		rc, err := getMediaBytes()
		if err != nil {
			return nil, err
		}
		fileRef, err := schema.WriteFileFromReader(r.Host.Target(), photo.Filename, io.TeeReader(rc, h))
		if err != nil {
			return nil, err
		}
		fileRefStr = fileRef.String()
		wholeRef := blob.RefFromHash(h)
		if pn, err := findExistingPermanode(r.Host.Searcher(), wholeRef); err == nil {
			return r.Host.ObjectFromRef(pn)
		}
		return r.Host.NewObject()
	})
	if err != nil {
		return err
	}

	const attrMediaURL = "picasaMediaURL"
	if fileRefStr == "" {
		fileRefStr = photoNode.Attr(nodeattr.CamliContent)
		// Only re-download the source photo if its URL has changed.
		// Empirically this seems to work: cropping a photo in the
		// photos.google.com UI causes its URL to change. And it makes
		// sense, looking at the ugliness of the URLs with all their
		// encoded/signed state.
		if !mediaURLsEqual(photoNode.Attr(attrMediaURL), photo.URL) {
			rc, err := getMediaBytes()
			if err != nil {
				return err
			}
			fileRef, err := schema.WriteFileFromReader(r.Host.Target(), photo.Filename, rc)
			rc.Close()
			if err != nil {
				return err
			}
			fileRefStr = fileRef.String()
		}
	}

	title := strings.TrimSpace(photo.Description)
	if strings.Contains(title, "\n") {
		title = title[:strings.Index(title, "\n")]
	}
	if title == "" && schema.IsInterestingTitle(photo.Filename) {
		title = photo.Filename
	}

	// TODO(tgulacsi): add more attrs (comments ?)
	// for names, see http://schema.org/ImageObject and http://schema.org/CreativeWork
	attrs := []string{
		nodeattr.CamliContent, fileRefStr,
		attrPicasaId, photo.ID,
		nodeattr.Title, title,
		nodeattr.Description, photo.Description,
		nodeattr.LocationText, photo.Location,
		nodeattr.DateModified, schema.RFC3339FromTime(photo.Updated),
		nodeattr.DatePublished, schema.RFC3339FromTime(photo.Published),
		nodeattr.URL, photo.PageURL,
	}
	if photo.Latitude != 0 || photo.Longitude != 0 {
		attrs = append(attrs,
			nodeattr.Latitude, fmt.Sprintf("%f", photo.Latitude),
			nodeattr.Longitude, fmt.Sprintf("%f", photo.Longitude),
		)
	}
	if err := photoNode.SetAttrs(attrs...); err != nil {
		return err
	}
	if err := photoNode.SetAttrValues("tag", photo.Keywords); err != nil {
		return err
	}
	if photo.Position > 0 {
		if err := albumNode.SetAttr(
			nodeattr.CamliPathOrderColon+strconv.Itoa(photo.Position-1),
			photoNode.PermanodeRef().String()); err != nil {
			return err
		}
	}

	// Do this last, after we're sure the "camliContent" attribute
	// has been saved successfully, because this is the one that
	// causes us to do it again in the future or not.
	if err := photoNode.SetAttrs(attrMediaURL, photo.URL); err != nil {
		return err
	}
	return nil
}
Example #26
0
func (im *imp) IsAccountReady(acctNode *importer.Object) (ok bool, err error) {
	if acctNode.Attr(acctAttrFeedURL) != "" {
		return true, nil
	}
	return false, nil
}
Example #27
0
// TODO(aa):
// * Parallelize: http://golang.org/doc/effective_go.html#concurrency
// * Do more than one "page" worth of results
// * Report progress and errors back through host interface
// * All the rest of the metadata (see photoMeta)
// * Conflicts: For all metadata changes, prefer any non-imported claims
// * Test!
func (r *run) importPhoto(parent *importer.Object, photo *photosSearchItem) error {
	filename := fmt.Sprintf("%s.%s", photo.Id, photo.OriginalFormat)
	photoNode, err := parent.ChildPathObject(filename)
	if err != nil {
		return err
	}

	// https://www.flickr.com/services/api/misc.dates.html
	dateTaken, err := time.ParseInLocation("2006-01-02 15:04:05", photo.DateTaken, schema.UnknownLocation)
	if err != nil {
		// default to the published date otherwise
		log.Printf("Flickr importer: problem with date taken of photo %v, defaulting to published date instead.", photo.Id)
		seconds, err := strconv.ParseInt(photo.DateUpload, 10, 64)
		if err != nil {
			return fmt.Errorf("could not parse date upload time %q for image %v: %v", photo.DateUpload, photo.Id, err)
		}
		dateTaken = time.Unix(seconds, 0)
	}

	attrs := []string{
		attrFlickrId, photo.Id,
		nodeattr.DateCreated, schema.RFC3339FromTime(dateTaken),
		nodeattr.Description, photo.Description.Content,
	}
	if schema.IsInterestingTitle(photo.Title) {
		attrs = append(attrs, nodeattr.Title, photo.Title)
	}
	// Import all the metadata. SetAttrs() is a no-op if the value hasn't changed, so there's no cost to doing these on every run.
	// And this way if we add more things to import, they will get picked up.
	if err := photoNode.SetAttrs(attrs...); err != nil {
		return err
	}

	// Import the photo itself. Since it is expensive to fetch the image, we store its lastupdate and only refetch if it might have changed.
	// lastupdate is a Unix timestamp according to https://www.flickr.com/services/api/flickr.photos.getInfo.html
	seconds, err := strconv.ParseInt(photo.LastUpdate, 10, 64)
	if err != nil {
		return fmt.Errorf("could not parse lastupdate time for image %v: %v", photo.Id, err)
	}
	lastUpdate := time.Unix(seconds, 0)
	if lastUpdateString := photoNode.Attr(nodeattr.DateModified); lastUpdateString != "" {
		oldLastUpdate, err := time.Parse(time.RFC3339, lastUpdateString)
		if err != nil {
			return fmt.Errorf("could not parse last stored update time for image %v: %v", photo.Id, err)
		}
		if lastUpdate.Equal(oldLastUpdate) {
			if err := r.updatePrimaryPhoto(photoNode); err != nil {
				return err
			}
			return nil
		}
	}
	form := url.Values{}
	form.Set("user_id", r.userID)
	res, err := r.fetch(photo.URL, form)
	if err != nil {
		log.Printf("Flickr importer: Could not fetch %s: %s", photo.URL, err)
		return err
	}
	defer res.Body.Close()

	fileRef, err := schema.WriteFileFromReader(r.Host.Target(), filename, res.Body)
	if err != nil {
		return err
	}
	if err := photoNode.SetAttr(nodeattr.CamliContent, fileRef.String()); err != nil {
		return err
	}
	if err := r.updatePrimaryPhoto(photoNode); err != nil {
		return err
	}
	// Write lastupdate last, so that if any of the preceding fails, we will try again next time.
	if err := photoNode.SetAttr(nodeattr.DateModified, schema.RFC3339FromTime(lastUpdate)); err != nil {
		return err
	}

	return nil
}
Example #28
0
func (im *imp) SummarizeAccount(acct *importer.Object) string {
	ok, err := im.IsAccountReady(acct)
	if err != nil {
		return "Not configured; error = " + err.Error()
	}
	if !ok {
		return "Not configured"
	}
	if acct.Attr(acctAttrUserFirst) == "" && acct.Attr(acctAttrUserLast) == "" {
		return fmt.Sprintf("@%s", acct.Attr(acctAttrScreenName))
	}
	return fmt.Sprintf("@%s (%s %s)", acct.Attr(acctAttrScreenName),
		acct.Attr(acctAttrUserFirst), acct.Attr(acctAttrUserLast))
}
Example #29
0
func (im *imp) IsAccountReady(acctNode *importer.Object) (ok bool, err error) {
	if acctNode.Attr(acctAttrUserID) != "" && acctNode.Attr(acctAttrAccessToken) != "" {
		return true, nil
	}
	return false, nil
}
Example #30
0
func (imp) IsAccountReady(acctNode *importer.Object) (ok bool, err error) {
	return acctNode.Attr(importer.AcctAttrUserName) != "" && acctNode.Attr(importer.AcctAttrAccessToken) != "", nil
}