예제 #1
0
func toTrailForAuthorization(authorization *models.ExternalServiceAuthorization) stream.MapFn {
	return func(data stream.T) stream.T {
		trailItem := toTrail(data)
		trail := trailItem.(*models.Trail)
		trail.SetParentKey(authorization.ParentKey())

		return trail
	}
}
예제 #2
0
func DropboxDelta(req *http.Request, ds *appx.Datastore, authorization *models.ExternalServiceAuthorization, existingItem stream.PredicateFn) {
	dropboxClient := dropboxClient(newappengine.NewContext(req), authorization.AccessToken)
	builder := DropboxDeltaProducerBuilder{Client: dropboxClient, CurrentCursor: authorization.LastCursor}

	deltaStream, debugStream := rivers.
		From(builder.Build()).
		Drop(nonMediaFiles).
		Map(toTrailForAuthorization(authorization)).
		Drop(existingItem).
		Split()

	println("About to split data")

	err := deltaStream.BatchBy(&appx.DatastoreBatch{Size: 500}).
		Each(saveBatch(ds)).
		Drain()

	println(fmt.Sprintf("Error while batch saving the data: %v", err))

	total, errTotal := debugStream.Reduce(0, func(sum, item stream.T) stream.T {
		return sum.(int) + 1
	}).CollectFirst()

	if errTotal != nil {
		println(fmt.Sprintf("########## Guaging error %v", err))
		return
	}
	println(fmt.Sprintf("Total entries: %v", total.(int)))

	authorization.LastCursor = builder.CurrentCursor
	ds.Save(authorization)

	if err != nil {
		println(fmt.Sprintf("########## Delta error %v", err))
	}

}