func toTrailForAuthorization(authorization *models.ExternalServiceAuthorization) stream.MapFn {
return func(data stream.T) stream.T {
trailItem := toTrail(data)
trail := trailItem.(*models.Trail)
trail.SetParentKey(authorization.ParentKey())
return trail
}
}
func DropboxDelta(req *http.Request, ds *appx.Datastore, authorization *models.ExternalServiceAuthorization, existingItem stream.PredicateFn) {
dropboxClient := dropboxClient(newappengine.NewContext(req), authorization.AccessToken)
builder := DropboxDeltaProducerBuilder{Client: dropboxClient, CurrentCursor: authorization.LastCursor}
deltaStream, debugStream := rivers.
From(builder.Build()).
Drop(nonMediaFiles).
Map(toTrailForAuthorization(authorization)).
Drop(existingItem).
Split()
println("About to split data")
err := deltaStream.BatchBy(&appx.DatastoreBatch{Size: 500}).
Each(saveBatch(ds)).
Drain()
println(fmt.Sprintf("Error while batch saving the data: %v", err))
total, errTotal := debugStream.Reduce(0, func(sum, item stream.T) stream.T {
return sum.(int) + 1
}).CollectFirst()
if errTotal != nil {
println(fmt.Sprintf("########## Guaging error %v", err))
return
}
println(fmt.Sprintf("Total entries: %v", total.(int)))
authorization.LastCursor = builder.CurrentCursor
ds.Save(authorization)
if err != nil {
println(fmt.Sprintf("########## Delta error %v", err))
}
}