// DeleteAll deletes across all roots
// DeleteAll deletes by kind alone.
func (fs *dsFileSys) DeleteAll() (string, error) {
msg := ""
{
q := datastore.NewQuery(tfil).KeysOnly()
var files []DsFile
keys, err := q.GetAll(fs.Ctx(), &files)
if err != nil {
msg += "could not get file keys\n"
return msg, err
}
if len(keys) >= 500 {
msg += "limited to 500 files. REPEAT operation.\n"
keys = keys[:500]
}
err = datastore.DeleteMulti(fs.Ctx(), keys)
if err != nil {
msg += "error deleting files\n"
return msg, err
}
msg += spf("%v files deleted\n", len(keys))
}
{
q := datastore.NewQuery(tdir).KeysOnly()
var dirs []DsDir
keys, err := q.GetAll(fs.Ctx(), &dirs)
if err != nil {
msg += "could not get dir keys\n"
return msg, err
}
if len(keys) >= 500 {
msg += "limited to 500 directories. REPEAT operation.\n"
keys = keys[:500]
}
err = datastore.DeleteMulti(fs.Ctx(), keys)
if err != nil {
msg += "error deleting directories\n"
return msg, err
}
msg += spf("%v directories deleted\n", len(keys))
}
err := memcache.Flush(fs.Ctx())
if err != nil {
msg += "error flushing memcache\n"
return msg, err
} else {
msg += "memcache flushed \n"
}
return msg, nil
}
func purgeuserHandler(w http.ResponseWriter, r *http.Request) {
ctx := req2ctx(r)
cdb := complaintdb.NewDB(ctx)
email := r.FormValue("email")
str := fmt.Sprintf("(purgeuser for %s)\n", email)
q := cdb.QueryAllByEmailAddress(email).KeysOnly()
keys, err := q.GetAll(cdb.Ctx(), nil)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
str += fmt.Sprintf("purge: %d complaints found\n", len(keys))
if r.FormValue("forrealz") == "1" {
maxRm := 400
for len(keys) > maxRm {
if err := datastore.DeleteMulti(cdb.Ctx(), keys[0:maxRm-1]); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
keys = keys[maxRm:]
}
str += "all deleted :O\n"
}
w.Header().Set("Content-Type", "text/plain")
w.Write([]byte(fmt.Sprintf("OK, purge\n%s", str)))
}
// CleanupDatastore is to remove all data in datastore
func CleanupDatastore(ctx context.Context, namespaces ...string) error {
var dummy []interface{}
logger := wcg.NewLogger(nil)
logger.Debugf("[Fixture] --------- CleanupDatastore ---------")
namespaceList := append(namespaces, "")
for _, ns := range namespaceList {
logger.Debugf("[Fixture] Cleanup: ns=%q", ns)
var _ctx = ctx
if ns != "" {
_ctx, _ = appengine.Namespace(_ctx, ns)
}
err := wcg.RetryUntil(func() error {
var keys []*datastore.Key
var err error
if keys, err = datastore.NewQuery("").KeysOnly().GetAll(_ctx, dummy); err != nil {
return err
}
if err := datastore.DeleteMulti(_ctx, keys); err != nil {
return err
}
count, _ := datastore.NewQuery("").KeysOnly().Count(_ctx)
if count == 0 {
return nil
}
return fmt.Errorf("Still have %d keys.", count)
}, 10*time.Second, 100*time.Millisecond)
if err != nil {
return err
}
}
return nil
}
func cleanup(s *testerator.Setup) error {
t := datastore.NewQuery("__kind__").KeysOnly().Run(s.Context)
kinds := make([]string, 0)
for {
key, err := t.Next(nil)
if err == datastore.Done {
break
}
if err != nil {
return err
}
kinds = append(kinds, key.StringID())
}
for _, kind := range kinds {
q := datastore.NewQuery(kind).KeysOnly()
keys, err := q.GetAll(s.Context, nil)
if err != nil {
return err
}
err = datastore.DeleteMulti(s.Context, keys)
if err != nil {
return err
}
}
return nil
}
func (d rdsImpl) DeleteMulti(ks []*ds.Key, cb ds.DeleteMultiCB) error {
keys, err := dsMF2R(d.aeCtx, ks)
if err == nil {
err = datastore.DeleteMulti(d.aeCtx, keys)
}
return idxCallbacker(err, len(ks), func(_ int, err error) {
cb(err)
})
}
func (cdb ComplaintDB) DeletAllGlobalStats() error {
fgs := []FrozenGlobalStats{}
q := datastore.NewQuery(kGlobalStatsKind).KeysOnly()
if keys, err := q.GetAll(cdb.Ctx(), &fgs); err != nil {
return err
} else {
return datastore.DeleteMulti(cdb.Ctx(), keys)
}
}
func (a *GaeDatastoreAccessor) deleteAllIdioms(c context.Context) error {
keys, err := datastore.NewQuery("Idiom").KeysOnly().GetAll(c, nil)
if err != nil {
return err
}
err = a.unindexAll(c)
if err != nil {
return err
}
return datastore.DeleteMulti(c, keys)
}
// clearEventData deletes all EventData entities and flushes cache.
func clearEventData(c context.Context) error {
if err := cache.flush(c); err != nil {
return err
}
q := datastore.NewQuery(kindEventData).
Ancestor(eventDataParent(c)).
KeysOnly()
keys, err := q.GetAll(c, nil)
if err != nil {
return fmt.Errorf("clearEventData: %v", err)
}
return datastore.DeleteMulti(c, keys)
}
func deleteSavedPosts(c context.Context, key string) error {
// flush db
q := datastore.NewQuery(key).Ancestor(getParentKey(c, key)).KeysOnly()
keys, err := q.GetAll(c, nil)
if err != nil {
return err
}
if err = datastore.DeleteMulti(c, keys); err != nil {
return err
}
// flush cache
return memcache.Delete(c, key)
}
func example() {
// [START batch]
// A batch put.
_, err = datastore.PutMulti(ctx, []*datastore.Key{k1, k2, k3}, []interface{}{e1, e2, e3})
// A batch get.
var entities = make([]*T, 3)
err = datastore.GetMulti(ctx, []*datastore.Key{k1, k2, k3}, entities)
// A batch delete.
err = datastore.DeleteMulti(ctx, []*datastore.Key{k1, k2, k3})
// [END batch]
_ = err
}
func (this *MainController) Delete() {
err := datastore.RunInTransaction(this.AppEngineCtx, func(c context.Context) error {
ks, err := datastore.NewQuery("Todo").KeysOnly().Ancestor(models.DefaultTodoList(c)).Filter("Done=", true).GetAll(c, nil)
if err != nil {
return err
}
return datastore.DeleteMulti(c, ks)
}, nil)
if err == nil {
this.Data["json"] = nil
} else {
this.Data["json"] = err
}
}
// ClearNamespace deletes every entry in a namespace
func ClearNamespace(con *Context, namespace string) error {
namespace = strings.ToLower(namespace)
q := datastore.NewQuery(itemTable).
Filter("Namespace =", namespace).
KeysOnly()
k, err := q.GetAll(con.C, nil)
if err != nil {
return err
}
clearCache(con, namespace)
con.Log.Infof("Going to delete %v items in the namespace %v", len(k), namespace)
return datastore.DeleteMulti(con.C, k)
}
func (cdb ComplaintDB) DeleteComplaints(keyStrings []string, ownerEmail string) error {
keys := []*datastore.Key{}
for _, s := range keyStrings {
k, err := datastore.DecodeKey(s)
if err != nil {
return err
}
if k.Parent() == nil {
return fmt.Errorf("key <%v> had no parent", k)
}
if k.Parent().StringID() != ownerEmail {
return fmt.Errorf("key <%v> owned by %s, not %s", k, k.Parent().StringID(), ownerEmail)
}
keys = append(keys, k)
}
return datastore.DeleteMulti(cdb.Ctx(), keys)
}
func adminMarkPaid(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
action := r.FormValue("action")
keys := make([]*datastore.Key, 0, len(r.Form["pay"]))
for _, s := range r.Form["pay"] {
k, err := datastore.DecodeKey(s)
if err != nil {
panic(err)
}
keys = append(keys, k)
}
if action == "Mark Paid" {
tasks := make([]LoggedTask, len(keys))
err := datastore.GetMulti(c, keys, tasks)
if err != nil {
panic(err)
}
now := time.Now().UTC()
for i := range tasks {
tasks[i].Paid = true
tasks[i].PaidTime = now
}
_, err = datastore.PutMulti(c, keys, tasks)
if err != nil {
panic(err)
}
} else if action == "Delete" {
err := datastore.DeleteMulti(c, keys)
if err != nil {
panic(err)
}
} else {
panic("Unhandled action: " + action)
}
http.Redirect(w, r, "/admin/", 303)
}
func DropAllFromRecord(res http.ResponseWriter, req *http.Request, params httprouter.Params) {
res.Header().Set("Access-Control-Allow-Origin", "*") // Allow for outside access.
// Get Parameters
sinfo, _ := GetStorageInfo(req)
if sinfo.Domain == "" {
ServeJsonOfStruct(res, JsonOptions{
Status: "Failure",
Reason: "Missing Parameter Domain",
Code: http.StatusNotAcceptable,
}, nil)
return
}
q := datastore.NewQuery(RecordsTable).Filter("Domain =", sinfo.Domain).KeysOnly()
ctx := appengine.NewContext(req) // Make Context
recordKeys, qErr := q.GetAll(ctx, nil)
if qErr != nil {
ServeJsonOfStruct(res, JsonOptions{
Status: "Failure",
Reason: qErr.Error(),
Code: 500,
}, nil)
return
}
delErr := datastore.DeleteMulti(ctx, recordKeys)
if delErr != nil {
ServeJsonOfStruct(res, JsonOptions{
Status: "Failure",
Reason: delErr.Error(),
Code: 500,
}, nil)
return
}
ServeJsonOfStruct(res, JsonOptions{
Status: "Success",
}, nil)
}
// CollectTokens runs a query against Datastore to find expired
// Tokens and deletes them.
func CollectTokens(ctx context.Context, w http.ResponseWriter, r *http.Request) (int, error) {
keys, err := model.NewQueryForToken().
Filter("Expiry <", time.Now()).
KeysOnly().
GetAll(ctx, nil)
if err != nil {
log.Warningf(ctx, "garbage collecting tokens failed: %s", err.Error())
return http.StatusInternalServerError, err
}
err = datastore.DeleteMulti(ctx, keys)
if err != nil {
log.Warningf(ctx, "garbage collecting tokens failed: %s", err.Error())
return http.StatusInternalServerError, err
}
return http.StatusOK, nil
}
func deletepost(rw http.ResponseWriter, req *http.Request) {
c := appengine.NewContext(req)
u := user.Current(c)
s := req.FormValue("encoded_key")
k, err := datastore.DecodeKey(s)
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
return
}
mypost := Post{}
err = datastore.Get(c, k, &mypost)
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
return
}
if mypost.Author == u.String() {
if mypost.OP {
q := datastore.NewQuery("post").Ancestor(k).KeysOnly()
keys, err := q.GetAll(c, struct{}{})
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
return
}
err = datastore.DeleteMulti(c, keys)
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
return
}
} else {
err := datastore.Delete(c, k)
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
return
}
}
// http.Redirect(rw, req, "/view", http.StatusTemporaryRedirect)
// } else {
http.Redirect(rw, req, "/", http.StatusTemporaryRedirect)
}
}
// DeleteMulti is a batch version of Delete.
func (g *Goon) DeleteMulti(keys []*datastore.Key) error {
if len(keys) == 0 {
return nil
// not an error, and it was "successful", so return nil
}
memkeys := make([]string, len(keys))
g.cacheLock.Lock()
for i, k := range keys {
mk := memkey(k)
memkeys[i] = mk
if g.inTransaction {
delete(g.toSet, mk)
g.toDelete[mk] = true
} else {
delete(g.cache, mk)
}
}
g.cacheLock.Unlock()
// Memcache needs to be updated after the datastore to prevent a common race condition,
// where a concurrent request will fetch the not-yet-updated data from the datastore
// and populate memcache with it.
if g.inTransaction {
for _, mk := range memkeys {
g.toDeleteMC[mk] = true
}
} else {
defer memcache.DeleteMulti(g.Context, memkeys)
}
multiErr, any := make(appengine.MultiError, len(keys)), false
goroutines := (len(keys)-1)/deleteMultiLimit + 1
var wg sync.WaitGroup
wg.Add(goroutines)
for i := 0; i < goroutines; i++ {
go func(i int) {
defer wg.Done()
lo := i * deleteMultiLimit
hi := (i + 1) * deleteMultiLimit
if hi > len(keys) {
hi = len(keys)
}
dmerr := datastore.DeleteMulti(g.Context, keys[lo:hi])
if dmerr != nil {
any = true // this flag tells DeleteMulti to return multiErr later
merr, ok := dmerr.(appengine.MultiError)
if !ok {
g.error(dmerr)
for j := lo; j < hi; j++ {
multiErr[j] = dmerr
}
return
}
copy(multiErr[lo:hi], merr)
}
}(i)
}
wg.Wait()
if any {
return realError(multiErr)
}
return nil
}
func handleRemove(w http.ResponseWriter, r *http.Request, apiKey APIKey) *appError {
if r.Method != "DELETE" {
return &appError{nil, fmt.Sprintf("Invalid request method: %s", r.Method), 401}
}
c := appengine.NewContext(r)
strChatID := r.FormValue("chatID")
rmPath := r.FormValue("path")
if rmPath == "" {
return &appError{nil, "Missing path.", 401}
}
var fbChatID int64 = -1
var chatKey *datastore.Key
var err error
if strChatID != "" {
fbChatID, err = strconv.ParseInt(strChatID, 10, 64)
if err != nil {
return &appError{nil, "Bad chat ID", 400}
}
chatKeys, err := datastore.NewQuery("Chat").Filter("FacebookChatID =", fbChatID).
KeysOnly().GetAll(c, nil)
if err != nil {
return &appError{err, "Datastore error: " + err.Error(), 400}
} else if len(chatKeys) == 0 {
return &appError{nil, "Bad chat ID", 400}
}
chatKey = chatKeys[0]
} else {
chatKey = nil
}
deleted := make([]Link, 0)
keysToRemove, err := datastore.NewQuery("Link").
Filter("Path =", rmPath).Filter("ChatKey =", chatKey).GetAll(c, &deleted)
if len(keysToRemove) != 0 {
newKeys := make([]*datastore.Key, len(keysToRemove))
for i := range keysToRemove {
newKeys[i] = datastore.NewIncompleteKey(c, "DeletedLink", nil)
}
err = datastore.RunInTransaction(c, func(tc context.Context) (err error) {
_, err = datastore.PutMulti(c, newKeys, deleted)
if err != nil {
return
}
err = datastore.DeleteMulti(c, keysToRemove)
return
}, nil)
}
var resp RemoveResponse
if err != nil {
resp = RemoveResponse{
false,
0,
err.Error(),
}
} else {
resp = RemoveResponse{
true,
len(keysToRemove),
"",
}
}
respJSON, _ := json.Marshal(resp)
w.Write(respJSON)
return nil
}
// DeleteMulti wraps datastore.DeleteMulti
func (d *Driver) DeleteMulti(key []*datastore.Key) error {
d.logOps(opDelete, len(key), "DeleteMulti")
return datastore.DeleteMulti(d.ctx, key)
}
func (d rdsImpl) DeleteMulti(ks []ds.Key, cb ds.DeleteMultiCB) error {
err := datastore.DeleteMulti(d, dsMF2R(ks))
return idxCallbacker(err, len(ks), func(_ int, err error) {
cb(err)
})
}
// ReplaceAuthDB updates database with given AuthDBSnapshot if it is new.
// The first return value indicates database was updated,
// and the second one has the latest AuthReplicationState.
func ReplaceAuthDB(c context.Context, newData AuthDBSnapshot) (bool, *AuthReplicationState, error) {
var stat *AuthReplicationState
updated := false
err := datastore.RunInTransaction(c, func(c context.Context) error {
curstat := &AuthReplicationState{}
if err := GetReplicationState(c, curstat); err != nil {
return err
}
if newData.ReplicationState.PrimaryID != curstat.PrimaryID {
return fmt.Errorf("primary id mismatch. incoming=%q; want=%q", newData.ReplicationState.PrimaryID, curstat.PrimaryID)
}
// database is already up-to-date.
if curstat.AuthDBRev >= newData.ReplicationState.AuthDBRev {
stat = curstat
return nil
}
dbsnap := &AuthDBSnapshot{}
if err := currentAuthDBSnapshot(c, dbsnap); err != nil {
return err
}
var newEntities []entity
var delKeys []*datastore.Key
// Going to update database.
if !reflect.DeepEqual(newData.GlobalConfig, dbsnap.GlobalConfig) {
newEntities = append(newEntities, newData.GlobalConfig)
}
newGrps, delGrKeys := groupsDiff(dbsnap.Groups, newData.Groups)
newEntities = append(newEntities, newGrps...)
delKeys = append(delKeys, delGrKeys...)
newWls, delWlKeys := whitelistsDiff(dbsnap.IPWhitelists, newData.IPWhitelists)
newEntities = append(newEntities, newWls...)
delKeys = append(delKeys, delWlKeys...)
if !reflect.DeepEqual(newData.IPWhitelistAssignments, dbsnap.IPWhitelistAssignments) {
newEntities = append(newEntities, newData.IPWhitelistAssignments)
}
curstat.AuthDBRev = newData.ReplicationState.AuthDBRev
curstat.ModifiedTimestamp = newData.ReplicationState.ModifiedTimestamp
var wg sync.WaitGroup
ch := make(chan error, 3)
wg.Add(3)
go func() {
defer wg.Done()
if _, err := datastore.Put(c, ReplicationStateKey(c), curstat); err != nil {
ch <- err
return
}
ch <- nil
}()
go func() {
defer wg.Done()
keys := make([]*datastore.Key, 0, len(newEntities))
for _, n := range newEntities {
keys = append(keys, n.key())
}
if _, err := datastore.PutMulti(c, keys, newEntities); err != nil {
ch <- err
return
}
ch <- nil
}()
go func() {
defer wg.Done()
if err := datastore.DeleteMulti(c, delKeys); err != nil {
ch <- err
return
}
ch <- nil
}()
go func() {
wg.Wait()
close(ch)
}()
for err := range ch {
if err != nil {
return err
}
}
stat = curstat
updated = true
return nil
}, &datastore.TransactionOptions{
XG: true,
})
if err != nil {
return false, nil, err
}
return updated, stat, nil
}
