Esempio n. 1
0
// DeleteAll deletes across all roots
// DeleteAll deletes by kind alone.
func (fs *dsFileSys) DeleteAll() (string, error) {

	msg := ""
	{
		q := datastore.NewQuery(tfil).KeysOnly()
		var files []DsFile
		keys, err := q.GetAll(fs.Ctx(), &files)
		if err != nil {
			msg += "could not get file keys\n"
			return msg, err
		}
		if len(keys) >= 500 {
			msg += "limited to 500 files. REPEAT operation.\n"
			keys = keys[:500]
		}

		err = datastore.DeleteMulti(fs.Ctx(), keys)
		if err != nil {
			msg += "error deleting files\n"
			return msg, err
		}

		msg += spf("%v files deleted\n", len(keys))

	}

	{
		q := datastore.NewQuery(tdir).KeysOnly()
		var dirs []DsDir
		keys, err := q.GetAll(fs.Ctx(), &dirs)
		if err != nil {
			msg += "could not get dir keys\n"
			return msg, err
		}
		if len(keys) >= 500 {
			msg += "limited to 500 directories. REPEAT operation.\n"
			keys = keys[:500]
		}

		err = datastore.DeleteMulti(fs.Ctx(), keys)
		if err != nil {
			msg += "error deleting directories\n"
			return msg, err
		}

		msg += spf("%v directories deleted\n", len(keys))
	}

	err := memcache.Flush(fs.Ctx())
	if err != nil {
		msg += "error flushing memcache\n"
		return msg, err
	} else {
		msg += "memcache flushed \n"
	}

	return msg, nil
}
Esempio n. 2
0
func purgeuserHandler(w http.ResponseWriter, r *http.Request) {
	ctx := req2ctx(r)
	cdb := complaintdb.NewDB(ctx)
	email := r.FormValue("email")

	str := fmt.Sprintf("(purgeuser for %s)\n", email)

	q := cdb.QueryAllByEmailAddress(email).KeysOnly()
	keys, err := q.GetAll(cdb.Ctx(), nil)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	str += fmt.Sprintf("purge: %d complaints found\n", len(keys))

	if r.FormValue("forrealz") == "1" {
		maxRm := 400
		for len(keys) > maxRm {
			if err := datastore.DeleteMulti(cdb.Ctx(), keys[0:maxRm-1]); err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			keys = keys[maxRm:]
		}
		str += "all deleted :O\n"
	}

	w.Header().Set("Content-Type", "text/plain")
	w.Write([]byte(fmt.Sprintf("OK, purge\n%s", str)))
}
Esempio n. 3
0
// CleanupDatastore is to remove all data in datastore
func CleanupDatastore(ctx context.Context, namespaces ...string) error {
	var dummy []interface{}
	logger := wcg.NewLogger(nil)
	logger.Debugf("[Fixture] --------- CleanupDatastore ---------")
	namespaceList := append(namespaces, "")
	for _, ns := range namespaceList {
		logger.Debugf("[Fixture] Cleanup: ns=%q", ns)
		var _ctx = ctx
		if ns != "" {
			_ctx, _ = appengine.Namespace(_ctx, ns)
		}
		err := wcg.RetryUntil(func() error {
			var keys []*datastore.Key
			var err error
			if keys, err = datastore.NewQuery("").KeysOnly().GetAll(_ctx, dummy); err != nil {
				return err
			}
			if err := datastore.DeleteMulti(_ctx, keys); err != nil {
				return err
			}
			count, _ := datastore.NewQuery("").KeysOnly().Count(_ctx)
			if count == 0 {
				return nil
			}
			return fmt.Errorf("Still have %d keys.", count)
		}, 10*time.Second, 100*time.Millisecond)
		if err != nil {
			return err
		}
	}
	return nil
}
Esempio n. 4
0
func cleanup(s *testerator.Setup) error {
	t := datastore.NewQuery("__kind__").KeysOnly().Run(s.Context)
	kinds := make([]string, 0)
	for {
		key, err := t.Next(nil)
		if err == datastore.Done {
			break
		}
		if err != nil {
			return err
		}
		kinds = append(kinds, key.StringID())
	}

	for _, kind := range kinds {
		q := datastore.NewQuery(kind).KeysOnly()
		keys, err := q.GetAll(s.Context, nil)
		if err != nil {
			return err
		}
		err = datastore.DeleteMulti(s.Context, keys)
		if err != nil {
			return err
		}
	}

	return nil
}
Esempio n. 5
0
func (d rdsImpl) DeleteMulti(ks []*ds.Key, cb ds.DeleteMultiCB) error {
	keys, err := dsMF2R(d.aeCtx, ks)
	if err == nil {
		err = datastore.DeleteMulti(d.aeCtx, keys)
	}
	return idxCallbacker(err, len(ks), func(_ int, err error) {
		cb(err)
	})
}
Esempio n. 6
0
func (cdb ComplaintDB) DeletAllGlobalStats() error {
	fgs := []FrozenGlobalStats{}
	q := datastore.NewQuery(kGlobalStatsKind).KeysOnly()

	if keys, err := q.GetAll(cdb.Ctx(), &fgs); err != nil {
		return err
	} else {
		return datastore.DeleteMulti(cdb.Ctx(), keys)
	}
}
func (a *GaeDatastoreAccessor) deleteAllIdioms(c context.Context) error {
	keys, err := datastore.NewQuery("Idiom").KeysOnly().GetAll(c, nil)
	if err != nil {
		return err
	}

	err = a.unindexAll(c)
	if err != nil {
		return err
	}

	return datastore.DeleteMulti(c, keys)
}
Esempio n. 8
0
// clearEventData deletes all EventData entities and flushes cache.
func clearEventData(c context.Context) error {
	if err := cache.flush(c); err != nil {
		return err
	}
	q := datastore.NewQuery(kindEventData).
		Ancestor(eventDataParent(c)).
		KeysOnly()
	keys, err := q.GetAll(c, nil)
	if err != nil {
		return fmt.Errorf("clearEventData: %v", err)
	}
	return datastore.DeleteMulti(c, keys)
}
Esempio n. 9
0
func deleteSavedPosts(c context.Context, key string) error {
	// flush db
	q := datastore.NewQuery(key).Ancestor(getParentKey(c, key)).KeysOnly()
	keys, err := q.GetAll(c, nil)
	if err != nil {
		return err
	}
	if err = datastore.DeleteMulti(c, keys); err != nil {
		return err
	}
	// flush cache
	return memcache.Delete(c, key)
}
func example() {
	// [START batch]
	// A batch put.
	_, err = datastore.PutMulti(ctx, []*datastore.Key{k1, k2, k3}, []interface{}{e1, e2, e3})

	// A batch get.
	var entities = make([]*T, 3)
	err = datastore.GetMulti(ctx, []*datastore.Key{k1, k2, k3}, entities)

	// A batch delete.
	err = datastore.DeleteMulti(ctx, []*datastore.Key{k1, k2, k3})
	// [END batch]
	_ = err
}
Esempio n. 11
0
func (this *MainController) Delete() {
	err := datastore.RunInTransaction(this.AppEngineCtx, func(c context.Context) error {
		ks, err := datastore.NewQuery("Todo").KeysOnly().Ancestor(models.DefaultTodoList(c)).Filter("Done=", true).GetAll(c, nil)
		if err != nil {
			return err
		}
		return datastore.DeleteMulti(c, ks)
	}, nil)

	if err == nil {
		this.Data["json"] = nil
	} else {
		this.Data["json"] = err
	}
}
Esempio n. 12
0
// ClearNamespace deletes every entry in a namespace
func ClearNamespace(con *Context, namespace string) error {
	namespace = strings.ToLower(namespace)
	q := datastore.NewQuery(itemTable).
		Filter("Namespace =", namespace).
		KeysOnly()

	k, err := q.GetAll(con.C, nil)
	if err != nil {
		return err
	}

	clearCache(con, namespace)

	con.Log.Infof("Going to delete %v items in the namespace %v", len(k), namespace)

	return datastore.DeleteMulti(con.C, k)
}
Esempio n. 13
0
func (cdb ComplaintDB) DeleteComplaints(keyStrings []string, ownerEmail string) error {
	keys := []*datastore.Key{}
	for _, s := range keyStrings {
		k, err := datastore.DecodeKey(s)
		if err != nil {
			return err
		}

		if k.Parent() == nil {
			return fmt.Errorf("key <%v> had no parent", k)
		}
		if k.Parent().StringID() != ownerEmail {
			return fmt.Errorf("key <%v> owned by %s, not %s", k, k.Parent().StringID(), ownerEmail)
		}
		keys = append(keys, k)
	}
	return datastore.DeleteMulti(cdb.Ctx(), keys)
}
Esempio n. 14
0
func adminMarkPaid(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	action := r.FormValue("action")

	keys := make([]*datastore.Key, 0, len(r.Form["pay"]))
	for _, s := range r.Form["pay"] {
		k, err := datastore.DecodeKey(s)
		if err != nil {
			panic(err)
		}
		keys = append(keys, k)
	}

	if action == "Mark Paid" {
		tasks := make([]LoggedTask, len(keys))

		err := datastore.GetMulti(c, keys, tasks)
		if err != nil {
			panic(err)
		}

		now := time.Now().UTC()
		for i := range tasks {
			tasks[i].Paid = true
			tasks[i].PaidTime = now
		}

		_, err = datastore.PutMulti(c, keys, tasks)
		if err != nil {
			panic(err)
		}
	} else if action == "Delete" {
		err := datastore.DeleteMulti(c, keys)
		if err != nil {
			panic(err)
		}
	} else {
		panic("Unhandled action: " + action)
	}

	http.Redirect(w, r, "/admin/", 303)
}
Esempio n. 15
0
func DropAllFromRecord(res http.ResponseWriter, req *http.Request, params httprouter.Params) {
	res.Header().Set("Access-Control-Allow-Origin", "*") // Allow for outside access.

	// Get Parameters
	sinfo, _ := GetStorageInfo(req)
	if sinfo.Domain == "" {
		ServeJsonOfStruct(res, JsonOptions{
			Status: "Failure",
			Reason: "Missing Parameter Domain",
			Code:   http.StatusNotAcceptable,
		}, nil)
		return
	}

	q := datastore.NewQuery(RecordsTable).Filter("Domain =", sinfo.Domain).KeysOnly()

	ctx := appengine.NewContext(req) // Make Context

	recordKeys, qErr := q.GetAll(ctx, nil)
	if qErr != nil {
		ServeJsonOfStruct(res, JsonOptions{
			Status: "Failure",
			Reason: qErr.Error(),
			Code:   500,
		}, nil)
		return
	}

	delErr := datastore.DeleteMulti(ctx, recordKeys)
	if delErr != nil {
		ServeJsonOfStruct(res, JsonOptions{
			Status: "Failure",
			Reason: delErr.Error(),
			Code:   500,
		}, nil)
		return
	}

	ServeJsonOfStruct(res, JsonOptions{
		Status: "Success",
	}, nil)
}
Esempio n. 16
0
// CollectTokens runs a query against Datastore to find expired
// Tokens and deletes them.
func CollectTokens(ctx context.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	keys, err := model.NewQueryForToken().
		Filter("Expiry <", time.Now()).
		KeysOnly().
		GetAll(ctx, nil)

	if err != nil {
		log.Warningf(ctx, "garbage collecting tokens failed: %s", err.Error())
		return http.StatusInternalServerError, err
	}

	err = datastore.DeleteMulti(ctx, keys)

	if err != nil {
		log.Warningf(ctx, "garbage collecting tokens failed: %s", err.Error())
		return http.StatusInternalServerError, err
	}

	return http.StatusOK, nil
}
Esempio n. 17
0
func deletepost(rw http.ResponseWriter, req *http.Request) {
	c := appengine.NewContext(req)
	u := user.Current(c)
	s := req.FormValue("encoded_key")
	k, err := datastore.DecodeKey(s)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}
	mypost := Post{}
	err = datastore.Get(c, k, &mypost)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}
	if mypost.Author == u.String() {
		if mypost.OP {
			q := datastore.NewQuery("post").Ancestor(k).KeysOnly()
			keys, err := q.GetAll(c, struct{}{})
			if err != nil {
				http.Error(rw, err.Error(), http.StatusInternalServerError)
				return
			}
			err = datastore.DeleteMulti(c, keys)
			if err != nil {
				http.Error(rw, err.Error(), http.StatusInternalServerError)
				return
			}
		} else {
			err := datastore.Delete(c, k)
			if err != nil {
				http.Error(rw, err.Error(), http.StatusInternalServerError)
				return
			}
		}
		// 	http.Redirect(rw, req, "/view", http.StatusTemporaryRedirect)
		// } else {
		http.Redirect(rw, req, "/", http.StatusTemporaryRedirect)
	}
}
Esempio n. 18
0
// DeleteMulti is a batch version of Delete.
func (g *Goon) DeleteMulti(keys []*datastore.Key) error {
	if len(keys) == 0 {
		return nil
		// not an error, and it was "successful", so return nil
	}
	memkeys := make([]string, len(keys))

	g.cacheLock.Lock()
	for i, k := range keys {
		mk := memkey(k)
		memkeys[i] = mk

		if g.inTransaction {
			delete(g.toSet, mk)
			g.toDelete[mk] = true
		} else {
			delete(g.cache, mk)
		}
	}
	g.cacheLock.Unlock()

	// Memcache needs to be updated after the datastore to prevent a common race condition,
	// where a concurrent request will fetch the not-yet-updated data from the datastore
	// and populate memcache with it.
	if g.inTransaction {
		for _, mk := range memkeys {
			g.toDeleteMC[mk] = true
		}
	} else {
		defer memcache.DeleteMulti(g.Context, memkeys)
	}

	multiErr, any := make(appengine.MultiError, len(keys)), false
	goroutines := (len(keys)-1)/deleteMultiLimit + 1
	var wg sync.WaitGroup
	wg.Add(goroutines)
	for i := 0; i < goroutines; i++ {
		go func(i int) {
			defer wg.Done()
			lo := i * deleteMultiLimit
			hi := (i + 1) * deleteMultiLimit
			if hi > len(keys) {
				hi = len(keys)
			}
			dmerr := datastore.DeleteMulti(g.Context, keys[lo:hi])
			if dmerr != nil {
				any = true // this flag tells DeleteMulti to return multiErr later
				merr, ok := dmerr.(appengine.MultiError)
				if !ok {
					g.error(dmerr)
					for j := lo; j < hi; j++ {
						multiErr[j] = dmerr
					}
					return
				}
				copy(multiErr[lo:hi], merr)
			}
		}(i)
	}
	wg.Wait()
	if any {
		return realError(multiErr)
	}
	return nil
}
Esempio n. 19
0
File: api.go Progetto: jordonwii/hms
func handleRemove(w http.ResponseWriter, r *http.Request, apiKey APIKey) *appError {
	if r.Method != "DELETE" {
		return &appError{nil, fmt.Sprintf("Invalid request method: %s", r.Method), 401}
	}

	c := appengine.NewContext(r)

	strChatID := r.FormValue("chatID")
	rmPath := r.FormValue("path")

	if rmPath == "" {
		return &appError{nil, "Missing path.", 401}
	}

	var fbChatID int64 = -1
	var chatKey *datastore.Key
	var err error

	if strChatID != "" {
		fbChatID, err = strconv.ParseInt(strChatID, 10, 64)
		if err != nil {
			return &appError{nil, "Bad chat ID", 400}
		}
		chatKeys, err := datastore.NewQuery("Chat").Filter("FacebookChatID =", fbChatID).
			KeysOnly().GetAll(c, nil)

		if err != nil {
			return &appError{err, "Datastore error: " + err.Error(), 400}
		} else if len(chatKeys) == 0 {
			return &appError{nil, "Bad chat ID", 400}
		}

		chatKey = chatKeys[0]
	} else {
		chatKey = nil
	}

	deleted := make([]Link, 0)
	keysToRemove, err := datastore.NewQuery("Link").
		Filter("Path =", rmPath).Filter("ChatKey =", chatKey).GetAll(c, &deleted)

	if len(keysToRemove) != 0 {
		newKeys := make([]*datastore.Key, len(keysToRemove))
		for i := range keysToRemove {
			newKeys[i] = datastore.NewIncompleteKey(c, "DeletedLink", nil)
		}

		err = datastore.RunInTransaction(c, func(tc context.Context) (err error) {
			_, err = datastore.PutMulti(c, newKeys, deleted)
			if err != nil {
				return
			}
			err = datastore.DeleteMulti(c, keysToRemove)
			return
		}, nil)

	}

	var resp RemoveResponse
	if err != nil {
		resp = RemoveResponse{
			false,
			0,
			err.Error(),
		}
	} else {
		resp = RemoveResponse{
			true,
			len(keysToRemove),
			"",
		}
	}

	respJSON, _ := json.Marshal(resp)
	w.Write(respJSON)
	return nil
}
Esempio n. 20
0
// DeleteMulti wraps datastore.DeleteMulti
func (d *Driver) DeleteMulti(key []*datastore.Key) error {
	d.logOps(opDelete, len(key), "DeleteMulti")
	return datastore.DeleteMulti(d.ctx, key)
}
Esempio n. 21
0
func (d rdsImpl) DeleteMulti(ks []ds.Key, cb ds.DeleteMultiCB) error {
	err := datastore.DeleteMulti(d, dsMF2R(ks))
	return idxCallbacker(err, len(ks), func(_ int, err error) {
		cb(err)
	})
}
Esempio n. 22
0
// ReplaceAuthDB updates database with given AuthDBSnapshot if it is new.
// The first return value indicates database was updated,
// and the second one has the latest AuthReplicationState.
func ReplaceAuthDB(c context.Context, newData AuthDBSnapshot) (bool, *AuthReplicationState, error) {
	var stat *AuthReplicationState
	updated := false

	err := datastore.RunInTransaction(c, func(c context.Context) error {
		curstat := &AuthReplicationState{}
		if err := GetReplicationState(c, curstat); err != nil {
			return err
		}
		if newData.ReplicationState.PrimaryID != curstat.PrimaryID {
			return fmt.Errorf("primary id mismatch. incoming=%q; want=%q", newData.ReplicationState.PrimaryID, curstat.PrimaryID)
		}
		// database is already up-to-date.
		if curstat.AuthDBRev >= newData.ReplicationState.AuthDBRev {
			stat = curstat
			return nil
		}

		dbsnap := &AuthDBSnapshot{}
		if err := currentAuthDBSnapshot(c, dbsnap); err != nil {
			return err
		}

		var newEntities []entity
		var delKeys []*datastore.Key
		// Going to update database.
		if !reflect.DeepEqual(newData.GlobalConfig, dbsnap.GlobalConfig) {
			newEntities = append(newEntities, newData.GlobalConfig)
		}

		newGrps, delGrKeys := groupsDiff(dbsnap.Groups, newData.Groups)
		newEntities = append(newEntities, newGrps...)
		delKeys = append(delKeys, delGrKeys...)

		newWls, delWlKeys := whitelistsDiff(dbsnap.IPWhitelists, newData.IPWhitelists)
		newEntities = append(newEntities, newWls...)
		delKeys = append(delKeys, delWlKeys...)

		if !reflect.DeepEqual(newData.IPWhitelistAssignments, dbsnap.IPWhitelistAssignments) {
			newEntities = append(newEntities, newData.IPWhitelistAssignments)
		}

		curstat.AuthDBRev = newData.ReplicationState.AuthDBRev
		curstat.ModifiedTimestamp = newData.ReplicationState.ModifiedTimestamp

		var wg sync.WaitGroup
		ch := make(chan error, 3)
		wg.Add(3)
		go func() {
			defer wg.Done()
			if _, err := datastore.Put(c, ReplicationStateKey(c), curstat); err != nil {
				ch <- err
				return
			}
			ch <- nil
		}()
		go func() {
			defer wg.Done()
			keys := make([]*datastore.Key, 0, len(newEntities))
			for _, n := range newEntities {
				keys = append(keys, n.key())
			}
			if _, err := datastore.PutMulti(c, keys, newEntities); err != nil {
				ch <- err
				return
			}
			ch <- nil
		}()
		go func() {
			defer wg.Done()
			if err := datastore.DeleteMulti(c, delKeys); err != nil {
				ch <- err
				return
			}
			ch <- nil
		}()
		go func() {
			wg.Wait()
			close(ch)
		}()

		for err := range ch {
			if err != nil {
				return err
			}
		}
		stat = curstat
		updated = true
		return nil
	}, &datastore.TransactionOptions{
		XG: true,
	})
	if err != nil {
		return false, nil, err
	}
	return updated, stat, nil
}