func SearchUsers(c appengine.Context, query string) ([]StoredUserMeta, error) { db := DB{c} index, err := search.Open("users") if err != nil { return nil, err } var users []StoredUserMeta opts := &search.SearchOptions{ Limit: 100, IDsOnly: true, } for t := index.Search(c, query, opts); ; { key, err := t.Next(nil) if err == search.Done { break } else if err != nil { return nil, err } id, _ := strconv.ParseInt(key, 10, 64) userMeta, err := db.GetUserMeta(id) users = append(users, userMeta.(StoredUserMeta)) } return users, nil }
func searchHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) index, err := search.Open("Package") if err != nil { log.Print(err) } c := appengine.NewContext(r) packages := []*SearchResult{} for t := index.Search(c, vars["query"], nil); ; { var doc Package id, err2 := t.Next(&doc) if err2 == search.Done { break } if err2 != nil { log.Print(w, "Search error: %v\n", err2) break } log.Print(id) packages = append(packages, &SearchResult{ Id: id, Url: doc.Url, }) } jsonPackages, err3 := json.Marshal(packages) if err3 != nil { http.Error(w, err3.Error(), http.StatusInternalServerError) return } w.Header().Set("Content-Type", "application/json") w.Write(jsonPackages) }
func debugIndex(w http.ResponseWriter, r *http.Request) { context := appengine.NewContext(r) // Fetch document index. index, _ := search.Open(ImageIndex) num := 0 for t := index.List(context, nil); ; { doc := &ImageDoc{} _, err := t.Next(doc) if err == search.Done { break } if err != nil { fmt.Fprintf(w, err.Error()) break } fmt.Fprintf(w, doc.String()) num++ } fmt.Fprintf(w, "%d documents in index", num) updated := getLastUpdated(context) fmt.Fprintf(w, "%d last updated", updated) }
func importHistory(c appengine.Context, commands []History) { log.Printf("Started delayed task") index, err := search.Open(HISTORY_INDEX) if err != nil { log.Printf("Could not connect to search service. %s", err.Error()) return } ctr := 0 for _, h := range commands { data := []byte(h.Command) docId := fmt.Sprintf("%x", md5.Sum(data)) _, err = index.Put(c, docId, &h) if err != nil { return } ctr += 1 } log.Printf("Finished delayed task. Imported %d", ctr) }
func index(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { c := appengine.NewContext(r) index, err := search.Open(xkcdIndex) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } var x *Comic comicNum := r.FormValue("id") if comicNum != "" { iComicNum, _ := strconv.Atoi(comicNum) x, _ = Get(c, iComicNum) } else { x, _ = GetCurrent(c) } xSearch := &ComicSearch{ strconv.Itoa(x.Num), x.Title, x.Img, x.Alt, x.Transcript, } id := strconv.Itoa(x.Num) _, err = index.Put(c, id, xSearch) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } w.WriteHeader(http.StatusNoContent) }
func backfill(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { c := appengine.NewContext(r) index, err := search.Open(xkcdIndex) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } current, _ := GetCurrent(c) for i := 1; i <= current.Num; i++ { // xcdc returns 404 with issue 404 if i == 404 { continue } comicNum := strconv.Itoa(i) force := r.FormValue("force") if force != "yes" { var s ComicSearch err := index.Get(c, comicNum, &s) if err == nil { continue } } t := taskqueue.NewPOSTTask("/index", map[string][]string{"id": {comicNum}}) if _, err := taskqueue.Add(c, t, ""); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } } }
func searchPut(w http.ResponseWriter, r *http.Request) { lg, b := loghttp.BuffLoggerUniversal(w, r) _ = b id := "PA6-5001" user := &User{ Customer: "Carl Corral", Comment: "I am <em>riled up</em> text", Visits: 1, LastVisit: time.Now(), Birthday: time.Date(1968, time.May, 19, 0, 0, 0, 0, time.UTC), } c := appengine.NewContext(r) index, err := search.Open("users") lg(err) ret_id, err := index.Put(c, id, user) lg(err) fmt.Fprint(w, "OK, saved "+ret_id+"\n\n") var u2 User err = index.Get(c, ret_id, &u2) lg(err) fmt.Fprint(w, "Retrieved document: ", u2) }
func searchBooks(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) // start search OMIT index, err := search.Open("Book") // HL if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } books := make([]*Book, 0, 10) t := index.Search(c, "Gopher Price >= 3000", nil) // HL for { var book Book _, err := t.Next(&book) if err == search.Done { break } else if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } books = append(books, &book) } // end search OMIT je := json.NewEncoder(w) if err := je.Encode(books); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } }
func searchSlashCommandHandler(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { c := appengine.NewContext(r) text := r.FormValue("text") if text == "" { http.Error(w, "Missing required fields", http.StatusBadRequest) return } index, err := search.Open(xkcdIndex) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } var comicList []*ComicSearch for t := index.Search(c, text, nil); ; { var xkcd ComicSearch _, err := t.Next(&xkcd) if err == search.Done { break } if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) break } comicList = append(comicList, &xkcd) } sr := new(searchSlashCommandResponse) if len(comicList) == 0 { sr.Text = "EPOCH FAIL!" } else { n := rand.Intn(len(comicList)) comic := comicList[n] attachment := &searchSlashCommandAttachment{ Text: comic.Alt, TitleLink: fmt.Sprintf("https://xkcd.com/%s/", comic.Num), Title: fmt.Sprintf("%s (#%s)", comic.Title, comic.Num), ImageUrl: comic.Img, } sr.Attachments = []*searchSlashCommandAttachment{attachment} sr.ResponseType = "in_channel" } rsp, _ := json.Marshal(sr) w.Header().Set("Content-Type", "application/json; charset=utf-8") fmt.Fprintf(w, string(rsp)) logSearch(c, r.FormValue("user_name"), r.FormValue("channel_name"), r.FormValue("team_domain"), text) }
func getHistory(res http.ResponseWriter, req *http.Request) { password := req.URL.Query().Get("password") if password != os.Getenv("SERVER_PASSWORD") { http.Error(res, "uh oh", http.StatusForbidden) return } c := appengine.NewContext(req) index, err := search.Open(HISTORY_INDEX) if err != nil { http.Error(res, err.Error(), http.StatusInternalServerError) return } // options := search.SearchOptions{ // Sort: &search.SortOptions{ // Expressions: []search.SortExpression{ // search.SortExpression{ // Expr: "", // Reverse: true, // }, // }, // }, // } history := HistorySlice{} for t := index.List(c, nil); ; { var h History _, err := t.Next(&h) if err == search.Done { break } if err != nil { fmt.Fprintf(res, "Search error: %v\n", err) break } history = append(history, h) } sort.Sort(history) for _, h := range history { fmt.Fprintf(res, "%v\n", h.Command) } }
func (r *WifiSpotRepository) GetIndex() *search.Index { // if r.Index == nil { index, err := search.Open("wifi_spots") if err != nil { panic(err.Error()) } // r.Index = index return index /** } else { return r.Index } **/ }
func addToIndex(p *Package, w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) index, err1 := search.Open("Package") if err1 != nil { http.Error(w, err1.Error(), http.StatusInternalServerError) return } id, err2 := index.Put(c, "", p) if err2 != nil { http.Error(w, err2.Error(), http.StatusInternalServerError) return } log.Print("index id " + id) }
func IndexPortal(c appengine.Context, portal Portal) error { index, err := search.Open("portals") if err != nil { return err } var sp SearchPortal sp.ToIndex(portal) _, err = index.Put(c, portal.Id, &sp) if err != nil { c.Infof("Error al indexar portal %s, id: %s", portal.Title, portal.Id) //http.Error(w, err.Error(), http.StatusInternalServerError) return err } return nil }
func DeleteHandler(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) index, _ := search.Open("portals") for t := index.List(c, nil); ; { var portal Portal id, err := t.Next(&portal) if err == search.Done { break } if err != nil { return } index.Delete(c, id) c.Infof("deleting index %s", id) } }
func handleSearchDelete(rw http.ResponseWriter, req *http.Request) { c := appengine.NewContext(req) getQuery := req.URL.Query() indexName := getQuery.Get("index") id := getQuery.Get("id") index, err := search.Open(indexName) if err != nil { jsonReplyMap(rw, http.StatusInternalServerError, "error", err.Error()) return } err = index.Delete(c, id) if err != nil { jsonReplyMap(rw, http.StatusInternalServerError, "error", err.Error()) return } jsonReplyMap(rw, http.StatusOK, "index", indexName, "id", id) }
func searchIndex(w http.ResponseWriter, r *http.Request) { context := appengine.NewContext(r) // Open document index. index, err := search.Open(ImageIndex) if err != nil { context.Errorf("%s", err.Error()) // fmt.Fprintf(w, "error: %s", err.Error()) return } // Get search terms. decoder := json.NewDecoder(r.Body) query := &Query{} if err := decoder.Decode(query); err != nil { context.Errorf("%s", err.Error()) // fmt.Fprintf(w, "error: %s", err.Error()) return } defer r.Body.Close() //fmt.Fprintf(w, "%s", query.Terms) // Execute the query using the terms from the request and saves the images images := make([]*ImageDoc, 0, 50) for t := index.Search(context, query.Terms, nil); ; { doc := &ImageDoc{} _, err := t.Next(doc) if err == search.Done { break } if err != nil { break } images = append(images, doc) // fmt.Fprintf(w, "Image: %s, URL: %s<br>", id, doc.URL) } // Encode images into json and send them out. encoder := json.NewEncoder(w) if err := encoder.Encode(images); err != nil { context.Errorf("%s", err.Error()) fmt.Fprintf(w, "%s", err.Error()) } }
func searchHistory(res http.ResponseWriter, req *http.Request) { password := req.URL.Query().Get("password") if password != os.Getenv("SERVER_PASSWORD") { http.Error(res, "uh oh", http.StatusForbidden) return } command := req.FormValue("command") if command == "" { http.Error(res, "missing param", http.StatusBadRequest) return } c := appengine.NewContext(req) index, err := search.Open(HISTORY_INDEX) if err != nil { http.Error(res, err.Error(), http.StatusInternalServerError) return } commands := []string{} for t := index.Search(c, command, nil); ; { var h History _, err := t.Next(&h) if err == search.Done { break } if err != nil { fmt.Fprintf(res, "Search error: %v\n", err) break } commands = append(commands, h.Command) } reverse(commands) fmt.Fprintf(res, "%v\n", strings.Join(commands, "\n")) }
func putSongDoc(c mp.Context, id int64, s *chordpro.Song) error { index, err := search.Open("songs") if err != nil { return err } doc := &data.SongDoc{ Title: m.Wordify(s.Title), Artist: m.Wordify(strings.Join(s.Artists, " ")), Album: m.Wordify(s.Album), Lyric: m.Wordify(s.Lyric()), } _, err = index.Put(c, strconv.FormatInt(id, 10), doc) if err != nil { return err } return nil }
func postHistory(res http.ResponseWriter, req *http.Request) { c := appengine.NewContext(req) command := req.FormValue("command") statusCode := http.StatusOK data := []byte(command) docId := fmt.Sprintf("%x", md5.Sum(data)) index, err := search.Open(HISTORY_INDEX) if err != nil { http.Error(res, err.Error(), http.StatusInternalServerError) return } var h History err = index.Get(c, docId, &h) if err == search.ErrNoSuchDocument { err = nil statusCode = http.StatusCreated } if err != nil { http.Error(res, err.Error(), http.StatusInternalServerError) return } h.Command = command h.Count += 1 h.Timestamp = time.Now() _, err = index.Put(c, docId, &h) if err != nil { log.Printf("Here") http.Error(res, err.Error(), http.StatusInternalServerError) return } res.WriteHeader(statusCode) fmt.Fprintf(res, "%+v", h) }
func searchRetrieve(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) index, err := search.Open("users") util_err.Err_log(err) for t := index.Search(c, "Comment:riled", nil); ; { var res User id, err := t.Next(&res) fmt.Fprintf(w, "\n-- ") if err == search.Done { break } if err != nil { fmt.Fprintf(w, "Search error: %v\n", err) break } fmt.Fprintf(w, "%s -> %#v\n", id, res) } }
func SearchPortals(c appengine.Context, query string, checkfavorited bool, favorited []string) ([]Portal, error) { c.Infof("%s", query) index, err := search.Open("portals") if err != nil { return nil, nil } var portals []SearchPortal var keys []*datastore.Key for t := index.Search(c, query, nil); ; { var sp SearchPortal id, err := t.Next(&sp) if err == search.Done { break } if err != nil { return nil, err } if checkfavorited { for _, portalid := range favorited { if portalid == id { portals = append(portals, sp) keys = append(keys, datastore.NewKey(c, "Portal", id, 0, nil)) } } } else { portals = append(portals, sp) keys = append(keys, datastore.NewKey(c, "Portal", id, 0, nil)) } } portals2 := make([]Portal, len(keys)) datastore.GetMulti(c, keys, portals2) for portalindex, portal := range portals2 { if _, err := datastore.NewQuery("Key").Filter("PortalId=", portal.Id).GetAll(c, &portal.Keys); err != nil { } portals2[portalindex] = portal } c.Infof("portals: %s", portals2) return portals2, nil }
func saveIndex(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) books := make([]*Book, 0, 3) books = append(books, &Book{ Title: "Perfect Go", Author: "Some Gopher", Price: 3000, CreatedAt: time.Now(), }) books = append(books, &Book{ Title: "Go In Practice", Author: "One Gopher", Price: 2500, CreatedAt: time.Now(), }) books = append(books, &Book{ Title: "Let it go", Author: "hogehoge", Price: 3000, CreatedAt: time.Now(), }) // start searchput OMIT index, err := search.Open("Book") // HL if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } for i, book := range books { _, err := index.Put(c, fmt.Sprintf("book%d", i), book) // HL if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } } // end searchput OMIT fmt.Fprint(w, "success") }
func updateAndIndex(w http.ResponseWriter, r *http.Request) { context := appengine.NewContext(r) images := getLatestImages(context) // Fetch document index. index, err := search.Open(ImageIndex) if err != nil { context.Errorf("%s", err.Error()) fmt.Fprint(w, err.Error()) return } // Create image docs from images so we can insert them into the index. for _, image := range images { imageDoc := &ImageDoc{ // ID: fmt.Sprintf("%d", image.ID), URL: image.URL, Created: time.Unix(image.Created, 0), Data: strings.Join(image.Tags, " "), } _, err := index.Put(context, fmt.Sprintf("%d", image.ID), imageDoc) if err != nil { fmt.Fprintf(w, err.Error()) } } // Debug fmt.Fprintf(w, "Added %d new images to index\n", len(images)) // Set new last updated time for next iteration. if len(images) > 0 { setLastUpdate(context, images[len(images)-1].Created) } else { // Hack for now. setLastUpdate(context, time.Now().Unix()) } }
func SearchSongs(c appengine.Context, query string) ([]*SongDoc, []string, error) { index, err := search.Open("songs") if err != nil { return nil, nil, err } docList := make([]*SongDoc, 0) docIDs := make([]string, 0) for t := index.Search(c, query, &search.SearchOptions{Limit: 50}); ; { var doc SongDoc id, err := t.Next(&doc) if err == search.Done { break } if err != nil { return nil, nil, err } docList = append(docList, &doc) docIDs = append(docIDs, id) } return docList, docIDs, nil }
func ReIndex(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) t := datastore.NewQuery("Portal").Run(c) index, _ := search.Open("portals") for { var portal Portal _, err := t.Next(&portal) if err == datastore.Done { break } if err != nil { return } var sp SearchPortal sp.ToIndex(portal) c.Infof("reindex - %s", portal) _, err = index.Put(c, portal.Id, &sp) if err != nil { c.Errorf("%s", err) return } c.Infof("indexint portal id: %s", portal.Id) } }
func (db *DB) FetchUser(user string) { vineApi := VineRequest{db.Context} data, err := vineApi.GetUser(user) if err != nil { if err.Error() == ErrUserDoesntExist.Error() { db.UnqueueUser(user) } return } else if data == nil { db.Context.Errorf("failed fetch on user %v. got err %v", user, err) return } else if data.Private == 1 { return } var userMeta StoredUserMeta var userData StoredUserData userId := data.UserId userIndex := &UserIndex{ Username: data.Username, Location: data.Location, Description: data.Description, } if len(data.VanityUrls) != 0 { userIndex.VanityUrl = strings.ToLower(data.VanityUrls[0]) } userMetaTemp, err := db.GetUserMeta(userId) if err == datastore.ErrNoSuchEntity { userMeta = StoredUserMeta{ Username: data.Username, UserId: data.UserIdStr, Location: data.Location, Description: data.Description, Verified: data.Verified == 1, AvatarUrl: data.AvatarUrl, Background: data.ProfileBackground, } if len(data.VanityUrls) != 0 { userMeta.VanityUrl = strings.ToLower(data.VanityUrls[0]) } if userMeta.Verified { userMeta.VerifiedDate = time.Now() } userData = StoredUserData{ LastUpdated: time.Now(), Followers: []int64{data.FollowerCount}, Following: []int64{data.FollowingCount}, Loops: []int64{data.LoopCount}, AuthoredPosts: []int64{data.AuthoredPostCount}, Revines: []int64{data.PostCount - data.AuthoredPostCount}, Likes: []int64{data.LikeCount}, Updated: []time.Time{time.Now()}, } } else { userMeta = userMetaTemp.(StoredUserMeta) if userMeta.UserId == "" { userMeta.UserId = data.UserIdStr } if userMeta.Location != data.Location { userMeta.Previous.Location = append(userMeta.Previous.Location, PreviousLocation{userMeta.Location, time.Now()}) userMeta.Location = data.Location } if userMeta.Username != data.Username { userMeta.Previous.Username = append(userMeta.Previous.Username, PreviousUsername{userMeta.Username, time.Now()}) userMeta.Username = data.Username } if userMeta.Description != data.Description { userMeta.Previous.Description = append(userMeta.Previous.Description, PreviousDescription{userMeta.Description, time.Now()}) userMeta.Description = data.Description } if userMeta.Background != data.ProfileBackground { userMeta.Previous.Background = append(userMeta.Previous.Background, PreviousBackground{userMeta.Background, time.Now()}) userMeta.Background = data.ProfileBackground } userDataTemp, err := db.GetUserData(userId) userData = userDataTemp.(StoredUserData) if err != datastore.ErrNoSuchEntity { userData.LastUpdated = time.Now() userData.Followers = append(userData.Followers, data.FollowerCount) userData.Following = append(userData.Following, data.FollowingCount) userData.Loops = append(userData.Loops, data.LoopCount) userData.AuthoredPosts = append(userData.AuthoredPosts, data.AuthoredPostCount) userData.Revines = append(userData.Revines, data.PostCount-data.AuthoredPostCount) userData.Likes = append(userData.Likes, data.LikeCount) userData.Updated = append(userData.Updated, time.Now()) } } userMeta.Current = StoredUserMetaCurrent{ Followers: data.FollowerCount, Following: data.FollowingCount, Loops: data.LoopCount, AuthoredPosts: data.AuthoredPostCount, Revines: data.PostCount - data.AuthoredPostCount, Likes: data.LikeCount, } dataKey := datastore.NewKey(db.Context, "UserData", "", userId, nil) metaKey := datastore.NewKey(db.Context, "UserMeta", "", userId, nil) index, err := search.Open("users") if err != nil { db.Context.Errorf(err.Error()) } else { index.Put(db.Context, data.UserIdStr, userIndex) } datastore.Put(db.Context, dataKey, &userData) datastore.Put(db.Context, metaKey, &userMeta) }