func GetPublicTimeline(c appengine.Context) ([]TweetTw, error) {
url := "http://api.twitter.com/statuses/public_timeline.json"
request, _ := http.NewRequest("GET", url, nil)
request.Header.Set("Authorization", oAuthHeader(c, "GET", url, map[string]interface{}{
"withApplicationAccessToken": true,
}))
client := urlfetch.Client(c)
response, err := client.Do(request)
if err != nil {
c.Errorf("GetPublicTimeline failed to api call: %v", err)
return nil, err
}
jsonVal, err2 := ioutil.ReadAll(response.Body)
if err2 != nil {
c.Errorf("GetPublicTimeline failed to read result: %v", err)
return nil, err
}
tweets := make([]TweetTw, 0, 20)
json.Unmarshal(jsonVal, &tweets)
if len(tweets) == 0 {
//maybe error.
var errRes ErrorResponse
json.Unmarshal(jsonVal, &errRes)
c.Errorf("GetPublicTimeline error message: %v", errRes.Error)
return nil, errors.New(errRes.Error)
}
for _, tweet := range tweets {
c.Debugf("GetPublicTimeline %v: %v", tweet.User.Screen_name, tweet.Text)
}
return tweets, nil
}
func getTickets(c appengine.Context, status string, newStatus string, limit int) ([]Ticket, []*datastore.Key, error) {
c.Debugf("getTickets: ", status)
statusKey, err := getStatusByType(c, status)
if err != nil {
return nil, nil, err
}
c.Debugf("statusKey: ", statusKey)
q := datastore.NewQuery(DATASTORE_TICKET).
Filter("Status =", statusKey).
//Filter("SellerUrl >", "").
Order("Modified").
Limit(limit)
var tickets []Ticket
ticket_keys, err := q.GetAll(c, &tickets)
if err != nil {
return nil, nil, err
}
if len(tickets) < 1 {
return nil, nil, fmt.Errorf("getTickets: No new tickets found with status: %s", status)
}
if newStatus != "" {
err = setTicketsStatus(c, tickets, ticket_keys, newStatus)
if err != nil {
return nil, nil, err
}
}
return tickets, ticket_keys, nil
}
// GetMulti is a batch version of Get. Cached values are returned from memcache, uncached values are returned from
// datastore and memcached for next time.
//
// dst must be a []S, []*S, []I or []P, for some struct type S, some interface type I, or some non-interface
// non-pointer type P such that P or *P implements PropertyLoadSaver. If an []I, each element must be a valid
// dst for Get: it must be a struct pointer or implement PropertyLoadSaver.
//
// As a special case, PropertyList is an invalid type for dst, even though a PropertyList is a slice of structs.
// It is treated as invalid to avoid being mistakenly passed when []PropertyList was intended.
func GetMulti(c appengine.Context, key []*datastore.Key, dst interface{}) error {
if len(key) == 0 {
return nil
}
// check cache
encodedKeys := encodeKeys(key)
itemMap, errm := memcache.GetMulti(c, encodedKeys)
if len(itemMap) != len(key) {
// TODO benchmark loading all vs loading missing
// load from datastore
errd := datastore.GetMulti(c, key, dst)
if Debug {
c.Debugf("reading from datastore: %#v", dst)
}
if errd != nil {
return errd
}
// cache for next time
errm = cache(key, dst, c)
} else {
errm = decodeItems(key, itemMap, dst)
if Debug {
c.Debugf("reading from memcache: %#v", dst)
}
}
return errm
}
func apiComfortStationGetAll(u *user.User, c appengine.Context, w http.ResponseWriter, r *http.Request) interface{} {
c.Debugf("apiComfortStationGetAll")
var stations []*ComfortStation
var keys []*datastore.Key
var context = struct {
Locations []*ComfortStation `json:"locations"`
}{}
intId, err := getIdFromRequest(r, c)
if noErrMsg(err, w, c, "Getting ID from Request") {
c.Debugf("load for %d", intId)
teamKey := datastore.NewKey(c, "Team", "", intId, nil)
q := datastore.NewQuery("ComfortStation").Filter("TeamKey =", teamKey)
keys, err = q.GetAll(c, &stations)
for x, _ := range stations {
s := stations[x]
s.setKey(keys[x])
}
context.Locations = stations
}
return context
}
// Log is a helper function that logs the given message to appenging
// with the given priority. Accepted priorities are "debug", "info",
// "warn", "error", and "crit". Other values default to "error".
func Log(c appengine.Context, r *http.Request, priority string,
message string, params ...interface{}) {
message = fmt.Sprintf("[%s] [%s] [%s]: %s", r.RemoteAddr, r.Method,
r.URL, message)
switch priority {
case "debug":
c.Debugf(message, params...)
case "info":
c.Infof(message, params...)
case "warn":
c.Warningf(message, params...)
case "error":
c.Errorf(message, params...)
case "crit":
c.Criticalf(message, params...)
default:
c.Errorf(message, params...)
}
}
func GetTrends(c appengine.Context) ([]Trend, error) {
url := "http://api.twitter.com/1/trends/1118370.json"
request, _ := http.NewRequest("GET", url, nil)
request.Header.Set("Authorization", oAuthHeader(c, "GET", url, map[string]interface{}{
"withApplicationAccessToken": true,
}))
client := urlfetch.Client(c)
response, err := client.Do(request)
if err != nil {
c.Errorf("GetTrends failed to api call: %v", err)
return nil, err
}
jsonVal, err2 := ioutil.ReadAll(response.Body)
if err2 != nil {
c.Errorf("GetTrends failed to read result: %v", err)
return nil, err
}
locales := make([]Locale, 0, 1)
json.Unmarshal(jsonVal, &locales)
if len(locales) == 0 || len(locales[0].Trends) == 0 {
//maybe error.
var errRes ErrorResponse
json.Unmarshal(jsonVal, &errRes)
return nil, errors.New(errRes.Error)
}
for _, trend := range locales[0].Trends {
c.Debugf("GetTrends trend.Name: %v", trend.Name)
}
return locales[0].Trends, nil
}
func sendGif(c appengine.Context, w http.ResponseWriter, bytes []byte) {
w.Header().Set("Content-Type", "image/gif")
w.Header().Set("Cache-Control", "max-age="+maxAge)
if _, err := w.Write(bytes); err != nil {
c.Debugf("can't write HTTP response: %v", err)
}
}
/*
Does the actual google books API call.
*/
func lookupISBN(ctx appengine.Context, country string, isbn isbn13.ISBN13) (resp *data.BookMetaData, err error) {
var r *http.Response
url := fmt.Sprintf(lookupURLTemplate, uint64(isbn), country, apiKey)
ctx.Debugf("Calling %s", fmt.Sprintf(lookupURLTemplate, uint64(isbn), country, "<hidden>"))
client := urlfetch.Client(ctx)
if r, err = client.Get(url); err == nil {
if r.StatusCode != http.StatusOK {
err = fmt.Errorf("Google API returned %s", r.Status)
} else {
reply := new(data.LookupReply)
decode := json.NewDecoder(r.Body)
defer r.Body.Close()
ctx.Infof("Completed API call, result %s\n", r.Status)
if err = decode.Decode(reply); err == nil {
if reply.Count == 1 {
resp = &reply.BookInfos[0]
} else {
ctx.Infof("Google books reported %d matching items: %v", reply.Count, reply)
resp = new(data.BookMetaData)
resp.Volume.Title = placeHolderText + isbn.String()
}
resp.Parent = country
resp.ISBN = isbn.String()
}
}
}
return
}
func RetrieveActDetails(c appengine.Context, actId int) (res *ActDetail, err os.Error) {
var d DSActDetail
if itm, err := memcache.Get(c, "actId__"+strconv.Itoa(actId)); err != nil && err != memcache.ErrCacheMiss {
return nil, err
} else if err == nil {
// Cache hit
buf := bytes.NewBuffer(itm.Value)
dec := gob.NewDecoder(buf)
dec.Decode(&d)
} else {
// Cache miss
key := datastore.NewKey(c, "DSActDetail", "", int64(actId), nil)
if err := datastore.Get(c, key, &d); err == datastore.ErrNoSuchEntity {
return &ActDetail{ActId: actId}, nil
} else if err != nil {
return nil, err
}
buf := bytes.NewBufferString("")
enc := gob.NewEncoder(buf)
enc.Encode(d)
itm := &memcache.Item{
Key: "actId__" + strconv.Itoa(actId),
Value: buf.Bytes(),
}
err = memcache.Set(c, itm)
c.Debugf("Request cache to memcache")
}
return d.fromDS(), nil
}
func apiComfortStationSave(u *user.User, c appengine.Context, w http.ResponseWriter, r *http.Request) interface{} {
c.Debugf("apiComfortStationSave")
var pd struct {
Station *ComfortStation
Team *Team
}
decoder := json.NewDecoder(r.Body)
jsonDecodeErr := decoder.Decode(&pd)
if jsonDecodeErr == io.EOF {
c.Infof("EOF, should it be?")
} else if noErrMsg(jsonDecodeErr, nil, c, "Parsing json from body") {
c.Infof("JSON from request: Station: %+v, Team: %+v", pd.Station, pd.Team)
mem, err := getMemberFromUser(c, u)
if noErrMsg(err, w, c, "Getting member from user") {
pd.Station.TeamKey = datastore.NewKey(c, "Team", "", pd.Team.KeyID, nil)
err = pd.Station.save(mem, c)
if noErrMsg(err, w, c, "Saving ComfortStation") {
}
}
}
return pd
}
// AppEngineLogHandler sends logs to AppEngine.
// The record must contain the appengine request context.
func AppEngineLogHandler() log15.Handler {
logFormat := log15.JsonFormat()
return log15.FuncHandler(func(r *log15.Record) error {
var c appengine.Context
index := 0
for i, e := range r.Ctx {
if ct, ok := e.(appengine.Context); ok {
c = ct
index = i
break
}
}
if c == nil {
// not in the context of a request
return nil
}
r.Ctx = append(r.Ctx[:index-1], r.Ctx[index+1:]...)
log := string(logFormat.Format(r))
switch r.Lvl {
case log15.LvlCrit:
c.Criticalf(log)
case log15.LvlError:
c.Errorf(log)
case log15.LvlWarn:
c.Warningf(log)
case log15.LvlInfo:
c.Infof(log)
case log15.LvlDebug:
c.Debugf(log)
}
return nil
})
}
func parseInput(c appengine.Context, r *http.Request) (input []string, err error) {
defer r.Body.Close()
var buf bytes.Buffer
buf.ReadFrom(r.Body)
c.Debugf("POST body is %d", buf.Len())
b := buf.Bytes()
payload, err := simplejson.NewJson(b)
if err != nil {
return
}
/*
dbg, err := payload.EncodePretty()
if err != nil {
return
}
c.Debugf("POST body is %v", string(dbg))
*/
search, err := payload.Get("search").Array()
if err != nil {
return
}
for _, value := range search {
i, ok := value.(string)
if ok {
input = append(input, i)
}
}
return
}
func FavoriteTweetByUser(c appengine.Context, statusId string, user TwitterUser) error {
url := fmt.Sprintf("http://api.twitter.com/1/favorites/create/%s.json", statusId)
request, _ := http.NewRequest("POST", url, bytes.NewBufferString(""))
request.Header.Set("Authorization", oAuthHeader(c, "POST", url, map[string]interface{}{
"withUserAccessToken": true,
"user": &user,
}))
c.Debugf("FavoriteTweetByUser Authorization: %v", request.Header.Get("Authorization"))
client := urlfetch.Client(c)
response, err := client.Do(request)
if err != nil {
c.Errorf("FavoriteTweetByUser failed to api call: %v", err)
return err
}
jsonVal, err2 := ioutil.ReadAll(response.Body)
if err2 != nil {
c.Errorf("FavoriteTweetByUser failed to read result: %v", err)
return err
}
c.Debugf("FavoriteTweetByUser response: %v", string(jsonVal))
if response.StatusCode != 200 {
c.Errorf("FavoriteTweetByUser failed to post status. StatusCode: %d", response.StatusCode)
return errors.New("FavoriteTweetByUser failed to post status.")
}
return nil
}
func filterUrls(c appengine.Context, base *url.URL, links []*url.URL) ([]*url.URL, error) {
newLinks := []*url.URL{}
err := isDomainBlacklisted(base)
if err == nil {
newLinks = append(newLinks, base)
}
c.Debugf("Base url:", base.String())
for _, link := range links {
link.Fragment = ""
/*c.Debugf("link:", link)
c.Debugf("Host:", link.Host)
c.Debugf("Empty domain?:", link.Host == "")
c.Debugf("Same domain?:", link.Host == base.Host)*/
err := validateDomain(base, link)
if err != nil {
c.Debugf(err.Error())
continue
}
if inSlice(link, newLinks) {
continue
}
if len(newLinks) > PARSER_MAX_URLS {
break
}
newLinks = append(newLinks, link)
}
return newLinks, nil
}
func DecrementOldHashtags(c appengine.Context, length int) error {
// 古くてカウントが多いもののカウントを減らす
q := datastore.NewQuery("Hashtag").Order("-Count").Limit(length)
hashtags := make([]Hashtag, 0, length)
if _, err := q.GetAll(c, &hashtags); err != nil {
c.Errorf("DecrementOldHashtags failed to search hashtags for decrement")
return err
}
c.Infof("DecrementOldHashtags got old hashtags len: %d", len(hashtags))
for _, h := range hashtags {
// if h.Count <= 4 {
// continue;
// }
c.Debugf("DecrementOldHashtags old hashtag decrement before: %v (%d)", h.Name, h.Count)
newInt, _ := rand.Int(rand.Reader, big.NewInt(2)) // 1 or 2 or 3
h.Count = int(newInt.Int64()) + 1
if h.Count > 5 {
h.Count = 5
}
if h.Count < 1 {
h.Count = 1
}
c.Infof("DecrementOldHashtags old hashtag decrement after : %v (%d)", h.Name, h.Count)
key := datastore.NewKey(c, "Hashtag", h.Name, 0, nil)
if _, err := datastore.Put(c, key, &h); err != nil {
c.Errorf("DecrementOldHashtags failed to put old hashtag decrement: %v", err)
return err
}
}
return nil
}
func PostTweetByUser(c appengine.Context, status string, user TwitterUser) (*TweetTw, error) {
encodedStatus := Encode(status)
url := "http://api.twitter.com/statuses/update.json"
body := strings.NewReader("status=" + encodedStatus)
request, _ := http.NewRequest("POST", url, body)
request.Header.Set("Authorization", oAuthHeader(c, "POST", url, map[string]interface{}{
"withUserAccessToken": true,
"user": &user,
"status": encodedStatus,
}))
c.Debugf("PostTweetByUser Authorization: %v", request.Header.Get("Authorization"))
client := urlfetch.Client(c)
response, err := client.Do(request)
if err != nil {
c.Errorf("PostTweetByUser failed to api call: %v", err)
return nil, err
}
jsonVal, err2 := ioutil.ReadAll(response.Body)
if err2 != nil {
c.Errorf("PostTweetByUser failed to read result: %v", err)
return nil, err
}
c.Debugf("PostTweetByUser response: %v", string(jsonVal))
if response.StatusCode != 200 {
c.Errorf("PostTweetByUser failed to post status. StatusCode: %d", response.StatusCode)
return nil, errors.New("PostTweetByUser failed to post status.")
}
var tweet TweetTw
json.Unmarshal(jsonVal, &tweet)
return &tweet, nil
}
func sendJson(c appengine.Context, w http.ResponseWriter, data []byte) {
w.Header().Add("Content-Type", "application/json")
_, err := w.Write(data)
if err != nil {
c.Debugf("can't write json to client: %v", err)
}
}
func getAzID(c appengine.Context, asset string) (azID string, notationIDs []string, err error) {
// Get ID Notation (IDs for underlying by expiry)
urlIDNotation, err := prepareURL(URLIDNOTATION, asset)
if err != nil {
return "", nil, err
}
txt, err := fetchContent(c, urlIDNotation)
if err != nil {
return "", nil, err
}
notationIDs, err = extractNotationIDs(txt)
if err != nil {
return "", nil, err
}
// Get the final azID
txt, err = fetchContent(c, URLID)
if err != nil {
return "", nil, err
}
azID = txt[strings.Index(txt, AuthID)+19 : strings.Index(txt, "==\";")+2]
c.Debugf("API AZID: %v", azID)
return azID, notationIDs, nil
}
func verifyMassageWithPaypal(ctx appengine.Context, content string, testIpnField string) error {
paypalIpnUrl := PaypalIpn
if testIpnField != "" {
if testIpnField == "1" {
paypalIpnUrl = PaypalIpnSandBox
} else {
paypalIpnUrl = localIpn
}
}
ctx.Infof("Sending msg to: " + paypalIpnUrl)
extraData := []byte("cmd=_notify-validate&")
client := urlfetch.Client(ctx)
resp, err := client.Post(paypalIpnUrl, FromEncodedContentType, bytes.NewBuffer(append(extraData, content...)))
if err != nil {
return err
}
respBody, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
ctx.Debugf("Ipn Validation response " + string(respBody))
if err == nil && string(respBody) != "VERIFIED" {
return IpnMessageCouldNotBeValidated
}
return err
}
func PutRoom(c appengine.Context, name string, room *Room) error {
room.Name = name
room.LastChanged = time.Now()
k := datastore.NewKey(c, "Room", name, 0, nil)
_, err := datastore.Put(c, k, room)
c.Debugf("Storing %+v", room)
return err
}
func expire(c appengine.Context, w http.ResponseWriter, r *http.Request) {
expireDelay.Call(c, time.Now(), "")
w.WriteHeader(200)
io.WriteString(w, "Expiring peers...")
c.Debugf("Successfully began expire")
}
func fetchContent(c appengine.Context, url string) (string, error) {
c.Debugf("Fetching URL: %v", url)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return "", err
}
return sendReq(c, req)
}
func (self *InitNavFilter) Filte(w http.ResponseWriter, r *http.Request, c appengine.Context) bool {
c.Debugf("initnavfilter filte function")
pageCube := self.GetApp().GetPageCube(r)
pageCube["ActiveNav"] = "home"
return true
}
// PutMulti is a batch version of Put.
//
// src must satisfy the same conditions as the dst argument to GetMulti.
func PutMulti(c appengine.Context, key []*datastore.Key, src interface{}) ([]*datastore.Key, error) {
if Debug {
c.Debugf("writing to datastore: %#v", src)
}
key, errd := datastore.PutMulti(c, key, src)
memcache.DeleteMulti(c, encodeKeys(key))
return key, errd
}
func DelRooms(c appengine.Context, rooms []Room) error {
keys := make([]*datastore.Key, 0)
for _, room := range rooms {
keys = append(keys, datastore.NewKey(c, "Room", room.Name, 0, nil))
}
c.Debugf("Deleting Rooms: %+v", keys)
err := datastore.DeleteMulti(c, keys)
return err
}
func postContent(c appengine.Context, url string, content string) (string, error) {
c.Debugf("Post URL: %v with Content: %v", url, content)
req, err := http.NewRequest("POST", url, strings.NewReader(content))
req.Header.Add("Content-Type", "text/plain; charset=UTF-8")
if err != nil {
return "", err
}
return sendReq(c, req)
}
// save updates this Notebook in the datastore
func (notebook *Notebook) save(c appengine.Context) error {
if Debug {
c.Debugf("updating notebook: %#v", *notebook)
}
_, err := cachestore.Put(c, notebook.Key(c), notebook)
if err != nil {
c.Errorf("updating notebook: %s", err)
}
return err
}
// cache writes structs and PropertyLoadSavers to memcache.
func cache(key []*datastore.Key, src interface{}, c appengine.Context) error {
items, err := encodeItems(key, src)
if len(items) > 0 && err == nil {
if Debug {
c.Debugf("writing to memcache: %#v", src)
}
err = memcache.SetMulti(c, items)
}
return err
}
// FetchUrl reads a feed from the given URL.
func fetchUrl(c appengine.Context, url string) (FeedInfo, Articles, error) {
var finfo FeedInfo
resp, err := urlfetch.Client(c).Get(url)
if err != nil {
return finfo, nil, err
}
defer resp.Body.Close()
feed, err := webfeed.Read(resp.Body)
if err != nil {
if _, ok := err.(webfeed.ErrBadTime); ok {
c.Debugf("%s: %s", url, err.Error())
err = nil
} else {
err = errors.New("failed to fetch " + url + ": " + err.Error())
return finfo, nil, err
}
}
finfo.Url = url
finfo.Title = feed.Title
if finfo.Title == "" {
finfo.Title = url
}
finfo.Link = feed.Link
finfo.LastFetch = time.Now()
as := make(Articles, len(feed.Entries))
for i, ent := range feed.Entries {
content := ent.Content
if len(ent.Content) == 0 {
content = ent.Summary
}
title := ent.Title
if title == "" && len(ent.Summary) > 0 {
n := len(ent.Summary)
if n > 20 {
n = 20
}
title = string(ent.Summary[:n]) + "…"
}
if title == "" {
title = ent.Link
}
as[i] = Article{
Title: title,
Link: ent.Link,
OriginTitle: feed.Title,
DescriptionData: content,
When: ent.When,
}
}
return finfo, as, nil
}
func processEmail(msg *mail.Message, c appengine.Context) {
c.Debugf("Yay, my own handler! email from %v", msg.Header)
// parse from address. return if error
addr, err := mail.ParseAddress(msg.Header["From"][0])
if err != nil {
c.Errorf("Wrong email from: %s", msg.Header["From"][0])
return
}
subject := msg.Header["Subject"][0]
// if subscribe request starting with "reg", send subscription code and return
if strings.HasPrefix(subject, "reg") {
sendSubscription(addr.Address, c)
return
}
// or unregsiter if "unreg"
if strings.HasPrefix(subject, "unreg") {
unregisterUser(addr.Address, c)
return
}
// if confirmation, fully regist it and return
pos := strings.Index(subject, "confirm ")
if pos >= 0 {
code := subject[pos+len("confirm "):]
confirmSubscription(addr.Address, code, c)
return
}
// check if email is there. return if none (not registered)
q := datastore.NewQuery(USER_MODEL).
Filter("Email =", addr.Address)
var u []UserStatus
keys, err := q.GetAll(c, &u)
if err != nil {
c.Errorf("Could not retrieve user status for %s: %v", addr.Address, err)
return
}
if len(u) != 1 {
c.Errorf("There's no such user %s, len(u) == %d", addr.Address, len(u))
return
}
// If the subject is on/On/oN/ON, it's on. Off otherwise
if strings.EqualFold(subject, "on") {
u[0].Status = "on"
} else {
u[0].Status = "off"
}
if _, err = datastore.Put(c, keys[0], &u[0]); err != nil {
c.Errorf("Could not write new status for %s: %v", addr.Address, err)
}
c.Infof("Updated %s to %s", addr.Address, u[0].Status)
return
}