/
server.go
200 lines (176 loc) · 6.8 KB
/
server.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
package main
import (
"bytes"
"fmt"
"github.com/ChimeraCoder/anaconda"
"github.com/ChimeraCoder/mtwerk"
"io/ioutil"
"launchpad.net/goamz/aws"
"log"
"net/url"
"os"
"text/template"
"time"
)
var (
HIT_KEYWORDS = []string{"twitter", "emoji"}
//The following variables can be defined using environment variables
//to avoid committing them by mistake
TWITTER_CONSUMER_KEY = []byte(os.Getenv("TWITTER_CONSUMER_KEY"))
TWITTER_CONSUMER_SECRET = []byte(os.Getenv("TWITTER_CONSUMER_SECRET"))
TWITTER_ACCESS_TOKEN = []byte(os.Getenv("TWITTER_ACCESS_TOKEN"))
TWITTER_ACCESS_TOKEN_SECRET = []byte(os.Getenv("TWITTER_ACCESS_TOKEN_SECRET"))
twitterBot *anaconda.TwitterApi
awsAuth *aws.Auth
)
const (
ASSIGNMENT_DURATION = 600 // How long, in seconds, a worker has to complete the assignment
LIFETIME = 1200 * time.Second // How long, in seconds, before the task expires
MAX_ASSIGNMENTS = 1 // Number of times the task needs to be performed
AUTO_APPROVAL_DELAY = 0 // Seconds before the request is auto-accepted. Set to 0 to accept immediately
MAX_QUESTION_SIZE = 65535 // Kilobytes
REWARD = "0.50" // In USD, represented as a string. "0.50" represents 50¢
//TODO use a command-line flag (or envvar) to set reward
)
// TODO move template to a separate file
const HTMLQuestionTemplate = `{{define "T"}}<HTMLQuestion xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2011-11-11/HTMLQuestion.xsd">
<HTMLContent><![CDATA[
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
<script type="text/javascript" src="https://s3.amazonaws.com/mturk-public/externalHIT_v1.js"></script>
</head>
<body>
<form name="mturk_form" method="post" id="mturk_form" action="https://www.mturk.com/mturk/externalSubmit">
<input type="hidden" value="" name="{{.AssignmentId}}" id="{{.AssignmentId}}"/>
<h2>{{.Title}}</h2>
<div style="border: 2px #000000 solid">
<h4>
{{.TweetEmbed.Html}}
</h4>
</div>
<div>
Pick the emoji that you feel would be the best translation of this tweet. For example, if the tweet were
<blockquote class="twitter-tweet" lang="en"><p>Call me Ishmael</p>— Aditya Mukerjee (@chimeracoder) <a href="https://twitter.com/chimeracoder/statuses/412631000544333824">December 16, 2013</a></blockquote>
<script async src="//platform.twitter.com/widgets.js" charset="utf-8"></script>
you might translate it as into the following emoji:
<img src="{{.ImageUrl}}">
</div>
<p><textarea name="comment" cols="80" rows="3"></textarea></p>
<p><input type="submit" id="submitButton" value="Submit" /></p></form>
<script language="Javascript">turkSetAssignmentID();</script>
</body>
</html>
]]></HTMLContent>
<FrameHeight>450</FrameHeight>
</HTMLQuestion>{{end}}
`
func parseQuestionContent(htmlQuestionContent mtwerk.HTMLQuestionContent) (result string, err error) {
// TODO move this elsewhere
bs := make([]byte, 0, MAX_QUESTION_SIZE)
bf := bytes.NewBuffer(bs)
tmpl, err := template.New("foo").Parse(HTMLQuestionTemplate)
if err != nil {
return
}
err = tmpl.ExecuteTemplate(bf, "T", htmlQuestionContent)
if err != nil {
return
}
bts, err := ioutil.ReadAll(bf)
if err != nil {
return
}
result = string(bts)
return
}
func CreateTranslationHIT(a *anaconda.TwitterApi, auth *aws.Auth, tweet anaconda.Tweet, title string, description string, displayName string, rewardAmount string, assignmentDuration int, lifetime time.Duration, keywords []string) (*mtwerk.CreateHITResponse, error) {
const rewardCurrencyCode = "USD" // This is the only one supported for now by Amazon, anyway
const responseGroup = "Minimal"
const autoApprovalDelay = 0 // auto-approve immediately
// Get the embedded tweet to render
embed, err := a.GetOEmbedId(tweet.Id, nil)
if err != nil {
return nil, err
}
hq := mtwerk.HTMLQuestionContent{tweet.IdStr, title, description, "http://www.emojidick.com/emoji.png", tweet, embed}
questionString, err := parseQuestionContent(hq)
if err != nil {
return nil, err
}
resp, err := mtwerk.CreateHIT(auth, title, description, questionString, rewardAmount, rewardCurrencyCode, assignmentDuration, lifetime, keywords, autoApprovalDelay, tweet.IdStr, tweet.IdStr, responseGroup)
return resp, err
}
func ScheduleTranslatedTweet(tweet anaconda.Tweet) {
title := `Translate tweet into emoji`
description := `Pick the emoji that you feel would be the best translation of this tweet.`
displayName := "How would you translate this tweet?"
hit, err := CreateTranslationHIT(twitterBot, awsAuth, tweet, title, description, displayName, REWARD, ASSIGNMENT_DURATION, LIFETIME, HIT_KEYWORDS)
if err != nil {
log.Printf("ERROR creating translation HIT: %s", err)
}
// Check every minute for the completed task
hitId := hit.HIT.HITId
ticker := time.NewTicker(time.Minute)
timeout := time.After(LIFETIME)
for {
select {
case <-ticker.C:
log.Printf("Fetching assignments for HITOperation")
result, err := mtwerk.GetAssignmentsForHITOperation(awsAuth, hitId)
if err != nil {
log.Printf("ERROR fetching assignments for HITOperation %s: %s", hitId, err)
}
answerText, err := result.GetAnswerText()
if err != nil {
log.Printf("ERROR getting text of response for HITOperation %s: %s", hitId, err)
}
if answerText == "" {
log.Printf("Assignments yielded an empty response")
continue
}
log.Printf("Received answerText %s", answerText)
v := url.Values{}
v.Set("in_reply_to_status_id", tweet.IdStr)
_, err = twitterBot.PostTweet(fmt.Sprintf("%s %s", tweet.User.ScreenName, answerText), v)
if err != nil {
log.Printf("ERROR updating tweet %s: %s", tweet.IdStr, err)
}
case <-timeout:
log.Printf("Timing out :( (tweet: %d)", tweet.Id)
return
}
}
}
func main() {
if tmp, err := aws.EnvAuth(); err != nil {
panic(err)
} else {
awsAuth = &tmp
}
anaconda.SetConsumerKey(string(TWITTER_CONSUMER_KEY))
anaconda.SetConsumerSecret(string(TWITTER_CONSUMER_SECRET))
twitterBot = anaconda.NewTwitterApi(string(TWITTER_ACCESS_TOKEN), string(TWITTER_ACCESS_TOKEN_SECRET))
me, err := twitterBot.GetSelf(nil)
if err != nil {
panic(err)
}
log.Printf("Emojibot posting as %s (%d)", me.ScreenName, me.Id)
for {
tweets, _ := twitterBot.GetHomeTimeline(nil)
for _, tweet := range tweets {
// Don't reply to own tweets
// Only reply to tweets within the last 23 hours
// Amazon guarantees idempotency of requests with the same unique identifier for 24 hours
if t, _ := tweet.CreatedAtTime(); time.Now().Add(-23*time.Hour).Before(t) && tweet.User.Id != me.Id {
log.Printf("Scheduling response to tweet %s", tweet.Text)
go ScheduleTranslatedTweet(tweet)
}
log.Printf("Ignoring tweet %s", tweet.Text)
}
// TODO fix pagination
log.Printf("Finished scanning all tweets")
<-time.After(10 * time.Minute)
}
}