// GetPlugin attempts to find a plugin and route for the given msg input if none
// can be found, it checks the database for the last route used and gets the
// plugin for that. If there is no previously used plugin, we return
// errMissingPlugin. The bool value return indicates whether this plugin is
// different from the last plugin used by the user.
func GetPlugin(db *sqlx.DB, m *dt.Msg) (p *dt.Plugin, route string, directroute,
followup bool, err error) {
// Iterate through all intents to see if any plugin has been registered
// for the route
for _, i := range m.StructuredInput.Intents {
route = "I_" + strings.ToLower(i)
log.Debug("searching for route", route)
if p = RegPlugins.Get(route); p != nil {
// Found route. Return it
return p, route, true, false, nil
}
}
log.Debug("getting last plugin route")
prevPlugin, prevRoute, err := m.GetLastPlugin(db)
if err != nil && err != sql.ErrNoRows {
return nil, "", false, false, err
}
if len(prevPlugin) > 0 {
log.Debugf("found user's last plugin route: %s - %s\n",
prevPlugin, prevRoute)
}
// Iterate over all command/object pairs and see if any plugin has been
// registered for the resulting route
eng := porter2.Stemmer
for _, c := range m.StructuredInput.Commands {
c = strings.ToLower(eng.Stem(c))
for _, o := range m.StructuredInput.Objects {
o = strings.ToLower(eng.Stem(o))
route := "CO_" + c + "_" + o
log.Debug("searching for route", route)
if p = RegPlugins.Get(route); p != nil {
// Found route. Return it
followup := prevPlugin == p.Config.Name
return p, route, true, followup, nil
}
}
}
// The user input didn't match any plugins. Let's see if the previous
// route does
if prevRoute != "" {
if p = RegPlugins.Get(prevRoute); p != nil {
// Prev route matches a pkg! Return it
return p, prevRoute, false, true, nil
}
}
// Sadly, if we've reached this point, we are at a loss.
log.Debug("could not match user input to any plugin")
return nil, "", false, false, errMissingPlugin
}
// NewMsg builds a message struct with Tokens, Stems, and a Structured Input.
func NewMsg(u *dt.User, cmd string) (*dt.Msg, error) {
tokens := TokenizeSentence(cmd)
stems := StemTokens(tokens)
si := ner.classifyTokens(tokens)
// Get the intents as determined by each plugin
for pluginID, c := range bClassifiers {
scores, idx, _ := c.ProbScores(stems)
log.Debug("intent score", pluginIntents[pluginID][idx],
scores[idx])
if scores[idx] > 0.7 {
si.Intents = append(si.Intents,
string(pluginIntents[pluginID][idx]))
}
}
m := &dt.Msg{
User: u,
Sentence: cmd,
Tokens: tokens,
Stems: stems,
StructuredInput: si,
}
if err := saveContext(db, m); err != nil {
return nil, err
}
if err := addContext(db, m); err != nil {
return nil, err
}
return m, nil
}
// isAdmin ensures that the current user is an admin. We trust the scopes
// presented by the client because they're validated through HMAC in
// isLoggedIn().
func isAdmin(w http.ResponseWriter, r *http.Request) bool {
log.Debug("validating admin")
cookie, err := r.Cookie("scopes")
if err == http.ErrNoCookie {
writeErrorAuth(w, err)
return false
}
if err != nil {
writeErrorInternal(w, err)
return false
}
scopes := strings.Fields(cookie.Value)
for _, scope := range scopes {
if scope == "admin" {
// Confirm the admin permission has not been deleted
// since the cookie was created by retrieving the
// current value from the DB.
cookie, err = r.Cookie("id")
if err == http.ErrNoCookie {
writeErrorAuth(w, err)
return false
}
if err != nil {
writeErrorInternal(w, err)
return false
}
var admin bool
q := `SELECT admin FROM users WHERE id=$1`
if err = db.Get(&admin, q, cookie.Value); err != nil {
writeErrorInternal(w, err)
return false
}
if !admin {
writeErrorAuth(w, errors.New("User is not an admin"))
return false
}
log.Debug("validated admin")
return true
}
}
writeErrorAuth(w, errors.New("user is not an admin"))
return false
}
// parseTime iterates through all known date formats on a normalized time
// string, using Golang's standard lib to do the heavy lifting.
//
// TODO This is a brute-force, "dumb" method of determining the time format and
// should be improved.
func parseTime(t string) (time.Time, error) {
for _, tf := range timeFormats {
time, err := time.Parse(tf, t)
if err == nil {
log.Debug("timeparse: found format", tf)
return time, nil
}
}
return time.Time{}, ErrInvalidTimeFormat
}
func clonePrivateRepo(name string, errChan chan errMsg, wg *sync.WaitGroup) {
wg.Add(1)
defer wg.Done()
parts := strings.Split(name, "/")
if len(parts) < 2 {
errChan <- errMsg{msg: "", err: errors.New("invalid dependency path: too few parts")}
return
}
// Ensure we don't delete a lower level directory
p := filepath.Join(os.Getenv("GOPATH"), "src")
tmp := filepath.Join(p, name)
if len(tmp)+4 <= len(p) {
errChan <- errMsg{msg: "", err: errors.New("invalid dependency path: too short")}
return
}
if strings.Contains(tmp, "..") {
errChan <- errMsg{msg: name, err: errors.New("invalid dependency path: contains '..'")}
return
}
cmd := fmt.Sprintf("rm -rf %s", tmp)
log.Debug("running:", cmd)
outC, err := exec.
Command("/bin/sh", "-c", cmd).
CombinedOutput()
if err != nil {
tmp = fmt.Sprintf("failed to fetch %s\n%s", name, string(outC))
errChan <- errMsg{msg: tmp, err: err}
return
}
name = strings.Join(parts[1:], "/")
cmd = fmt.Sprintf("git clone [email protected]:%s.git %s", name, tmp)
log.Debug("running:", cmd)
outC, err = exec.
Command("/bin/sh", "-c", cmd).
CombinedOutput()
if err != nil {
tmp = fmt.Sprintf("failed to fetch %s\n%s", name, string(outC))
errChan <- errMsg{msg: tmp, err: err}
}
}
// compileAssets compresses and merges assets from Abot core and all plugins on
// boot. In development, this step is repeated on each server HTTP request prior
// to serving any assets.
func compileAssets() error {
p := filepath.Join("cmd", "compileassets.sh")
outC, err := exec.
Command("/bin/sh", "-c", p).
CombinedOutput()
if err != nil {
log.Debug(string(outC))
return err
}
return nil
}
func TestParse(t *testing.T) {
n := time.Now()
// _, zone := n.Zone()
n.Add(-6 * time.Hour)
tests := map[string][]time.Time{
"2pm": []time.Time{time.Date(n.Year(), n.Month(), n.Day(), 14, 0, 0, 0, n.Location())},
"2 am": []time.Time{time.Date(n.Year(), n.Month(), n.Day(), 2, 0, 0, 0, n.Location())},
"at 2 p.m.": []time.Time{time.Date(n.Year(), n.Month(), n.Day(), 14, 0, 0, 0, n.Location())},
"2pm tomorrow": []time.Time{time.Date(n.Year(), n.Month(), n.Day()+1, 14, 0, 0, 0, n.Location())},
"2am yesterday": []time.Time{time.Date(n.Year(), n.Month(), n.Day()-1, 2, 0, 0, 0, n.Location())},
"2 days ago": []time.Time{time.Date(n.Year(), n.Month(), n.Day()-2, 9, 0, 0, 0, n.Location())},
"in 3 days from now": []time.Time{time.Date(n.Year(), n.Month(), n.Day()+3, 9, 0, 0, 0, n.Location())},
"1 week": []time.Time{time.Date(n.Year(), n.Month(), n.Day()+7, 9, 0, 0, 0, n.Location())},
"1 week ago": []time.Time{time.Date(n.Year(), n.Month(), n.Day()-7, 9, 0, 0, 0, n.Location())},
"in a year": []time.Time{time.Date(n.Year()+1, n.Month(), n.Day(), 9, 0, 0, 0, n.Location())},
"next year": []time.Time{time.Date(n.Year()+1, n.Month(), n.Day(), 9, 0, 0, 0, n.Location())},
"in 4 weeks": []time.Time{time.Date(n.Year(), n.Month(), n.Day()+28, 9, 0, 0, 0, n.Location())},
"later today": []time.Time{time.Date(n.Year(), n.Month(), n.Day(), n.Hour()+6, n.Minute(), 0, 0, n.Location())},
"a few hours": []time.Time{time.Date(n.Year(), n.Month(), n.Day(), n.Hour()+2, n.Minute(), 0, 0, n.Location())},
"in 30 mins": []time.Time{time.Date(n.Year(), n.Month(), n.Day(), n.Hour(), n.Minute()+30, 0, 0, n.Location())},
"in 2 hours": []time.Time{time.Date(n.Year(), n.Month(), n.Day(), n.Hour()+2, n.Minute(), 0, 0, n.Location())},
"invalid time": []time.Time{},
"May 2050": []time.Time{time.Date(2050, 5, 1, 0, 0, 0, 0, n.Location())},
"June 26 2050": []time.Time{time.Date(2050, 6, 26, 0, 0, 0, 0, n.Location())},
"June 26th 2050": []time.Time{time.Date(2050, 6, 26, 0, 0, 0, 0, n.Location())},
"at 2 tomorrow": []time.Time{
time.Date(n.Year(), n.Month(), n.Day()+1, 2, 0, 0, 0, n.Location()),
time.Date(n.Year(), n.Month(), n.Day()+1, 14, 0, 0, 0, n.Location()),
},
"now": []time.Time{time.Date(n.Year(), n.Month(), n.Day(), n.Hour(), n.Minute(), n.Second(), n.Nanosecond(), n.Location())},
"noon": []time.Time{time.Date(n.Year(), n.Month(), n.Day(), 12, 0, 0, 0, n.Location())},
/*
"2 days ago at 6PM": []time.Time{time.Date(n.Year(), n.Month(), n.Day()-2, 18, 0, 0, 0, n.Location())},
"12PM EST": []time.Time{time.Date(n.Year(), n.Month(), n.Day(), 12-zone, n.Minute(), 0, 0, n.Location())},
*/
}
for test, exp := range tests {
log.Debug("test:", test)
res := Parse(test)
if len(res) == 0 {
if len(exp) == 0 {
continue
}
t.Fatalf("expected %q, got none", exp)
}
if len(exp) == 0 && len(res) > 0 {
t.Fatalf("expected none, but got %q", res)
}
if !exp[0].Equal(res[0]) && exp[0].Sub(res[0]) > 2*time.Minute {
t.Fatalf("expected %q, got %q", exp, res)
}
}
}
// ConnectDB opens a connection to the database. The name is the name of the
// database to connect to. If empty, it defaults to the current directory's
// name.
func ConnectDB(name string) (*sqlx.DB, error) {
if len(name) == 0 {
dir, err := os.Getwd()
if err != nil {
return nil, err
}
name = filepath.Base(dir)
}
dbConnStr := DBConnectionString(name)
log.Debug("connecting to db")
return sqlx.Connect("postgres", dbConnStr)
}
// isValidCSRF ensures that any forms posted to Abot are protected against
// Cross-Site Request Forgery. Without this function, Abot would be vulnerable
// to the attack because tokens are stored client-side in cookies.
func isValidCSRF(w http.ResponseWriter, r *http.Request) bool {
// TODO look into other session-based temporary storage systems for
// these csrf tokens to prevent hitting the database. Whatever is
// selected must *not* introduce an external (system) dependency like
// memcached/Redis. Bolt might be an option.
log.Debug("validating csrf")
var label string
q := `SELECT label FROM sessions
WHERE userid=$1 AND label='csrfToken' AND token=$2`
cookie, err := r.Cookie("id")
if err == http.ErrNoCookie {
writeErrorAuth(w, err)
return false
}
if err != nil {
writeErrorInternal(w, err)
return false
}
uid := cookie.Value
cookie, err = r.Cookie("csrfToken")
if err == http.ErrNoCookie {
writeErrorAuth(w, err)
return false
}
if err != nil {
writeErrorInternal(w, err)
return false
}
err = db.Get(&label, q, uid, cookie.Value)
if err == sql.ErrNoRows {
writeErrorAuth(w, errors.New("invalid CSRF token"))
return false
}
if err != nil {
writeErrorInternal(w, err)
return false
}
log.Debug("validated csrf")
return true
}
// ExtractCurrency returns an int64 if a currency is found, and throws an
// error if one isn't.
func ExtractCurrency(s string) (int64, error) {
s = regexCurrency.FindString(s)
if len(s) == 0 {
return 0, ErrNotFound
}
val, err := strconv.ParseFloat(s, 64)
if err != nil {
return 0, err
}
log.Debug("found value", val)
// Convert parsed float into an int64 with precision of 2 decimal places
return int64(val * 100), nil
}
// Positive returns a randomized positive response to a user message.
func Positive() string {
n := rand.Intn(3)
switch n {
case 0:
return "Great!"
case 1:
return "I'm glad to hear that!"
case 2:
return "Great to hear!"
}
log.Debug("positive failed to return a response")
return ""
}
// trainClassifiers trains classifiers for each plugin.
func trainClassifiers() error {
for _, pconf := range PluginsGo {
ss, err := fetchTrainingSentences(pconf.ID, pconf.Name)
if err != nil {
return err
}
// Assemble list of Bayesian classes from all trained intents
// for this plugin. m is used to keep track of the classes
// already taught to each classifier.
m := map[string]struct{}{}
for _, s := range ss {
_, ok := m[s.Intent]
if ok {
continue
}
log.Debug("learning intent", s.Intent)
m[s.Intent] = struct{}{}
pluginIntents[s.PluginID] = append(pluginIntents[s.PluginID],
bayesian.Class(s.Intent))
}
// Build classifier from complete sets of intents
for _, s := range ss {
intents := pluginIntents[s.PluginID]
// Calling bayesian.NewClassifier() with 0 or 1
// classes causes a panic.
if len(intents) == 0 {
break
}
if len(intents) == 1 {
intents = append(intents, bayesian.Class("__no_intent"))
}
c := bayesian.NewClassifier(intents...)
bClassifiers[s.PluginID] = c
}
// With classifiers initialized, train each of them on a
// sentence's stems.
for _, s := range ss {
tokens := TokenizeSentence(s.Sentence)
stems := StemTokens(tokens)
c, exists := bClassifiers[s.PluginID]
if exists {
c.Learn(stems, bayesian.Class(s.Intent))
}
}
}
return nil
}
// GetUser from an HTTP request.
func GetUser(db *sqlx.DB, req *Request) (*User, error) {
u := &User{}
u.FlexID = req.FlexID
u.FlexIDType = req.FlexIDType
if req.UserID == 0 {
if req.FlexID == "" {
return nil, ErrMissingFlexID
}
switch req.FlexIDType {
case FIDTEmail, FIDTPhone, FIDTSession:
// Do nothing
default:
return nil, ErrInvalidFlexIDType
}
log.Debug("searching for user from", req.FlexID, req.FlexIDType)
q := `SELECT userid
FROM userflexids
WHERE flexid=$1 AND flexidtype=$2
ORDER BY createdat DESC`
err := db.Get(&req.UserID, q, req.FlexID, req.FlexIDType)
if err == sql.ErrNoRows {
return u, nil
}
log.Debug("got uid", req.UserID)
if err != nil {
return nil, err
}
}
q := `SELECT id, name, email FROM users WHERE id=$1`
if err := db.Get(u, q, req.UserID); err != nil {
if err == sql.ErrNoRows {
return u, nil
}
return nil, err
}
return u, nil
}
// SuggestedPlace returns a randomized place suggestion useful for recommending
// restaurants, businesses, etc.
func SuggestedPlace(s string) string {
n := rand.Intn(4)
switch n {
case 0:
return "How does this place look? " + s
case 1:
return "How about " + s + "?"
case 2:
return "Have you been here before? " + s
case 3:
return "You could try this: " + s
}
log.Debug("suggestedPlace failed to return a response")
return ""
}
// ConfusedLang returns a randomized response signalling that Abot is confused
// or could not understand the user's request.
func ConfusedLang() string {
n := rand.Intn(4)
switch n {
case 0:
return "I'm not sure I understand you."
case 1:
return "I'm sorry, I don't understand that."
case 2:
return "Uh, what are you telling me to do?"
case 3:
return "What should I do?"
}
log.Debug("confused failed to return a response")
return ""
}
// NiceMeetingYou is used to greet the user and request signup during an
// onboarding process.
func NiceMeetingYou() string {
n := rand.Intn(3)
switch n {
case 0:
return "It's nice to meet you. If we're going to work " +
"together, can you sign up for me here? "
case 1:
return "Nice meeting you. Before we take this further, can " +
"you sign up for me here? "
case 2:
return "Great to meet you! Can you sign up for me here to " +
"get started? "
}
log.Debug("nicemeetingyou failed to return a response")
return ""
}
// Welcome returns a randomized "you're welcome" response to a user message.
func Welcome() string {
n := rand.Intn(5)
switch n {
case 0:
return "You're welcome!"
case 1:
return "Sure thing!"
case 2:
return "I'm happy to help!"
case 3:
return "My pleasure."
case 4:
return "Sure."
}
log.Debug("welcome failed to return a response")
return ""
}
func sendEventsTick(evtChan chan *dt.ScheduledEvent, t time.Time) {
// Listen for events that need to be sent.
go func(chan *dt.ScheduledEvent) {
q := `UPDATE scheduledevents SET sent=TRUE WHERE id=$1`
select {
case evt := <-evtChan:
log.Debug("received event")
if smsConn == nil {
log.Info("failed to send scheduled event (missing SMS driver). will retry.")
return
}
// Send event. On error, event will be retried next
// minute.
if err := evt.Send(smsConn); err != nil {
log.Info("failed to send scheduled event", err)
return
}
// Update event as sent
if _, err := db.Exec(q, evt.ID); err != nil {
log.Info("failed to update scheduled event as sent",
err)
return
}
}
}(evtChan)
q := `SELECT id, content, flexid, flexidtype
FROM scheduledevents
WHERE sent=false AND sendat<=$1`
evts := []*dt.ScheduledEvent{}
if err := db.Select(&evts, q, time.Now()); err != nil {
log.Info("failed to queue scheduled event", err)
return
}
for _, evt := range evts {
// Queue the event for sending
evtChan <- evt
}
}
// Greeting returns a randomized greeting.
func Greeting(r *rand.Rand, name string) string {
var n int
if len(name) == 0 {
n = r.Intn(3)
switch n {
case 0:
return fmt.Sprintf("Hi, %s.", name)
case 1:
return fmt.Sprintf("Hello, %s.", name)
case 2:
return fmt.Sprintf("Hi there, %s.", name)
}
} else {
n = r.Intn(3)
switch n {
case 0:
return "Hi. How can I help you?"
case 1:
return "Hello. What can I do for you?"
}
}
log.Debug("greeting failed to return a response")
return ""
}
// ParseFromTime parses a natural language string to determine most likely times
// based on a set time "context." The time context changes the meaning of words
// like "this Tuesday," "next Tuesday," etc.
func ParseFromTime(t time.Time, nlTime string) []time.Time {
r := strings.NewReplacer(
".", "",
",", "",
"(", "",
")", "",
"'", "",
)
nlTime = r.Replace(nlTime)
nlTime = strings.ToLower(nlTime)
r = strings.NewReplacer(
"at ", "",
"time", "",
"oclock", "",
"am", "AM",
"pm", "PM",
" am", "AM",
" pm", "PM",
// 1st, 2nd, 3rd, 4th, 5th, etc.
"1st", "1",
"2nd", "2",
"3rd", "3",
"th", "",
"21st", "21",
"22nd", "22",
"23rd", "23",
"31st", "31",
)
nlTime = r.Replace(nlTime)
nlTime = strings.Title(nlTime)
if nlTime == "Now" {
return []time.Time{time.Now()}
}
st := strings.Fields(nlTime)
transform := struct {
Transform int
Type timeTransform
Multiplier int
}{
Multiplier: 1,
}
stFull := ""
var closeTime bool
var idxRel int
var loc *time.Location
for i := range st {
// Normalize days
switch st[i] {
case "Monday":
st[i] = "Mon"
transform.Type = transformDay
case "Tuesday", "Tues":
st[i] = "Tue"
transform.Type = transformDay
case "Wednesday":
st[i] = "Wed"
transform.Type = transformDay
case "Thursday", "Thur", "Thurs":
st[i] = "Thu"
transform.Type = transformDay
case "Friday":
st[i] = "Fri"
transform.Type = transformDay
case "Saturday":
st[i] = "Sat"
transform.Type = transformDay
case "Sunday":
st[i] = "Sun"
transform.Type = transformDay
// Normalize months
case "January":
st[i] = "Jan"
transform.Type = transformMonth
case "February":
st[i] = "Feb"
transform.Type = transformMonth
case "March":
st[i] = "Mar"
transform.Type = transformMonth
case "April":
st[i] = "Apr"
transform.Type = transformMonth
case "May":
// No translation needed for May
transform.Type = transformMonth
case "June":
st[i] = "Jun"
transform.Type = transformMonth
case "July":
st[i] = "Jul"
transform.Type = transformMonth
case "August":
st[i] = "Aug"
transform.Type = transformMonth
case "September", "Sept":
st[i] = "Sep"
transform.Type = transformMonth
case "October":
st[i] = "Oct"
transform.Type = transformMonth
case "November":
st[i] = "Nov"
transform.Type = transformMonth
case "December":
st[i] = "Dec"
transform.Type = transformMonth
// If non-deterministic timezone information is provided,
// e.g. ET or Eastern rather than EST, then load the location.
// Daylight Savings will be determined on parsing
case "Pacific", "PT":
st[i] = ""
loc = loadLocation("America/Los_Angeles")
case "Mountain", "MT":
st[i] = ""
loc = loadLocation("America/Denver")
case "Central", "CT":
st[i] = ""
loc = loadLocation("America/Chicago")
case "Eastern", "ET":
st[i] = ""
loc = loadLocation("America/New_York")
// TODO Add the remaining timezones
// Handle relative times. This currently does not handle
// complex cases like "in 3 months and 2 days"
case "Yesterday":
st[i] = ""
transform.Type = transformDay
transform.Transform = 1
transform.Multiplier = -1
case "Tomorrow":
st[i] = ""
transform.Transform = 1
transform.Type = transformDay
case "Today":
st[i] = ""
closeTime = true
transform.Type = transformHour
case "Ago", "Last":
st[i] = ""
transform.Transform = 1
transform.Multiplier *= -1
// e.g. "In an hour"
case "Next", "From", "Now", "In":
st[i] = ""
transform.Transform = 1
case "Later":
st[i] = ""
transform.Transform = 6
case "Hour", "Hours":
st[i] = ""
idxRel = i
closeTime = true
transform.Type = transformHour
case "Few", "Couple":
st[i] = ""
transform.Transform = 2
case "Min", "Mins", "Minute", "Minutes":
st[i] = ""
idxRel = i
closeTime = true
transform.Type = transformMinute
case "Day", "Days":
st[i] = ""
idxRel = i
transform.Type = transformDay
case "Week", "Weeks":
st[i] = ""
idxRel = i
transform.Type = transformDay
transform.Multiplier = 7
case "Month", "Months":
st[i] = ""
idxRel = i
transform.Type = transformMonth
case "Year", "Years":
st[i] = ""
idxRel = i
transform.Type = transformYear
// Remove unnecessary but common expressions like "at", "time",
// "oclock".
case "At", "Time", "Oclock", "This", "The":
st[i] = ""
case "Noon":
st[i] = "12PM"
case "Supper", "Dinner":
st[i] = "6PM"
}
if len(st[i]) > 0 {
stFull += st[i] + " "
}
}
normalized := strings.TrimRight(stFull, " ")
var timeEmpty bool
ts := []time.Time{}
tme, err := parseTime(normalized)
if err != nil {
// Set the hour to 9am
timeEmpty = true
tme = time.Now().Round(time.Hour)
val := 9 - tme.Hour()
tme = tme.Add(time.Duration(val) * time.Hour)
}
if closeTime {
tme = time.Now().Round(time.Minute)
}
ts = append(ts, tme)
// TODO make more efficient. Handle in switch?
tloc := timeLocation{loc: loc}
ctx := &TimeContext{ampm: ampmNoTime, tz: tloc}
if strings.Contains(normalized, "AM") {
ctx.ampm = amTime
}
if strings.Contains(normalized, "UTC") {
ctx.tz.utc = true
}
for _, ti := range ts {
ctx = updateContext(ctx, ti, false)
}
// Ensure dates are reasonable even in the absence of information.
// e.g. 2AM should parse to the current year, not 0000
ctx = completeContext(ctx, t)
// Loop through a second time to apply the discovered context to each
// time. Note that this doesn't support context switching,
// e.g. "5PM CST or PST" or "5PM EST or 6PM PST", which is rare in
// practice. Future versions may be adapted to support it.
if ctx.ampm == ampmNoTime {
halfLen := len(ts)
// Double size of times for AM/PM
ts = append(ts, ts...)
for i := range ts {
var hour int
t := ts[i]
if i < halfLen {
hour = t.Hour()
} else {
hour = t.Hour() + 12
}
ts[i] = time.Date(ctx.year,
ctx.month,
ctx.day,
hour,
t.Minute(),
t.Second(),
t.Nanosecond(),
ctx.tz.loc)
}
} else {
for i := range ts {
t := ts[i]
ts[i] = time.Date(ctx.year,
ctx.month,
ctx.day,
t.Hour(),
t.Minute(),
t.Second(),
t.Nanosecond(),
ctx.tz.loc)
}
}
// If there's no relative transform, we're done.
if transform.Type == transformInvalid {
if timeEmpty {
return []time.Time{}
}
if idxRel == 0 {
return ts
}
}
// Check our idxRel term for the word that preceeds it. If that's a
// number, e.g. 2 days, then that number is our Transform. Note that
// this doesn't handle fractional modifiers, like 2.5 days.
if idxRel > 0 {
val, err := strconv.Atoi(st[idxRel-1])
if err == nil {
transform.Transform = val
}
}
// Apply the transform
log.Debugf("timeparse: normalized %q. %+v\n", normalized, transform)
for i := range ts {
switch transform.Type {
case transformYear:
ts[i] = ts[i].AddDate(transform.Transform*transform.Multiplier, 0, 0)
case transformMonth:
ts[i] = ts[i].AddDate(0, transform.Multiplier*transform.Transform, 0)
case transformDay:
ts[i] = ts[i].AddDate(0, 0, transform.Multiplier*transform.Transform)
case transformHour:
ts[i] = ts[i].Add(time.Duration(transform.Transform*transform.Multiplier) * time.Hour)
case transformMinute:
ts[i] = ts[i].Add(time.Duration(transform.Transform*transform.Multiplier) * time.Minute)
}
}
log.Debug("timeparse: parsed times", ts)
return ts
}
// TokenizeSentence returns a sentence broken into tokens. Tokens are individual
// words as well as punctuation. For example, "Hi! How are you?" becomes
// []string{"Hi", "!", "How", "are", "you", "?"}. This also expands
// contractions into the words they represent, e.g. "How're you?" becomes
// []string{"How", "'", "are", "you", "?"}.
func TokenizeSentence(sent string) []string {
tokens := []string{}
for _, w := range strings.Fields(sent) {
found := []int{}
for i, r := range w {
switch r {
case '\'', '"', ':', ';', '!', '?':
found = append(found, i)
// Handle case of currencies and fractional percents.
case '.', ',':
if i+1 < len(w) {
switch w[i+1] {
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
continue
}
}
found = append(found, i)
i++
}
}
if len(found) == 0 {
tokens = append(tokens, w)
continue
}
for i, j := range found {
// If the token marker is not the first character in the
// sentence, then include all characters leading up to
// the prior found token.
if j > 0 {
if i == 0 {
tokens = append(tokens, w[:j])
} else if i-1 < len(found) {
// Handle case where multiple tokens are
// found in the same word.
tokens = append(tokens, w[found[i-1]+1:j])
}
}
// Append the token marker itself
tokens = append(tokens, string(w[j]))
// If we're on the last token marker, append all
// remaining parts of the word.
if i+1 == len(found) {
tokens = append(tokens, w[j+1:])
}
}
}
// Expand contractions. This isn't perfect and doesn't need to be to
// fulfill its purpose, which is fundamentally making it easier to find
// times in a sentence containing contractions.
for i, t := range tokens {
switch t {
case "s":
tokens[i] = "is"
case "re":
tokens[i] = "are"
case "m":
tokens[i] = "am"
case "t":
tokens[i] = "not"
case "ve":
tokens[i] = "have"
case "ll":
tokens[i] = "will"
case "d":
tokens[i] = "would"
}
}
log.Debug("found tokens", tokens)
return tokens
}
// Parse a string to return a fully-validated U.S. address.
func Parse(s string) (*dt.Address, error) {
s = regexAddress.FindString(s)
if len(s) == 0 {
log.Debug("missing address")
return nil, ErrInvalidAddress
}
log.Debug("address", s)
tmp := regexZip.FindStringIndex(s)
var zip string
if tmp != nil {
zip = s[tmp[0]:tmp[1]]
s = s[:tmp[0]]
} else {
log.Debug("no zip found")
}
tmp2 := regexState.FindStringIndex(s)
if tmp2 == nil && tmp == nil {
log.Debug("no state found AND no zip found")
return &dt.Address{}, ErrInvalidAddress
}
var city, state string
if tmp2 != nil {
state = s[tmp2[0]:tmp2[1]]
s = s[:tmp2[0]]
state = strings.Trim(state, ", \n")
if len(state) > 2 {
state = strings.ToLower(state)
state = states[state]
}
tmp = regexCity.FindStringIndex(s)
if tmp == nil {
log.Debug("no city found")
return &dt.Address{}, ErrInvalidAddress
}
city = s[tmp[0]:tmp[1]]
s = s[:tmp[0]]
} else {
log.Debug("no state found")
}
tmp = regexApartment.FindStringIndex(s)
var apartment string
if tmp != nil {
apartment = s[tmp[0]:tmp[1]]
s2 := s[:tmp[0]]
if len(s2) == 0 {
apartment = ""
} else {
s = s2
}
} else {
log.Debug("no apartment found")
}
tmp = regexStreet.FindStringIndex(s)
if tmp == nil {
log.Debug(s)
log.Debug("no street found")
return &dt.Address{}, ErrInvalidAddress
}
street := s[tmp[0]:tmp[1]]
return &dt.Address{
Line1: strings.Trim(street, " \n,"),
Line2: strings.Trim(apartment, " \n,"),
City: strings.Trim(city, " \n,"),
State: strings.Trim(state, " \n,"),
Zip: strings.Trim(zip, " \n,"),
Country: "USA",
}, nil
}
// ProcessText is Abot's core logic. This function processes a user's message,
// routes it to the correct plugin, and handles edge cases like offensive
// language before returning a response to the user. Any user-presentable error
// is returned in the string. Errors returned from this function are not for the
// user, so they are handled by Abot explicitly on this function's return
// (logging, notifying admins, etc.).
func ProcessText(r *http.Request) (ret string, err error) {
// Process message
in, err := preprocess(r)
if err != nil {
return "", err
}
log.Debug("processed input into message...")
log.Debug("commands:", in.StructuredInput.Commands)
log.Debug(" objects:", in.StructuredInput.Objects)
log.Debug(" intents:", in.StructuredInput.Intents)
log.Debug(" people:", in.StructuredInput.People)
log.Debug(" times:", in.StructuredInput.Times)
plugin, route, directRoute, followup, pluginErr := GetPlugin(db, in)
if pluginErr != nil && pluginErr != errMissingPlugin {
return "", pluginErr
}
in.Route = route
in.Plugin = plugin
if err = in.Save(db); err != nil {
return "", err
}
sendPostProcessingEvent(in)
// Determine appropriate response
var smAnswered bool
resp := &dt.Msg{}
resp.AbotSent = true
resp.User = in.User
resp.Sentence = RespondWithOffense(in)
if len(resp.Sentence) > 0 {
goto saveAndReturn
}
resp.Sentence = RespondWithHelp(in)
if len(resp.Sentence) > 0 {
goto saveAndReturn
}
resp.Sentence = RespondWithNicety(in)
if len(resp.Sentence) > 0 {
goto saveAndReturn
}
if pluginErr != errMissingPlugin {
resp.Sentence, smAnswered = dt.CallPlugin(plugin, in, followup)
}
if len(resp.Sentence) == 0 {
resp.Sentence = RespondWithHelpConfused(in)
in.NeedsTraining = true
if err = in.Update(db); err != nil {
return "", err
}
} else {
state := plugin.GetMemory(in, dt.StateKey).Int64()
if plugin != nil && state == 0 && !directRoute {
in.NeedsTraining = true
if !smAnswered {
resp.Sentence = RespondWithHelpConfused(in)
if err = in.Update(db); err != nil {
return "", err
}
}
}
}
if plugin != nil {
resp.Plugin = plugin
}
saveAndReturn:
sendPreResponseEvent(in, &resp.Sentence)
if err = resp.Save(db); err != nil {
return "", err
}
return resp.Sentence, nil
}
func embedPluginConfs(plugins *core.PluginJSON, l *log.Logger) {
log.Debug("embedding plugin confs")
// Open plugins.go file for writing
fi, err := os.OpenFile("plugins.go", os.O_WRONLY|os.O_APPEND, 0666)
if err != nil {
l.Fatal(err)
}
defer func() {
if err = fi.Close(); err != nil {
l.Fatal(err)
}
}()
p := os.Getenv("GOPATH")
tokenizedPath := strings.Split(p, string(os.PathListSeparator))
// Insert plugin.json text as comments
s := "\n\n/*\n"
for u := range plugins.Dependencies {
s += u + "\n"
log.Debug("reading file", p)
p = filepath.Join(tokenizedPath[0], "src", u, "plugin.json")
fi2, err2 := os.Open(p)
if err2 != nil {
l.Fatal(err2)
}
scn := bufio.NewScanner(fi2)
var tmp string
for scn.Scan() {
line := scn.Text() + "\n"
s += line
tmp += line
}
if err2 = scn.Err(); err2 != nil {
l.Fatal(err2)
}
if err2 = fi2.Close(); err2 != nil {
l.Fatal(err2)
}
var plg struct{ Name string }
if err2 = json.Unmarshal([]byte(tmp), &plg); err2 != nil {
l.Fatal(err2)
}
// Fetch remote plugin IDs to be included in the plugin confs
plg.Name = url.QueryEscape(plg.Name)
ul := os.Getenv("ITSABOT_URL") + "/api/plugins/by_name/" + plg.Name
req, err2 := http.NewRequest("GET", ul, nil)
if err2 != nil {
l.Fatal(err2)
}
client := &http.Client{Timeout: 10 * time.Second}
resp, err2 := client.Do(req)
if err2 != nil {
l.Fatal(err2)
}
var data struct{ ID uint64 }
if err2 := json.NewDecoder(resp.Body).Decode(&data); err2 != nil {
l.Fatal(err2)
}
id := strconv.FormatUint(data.ID, 10)
// Remove closing characters to insert additional ID data
s = s[:len(s)-3]
s += ",\n\t\"ID\": " + id + "\n}\n"
}
s += "*/"
_, err = fi.WriteString(s)
if err != nil {
l.Fatal(err)
}
}
// LoadEnvVars from abot.env into memory
func LoadEnvVars() error {
if envLoaded {
return nil
}
if len(os.Getenv("ABOT_PATH")) == 0 {
p := filepath.Join(os.Getenv("GOPATH"), "src", "github.com",
"itsabot", "abot")
log.Debug("ABOT_PATH not set. defaulting to", p)
if err := os.Setenv("ABOT_PATH", p); err != nil {
return err
}
}
if len(os.Getenv("ITSABOT_URL")) == 0 {
log.Debug("ITSABOT_URL not set, using https://www.itsabot.org")
err := os.Setenv("ITSABOT_URL", "https://www.itsabot.org")
if err != nil {
return err
}
}
p := filepath.Join("abot.env")
fi, err := os.Open(p)
if os.IsNotExist(err) {
// Assume the user has loaded their env variables into their
// path
return nil
}
if err != nil {
return err
}
defer func() {
if err = fi.Close(); err != nil {
log.Info("failed to close file")
}
}()
scn := bufio.NewScanner(fi)
for scn.Scan() {
line := scn.Text()
fields := strings.SplitN(line, "=", 2)
if len(fields) != 2 {
continue
}
key := strings.TrimSpace(fields[0])
if len(key) == 0 {
continue
}
if len(os.Getenv(fields[0])) > 0 {
continue
}
val := strings.TrimSpace(fields[1])
if len(val) >= 2 {
if val[0] == '"' || val[0] == '\'' {
val = val[1 : len(val)-1]
}
}
if err = os.Setenv(key, val); err != nil {
return err
}
}
if err = scn.Err(); err != nil {
return err
}
envLoaded = true
return nil
}
// ExtractAddress will return an address from a user's message, whether it's a
// labeled address (e.g. "home", "office"), or a full U.S. address (e.g. 100
// Penn St., CA 90000)
func ExtractAddress(db *sqlx.DB, u *dt.User, s string) (*dt.Address, bool, error) {
addr, err := address.Parse(s)
if err != nil {
return nil, false, err
}
type addr2S struct {
XMLName xml.Name `xml:"Address"`
ID string `xml:"ID,attr"`
FirmName string
Address1 string
Address2 string
City string
State string
Zip5 string
Zip4 string
}
addr2Stmp := addr2S{
ID: "0",
Address1: addr.Line2,
Address2: addr.Line1,
City: addr.City,
State: addr.State,
Zip5: addr.Zip5,
Zip4: addr.Zip4,
}
if len(addr.Zip) > 0 {
addr2Stmp.Zip5 = addr.Zip[:5]
}
if len(addr.Zip) > 5 {
addr2Stmp.Zip4 = addr.Zip[5:]
}
addrS := struct {
XMLName xml.Name `xml:"AddressValidateRequest"`
USPSUserID string `xml:"USERID,attr"`
Address addr2S
}{
USPSUserID: os.Getenv("USPS_USER_ID"),
Address: addr2Stmp,
}
xmlAddr, err := xml.Marshal(addrS)
if err != nil {
return nil, false, err
}
log.Debug(string(xmlAddr))
ul := "https://secure.shippingapis.com/ShippingAPI.dll?API=Verify&XML="
ul += url.QueryEscape(string(xmlAddr))
response, err := http.Get(ul)
if err != nil {
return nil, false, err
}
contents, err := ioutil.ReadAll(response.Body)
if err != nil {
return nil, false, err
}
if err = response.Body.Close(); err != nil {
return nil, false, err
}
resp := struct {
XMLName xml.Name `xml:"AddressValidateResponse"`
USPSUserID string `xml:"USERID,attr"`
Address addr2S
}{
USPSUserID: os.Getenv("USPS_USER_ID"),
Address: addr2Stmp,
}
if err = xml.Unmarshal(contents, &resp); err != nil {
log.Debug("USPS response", string(contents))
return nil, false, err
}
a := dt.Address{
Name: resp.Address.FirmName,
Line1: resp.Address.Address2,
Line2: resp.Address.Address1,
City: resp.Address.City,
State: resp.Address.State,
Zip5: resp.Address.Zip5,
Zip4: resp.Address.Zip4,
}
if len(resp.Address.Zip4) > 0 {
a.Zip = resp.Address.Zip5 + "-" + resp.Address.Zip4
} else {
a.Zip = resp.Address.Zip5
}
return &a, false, nil
}
// NewServer connects to the database and boots all plugins before returning a
// server connection, database connection, and map of offensive words.
func NewServer() (r *httprouter.Router, err error) {
if err = LoadEnvVars(); err != nil {
return nil, err
}
if len(os.Getenv("ABOT_SECRET")) < 32 && os.Getenv("ABOT_ENV") == "production" {
return nil, errors.New("must set ABOT_SECRET env var in production to >= 32 characters")
}
if db == nil {
db, err = ConnectDB("")
if err != nil {
return nil, fmt.Errorf("could not connect to database: %s", err.Error())
}
}
err = LoadConf()
if err != nil && os.Getenv("ABOT_ENV") != "test" {
log.Info("failed loading conf", err)
return nil, err
}
if err = checkRequiredEnvVars(); err != nil {
return nil, err
}
err = LoadPluginsGo()
if err != nil && os.Getenv("ABOT_ENV") != "test" {
log.Info("failed loading plugins.go", err)
return nil, err
}
ner, err = buildClassifier()
if err != nil {
log.Debug("could not build classifier", err)
}
go func() {
if os.Getenv("ABOT_ENV") != "test" {
log.Info("training classifiers")
}
if err = trainClassifiers(); err != nil {
log.Info("could not train classifiers", err)
}
}()
offensive, err = buildOffensiveMap()
if err != nil {
log.Debug("could not build offensive map", err)
}
var p string
if os.Getenv("ABOT_ENV") == "test" {
p = filepath.Join(os.Getenv("ABOT_PATH"), "base", "assets",
"html", "layout.html")
} else {
p = filepath.Join("assets", "html", "layout.html")
}
if err = loadHTMLTemplate(p); err != nil {
log.Info("failed loading HTML template", err)
return nil, err
}
// Initialize a router with routes
r = newRouter()
// Open a connection to an SMS service
if len(sms.Drivers()) > 0 {
drv := sms.Drivers()[0]
smsConn, err = sms.Open(drv, r)
if err != nil {
log.Info("failed to open sms driver connection", drv,
err)
}
} else {
log.Debug("no sms drivers imported")
}
// Open a connection to an email service
if len(email.Drivers()) > 0 {
drv := email.Drivers()[0]
emailConn, err = email.Open(drv, r)
if err != nil {
log.Info("failed to open email driver connection", drv,
err)
}
} else {
log.Debug("no email drivers imported")
}
// Send any scheduled events on boot and every minute
evtChan := make(chan *dt.ScheduledEvent)
go sendEventsTick(evtChan, time.Now())
go sendEvents(evtChan, 1*time.Minute)
// Update cached analytics data on boot and every 15 minutes
go updateAnalyticsTick(time.Now())
go updateAnalytics(15 * time.Minute)
return r, nil
}
// isLoggedIn determines if the user is currently logged in.
func isLoggedIn(w http.ResponseWriter, r *http.Request) bool {
log.Debug("validating logged in")
w.Header().Set("WWW-Authenticate", bearerAuthKey+" realm=Restricted")
auth := r.Header.Get("Authorization")
l := len(bearerAuthKey)
// Ensure client sent the token
if len(auth) <= l+1 || auth[:l] != bearerAuthKey {
log.Debug("client did not send token")
writeErrorAuth(w, errors.New("missing Bearer token"))
return false
}
// Ensure the token is still valid
cookie, err := r.Cookie("issuedAt")
if err == http.ErrNoCookie {
writeErrorAuth(w, err)
return false
}
if err != nil {
writeErrorInternal(w, err)
return false
}
if len(cookie.Value) == 0 || cookie.Value == "undefined" {
writeErrorAuth(w, errors.New("missing issuedAt"))
return false
}
issuedAt, err := strconv.ParseInt(cookie.Value, 10, 64)
if err != nil {
writeErrorInternal(w, err)
return false
}
t := time.Unix(issuedAt, 0)
if t.Add(72 * time.Hour).Before(time.Now()) {
log.Debug("token expired")
writeErrorAuth(w, errors.New("missing Bearer token"))
return false
}
// Ensure the token has not been tampered with
b, err := base64.StdEncoding.DecodeString(auth[l+1:])
if err != nil {
writeErrorInternal(w, err)
return false
}
cookie, err = r.Cookie("scopes")
if err == http.ErrNoCookie {
writeErrorAuth(w, err)
return false
}
if err != nil {
writeErrorInternal(w, err)
return false
}
scopes := strings.Fields(cookie.Value)
cookie, err = r.Cookie("id")
if err == http.ErrNoCookie {
writeErrorAuth(w, err)
return false
}
if err != nil {
writeErrorInternal(w, err)
return false
}
userID, err := strconv.ParseUint(cookie.Value, 10, 64)
if err != nil {
writeErrorInternal(w, err)
return false
}
cookie, err = r.Cookie("email")
if err == http.ErrNoCookie {
writeErrorAuth(w, err)
return false
}
if err != nil {
writeErrorInternal(w, err)
return false
}
email, err := url.QueryUnescape(cookie.Value)
if err != nil {
writeErrorInternal(w, err)
return false
}
a := Header{
ID: userID,
Email: email,
Scopes: scopes,
IssuedAt: issuedAt,
}
byt, err := json.Marshal(a)
if err != nil {
writeErrorInternal(w, err)
return false
}
known := hmac.New(sha512.New, []byte(os.Getenv("ABOT_SECRET")))
_, err = known.Write(byt)
if err != nil {
writeErrorInternal(w, err)
return false
}
ok := hmac.Equal(known.Sum(nil), b)
if !ok {
log.Info("token tampered for user", userID)
writeErrorAuth(w, errors.New("Bearer token tampered"))
return false
}
log.Debug("validated logged in")
return true
}