// getEmails will fetch the full bodies of all emails listed in the given command.
func getEmails(client *imap.Client, cmd *imap.Command) ([]map[string]interface{}, error) {
var emails []map[string]interface{}
seq := new(imap.SeqSet)
for _, rsp := range cmd.Data {
for _, uid := range rsp.SearchResults() {
seq.AddNum(uid)
}
}
if seq.Empty() {
return emails, nil
}
fCmd, err := imap.Wait(client.UIDFetch(seq, "INTERNALDATE", "BODY[]", "UID", "RFC822.HEADER"))
if err != nil {
return emails, err
}
var email map[string]interface{}
for _, msgData := range fCmd.Data {
msgFields := msgData.MessageInfo().Attrs
email, err = newEmailMessage(msgFields)
if err != nil {
return emails, err
}
emails = append(emails, email)
// mark message as read
fSeq := new(imap.SeqSet)
fSeq.AddNum(imap.AsNumber(msgFields["UID"]))
_, err = imap.Wait(client.UIDStore(fSeq, "+FLAGS", "\\SEEN"))
if err != nil {
return emails, err
}
}
return emails, nil
}
func FetchAllUIDs(c *imap.Client) (uids []uint32, err error) {
maxmessages := 150000
uids = make([]uint32, maxmessages)
set, errS := imap.NewSeqSet("1:*")
if errS != nil {
err = errS
return
}
cmd, errF := c.UIDFetch(set, "RFC822.SIZE")
if errF != nil {
err = errF
return
}
messagenum := uint32(0)
for cmd.InProgress() {
errC := c.Recv(-1)
if errC != nil {
continue
}
for _, rsp := range cmd.Data {
uid := imap.AsNumber(rsp.MessageInfo().Attrs["UID"])
uids[messagenum] = uid
}
cmd.Data = nil
messagenum++
}
uids = uids[:messagenum]
return
}
func FetchMessages(c *imap.Client, uidSet *imap.SeqSet) (fetched []MsgData, err error) {
cmd, errF := c.UIDFetch(uidSet, "RFC822")
if errF != nil {
err = errF
return
}
for cmd.InProgress() {
errC := c.Recv(-1)
if errC != nil {
return
}
for _, rsp := range cmd.Data {
uid := imap.AsNumber(rsp.MessageInfo().Attrs["UID"])
mime := imap.AsBytes(rsp.MessageInfo().Attrs["RFC822"])
msg, errR := mail.ReadMessage(bytes.NewReader(mime))
if errR != nil {
continue
}
if msg != nil {
msgdata := GetMessage(msg, uid)
fetched = append(fetched, msgdata)
}
}
cmd.Data = nil
}
return
}
// findUnreadEmails will run a find the UIDs of any unread emails in the
// mailbox.
func findUnreadEmails(conn *imap.Client) (*imap.Command, error) {
// get headers and UID for UnSeen message in src inbox...
cmd, err := imap.Wait(conn.UIDSearch("UNSEEN"))
if err != nil {
return &imap.Command{}, err
}
return cmd, nil
}
// checkAndStoreMessages will wait for WorkRequests to come acorss the pipe. When it receives a request, it will search
// the given destination inbox for the message. If it is not found, this method will attempt to pull the messages data
// from fetchRequests and then append it to the destination.
func CheckAndAppendMessages(dstConn *imap.Client, storeRequests chan WorkRequest, fetchRequests chan fetchRequest, wg *sync.WaitGroup) {
defer wg.Done()
// noop it every few to keep things alive
timeout := time.NewTicker(NoopMinutes * time.Minute)
done := false
for {
select {
case request, ok := <-storeRequests:
if !ok {
done = true
break
}
// search for in dst
cmd, err := imap.Wait(dstConn.UIDSearch([]imap.Field{"HEADER", request.Header, request.Value}))
if err != nil {
log.Printf("Unable to search for message (%s): %s. skippin!", request.Value, err.Error())
continue
}
results := cmd.Data[0].SearchResults()
// if not found, PULL from SRC and STORE in DST
if len(results) == 0 {
// only fetch if we dont have data already
if len(request.Msg.Body) == 0 {
// build and send fetch request
response := make(chan MessageData)
fr := fetchRequest{MessageId: request.Value, UID: request.UID, Response: response}
fetchRequests <- fr
// grab response from fetchers
request.Msg = <-response
}
if len(request.Msg.Body) == 0 {
log.Printf("No data found for from fetch request (%s). giving up", request.Value)
continue
}
err = AppendMessage(dstConn, request.Msg)
if err != nil {
log.Printf("Problems appending message to dst: %s. quitting.", err.Error())
return
}
}
case <-timeout.C:
imap.Wait(dstConn.Noop())
}
if done {
break
}
}
log.Print("storer complete!")
return
}
// FetchEmails will sit and wait for fetchRequests from the destination workers.
func fetchEmails(conn *imap.Client, requests chan fetchRequest, cache *Cache) {
// noop every few to keep things alive
timeout := time.NewTicker(NoopMinutes * time.Minute)
done := false
for {
select {
case request, ok := <-requests:
if !ok {
done = true
break
}
found := true
// check if the message body is in cache
data, err := cache.Get(request.MessageId)
if err != nil {
found = false
if err != ErrNotFound {
log.Printf("problems pulling message data from cache: %s. Pulling message from src...", err.Error())
}
data = MessageData{}
}
if found {
log.Print("cache success!")
request.Response <- data
continue
}
msgData, err := FetchMessage(conn, request.UID)
if err != nil {
if err == NotFound {
log.Printf("No data found for UID: %d", request.UID)
} else {
log.Printf("Problems fetching message (%s) data: %s. Passing request and quitting.", request.MessageId, err.Error())
requests <- request
return
}
}
request.Response <- msgData
err = cache.Put(request.MessageId, msgData)
if err != nil {
log.Printf("Unable to add message (%s) to cache: %s", request.MessageId, err.Error())
}
case <-timeout.C:
imap.Wait(conn.Noop())
}
if done {
break
}
}
}
func ResetConnection(conn *imap.Client, readOnly bool) error {
// dont check for error because its possible it's already closed.
conn.Close(!readOnly)
_, err := imap.Wait(conn.Select("INBOX", readOnly))
if err != nil {
return err
}
return nil
}
func GetAllMessages(conn *imap.Client) (*imap.Command, error) {
// get headers and UID for ALL message in src inbox...
allMsgs, _ := imap.NewSeqSet("")
allMsgs.Add("1:*")
cmd, err := imap.Wait(conn.Fetch(allMsgs, "RFC822.HEADER", "UID"))
if err != nil {
return &imap.Command{}, err
}
return cmd, nil
}
func SearchUIDs(c *imap.Client, query string) (uids []uint32, err error) {
cmd, err := c.UIDSearch("X-GM-RAW", fmt.Sprint("\"", query, "\""))
for cmd.InProgress() {
c.Recv(-1)
for _, rsp := range cmd.Data {
uids = rsp.SearchResults()
}
cmd.Data = nil
}
return
}
func FetchMessages(c *imap.Client, uidSet *imap.SeqSet) (err error) {
cmd, err := c.UIDFetch(uidSet, "RFC822")
for cmd.InProgress() {
c.Recv(-1)
for _, rsp := range cmd.Data {
uid := imap.AsNumber(rsp.MessageInfo().Attrs["UID"])
mime := imap.AsBytes(rsp.MessageInfo().Attrs["RFC822"])
if msg, _ := mail.ReadMessage(bytes.NewReader(mime)); msg != nil {
PrintMessageAsJSON(msg, uid)
}
}
cmd.Data = nil
}
return
}
func checkMessagesExist(srcConn *imap.Client, checkRequests chan checkExistsRequest, wg *sync.WaitGroup) {
defer wg.Done()
// get memcache client
cache := memcache.New(MemcacheServer)
timeout := time.NewTicker(NoopMinutes * time.Minute)
done := false
for {
select {
case request, ok := <-checkRequests:
if !ok {
done = true
break
}
// check if it exists in src
// search for in src
cmd, err := imap.Wait(srcConn.UIDSearch([]imap.Field{"HEADER", "Message-Id", request.MessageId}))
if err != nil {
log.Printf("Unable to search source: %s", err.Error())
request.Response <- true
continue
}
results := cmd.Data[0].SearchResults()
// if not found, mark for deletion in DST
found := (len(results) > 0)
// response with found bool
request.Response <- found
// if it doesnt exist, attempt to remove it from memcached
if !found {
cache.Delete(request.MessageId)
}
case <-timeout.C:
imap.Wait(srcConn.Noop())
}
if done {
break
}
}
}
func FetchMessage(conn *imap.Client, messageUID uint32) (msg MessageData, err error) {
seq, _ := imap.NewSeqSet("")
seq.AddNum(messageUID)
var cmd *imap.Command
cmd, err = imap.Wait(conn.UIDFetch(seq, "INTERNALDATE", "BODY[]", "UID", "RFC822.HEADER"))
if err != nil {
log.Printf("Unable to fetch message (%d): %s", messageUID, err.Error())
return
}
if len(cmd.Data) == 0 {
log.Printf("Unable to fetch message (%d) from src: NO DATA", messageUID)
return msg, NotFound
}
msgFields := cmd.Data[0].MessageInfo().Attrs
msg = MessageData{InternalDate: imap.AsDateTime(msgFields["INTERNALDATE"]), Body: imap.AsBytes(msgFields["BODY[]"])}
return msg, nil
}
func checkAndPurgeMessages(conn *imap.Client, requests chan WorkRequest, checkRequests chan checkExistsRequest, wg *sync.WaitGroup) {
defer wg.Done()
timeout := time.NewTicker(NoopMinutes * time.Minute)
done := false
for {
select {
case request, ok := <-requests:
if !ok {
done = true
break
}
// check and wait for response
response := make(chan bool)
cr := checkExistsRequest{UID: request.UID, MessageId: request.Value, Response: response}
checkRequests <- cr
// if response is false (does not exist), flag as Deleted
if exists := <-response; !exists {
log.Printf("not found in src. marking for deletion: %s", request.Value)
err := AddDeletedFlag(conn, request.UID)
if err != nil {
log.Printf("Problems removing message from dst: %s", err.Error())
}
}
case <-timeout.C:
imap.Wait(conn.Noop())
}
if done {
break
}
}
log.Printf("expunging...")
// expunge at the end
allMsgs, _ := imap.NewSeqSet("")
allMsgs.Add("1:*")
imap.Wait(conn.Expunge(allMsgs))
log.Printf("expunge complete.")
}
func GetUIDs(mbox string, client *imap.Client) ([]uint32, error) {
uids := make([]uint32, 0)
cmd, err := client.Select(mbox, true)
if err != nil {
return uids, err
}
//== Get UIDS of all messages
cmd, err = imap.Wait(client.UIDSearch("", "ALL"))
if err != nil {
return uids, err
}
for idx := range cmd.Data {
for _, uid := range cmd.Data[idx].SearchResults() {
uids = append(uids, uid)
}
}
return uids, nil
}
// getNextUID will grab the next message UID from the inbox. Client.Mailbox.UIDNext is cached so we can't use it.
func getNextUID(conn *imap.Client) (uint32, error) {
cmd, err := imap.Wait(conn.Status("INBOX", "UIDNEXT"))
if err != nil {
return 0, err
}
if len(cmd.Data) == 0 {
return 0, errors.New("no data returned!")
}
var status *imap.MailboxStatus
for _, resp := range cmd.Data {
switch resp.Type {
case imap.Data:
status = resp.MailboxStatus()
if status != nil {
break
}
}
}
return status.UIDNext, nil
}
func sensitive(c *imap.Client, action string) imap.LogMask {
mask := c.SetLogMask(imap.LogConn)
hide := imap.LogCmd | imap.LogRaw
if mask&hide != 0 {
c.Logln(imap.LogConn, "Raw logging disabled during", action)
}
c.SetLogMask(mask &^ hide)
return mask
}
//= Return list of IMAP folders
func GetFolders(client *imap.Client) ([]*Folder, error) {
folders := make([]*Folder, 0)
cmd, err := imap.Wait(client.List("", "*"))
if err != nil {
return folders, err
}
for idx := range cmd.Data {
info := cmd.Data[idx].MailboxInfo()
fol := new(Folder)
fol.Name = info.Name
for flag, boo := range info.Attrs {
fmt.Println(info.Name, boo, flag)
if info.Name == "INBOX" && boo {
fol.Type = "inbox"
} else if flag == "\\Junk" && boo {
fol.Type = "junk"
} else if flag == "\\Trash" && boo {
fol.Type = "trash"
} else if flag == "\\Sent" && boo {
fol.Type = "sent"
} else if flag == "\\Drafts" && boo {
fol.Type = "drafts"
}
}
folders = append(folders, fol)
}
return folders, nil
}
// use whatever imap connection type is specified by s.UseTLS
func (s *Server) Connect() (*imap.Client, error) {
if s.client != nil {
s.disconnectTimer.Reset(NoUsageDisconnect)
return s.client, nil
}
var c *imap.Client
var err error
// actual dailing happens
if s.UseTLS {
c, err = s.dialTLS()
} else {
c, err = s.dial()
}
if err != nil {
return nil, err
}
// log in
if c.State() == imap.Login {
_, err := c.Login(s.Username, s.Password)
if err != nil {
s.Close()
return nil, err
}
} else {
s.Close()
return nil, fmt.Errorf("expected imap.Login state, instead was %v.", c.State())
}
// auto-disconnect after a certain timeout
s.disconnectTimer = time.AfterFunc(NoUsageDisconnect, func() {
s.Close()
})
return c, nil
}
func login(c *imap.Client, user, pass string) (cmd *imap.Command, err error) {
defer c.SetLogMask(sensitive(c, "LOGIN"))
return c.Login(user, pass)
}
func ExampleClient() {
//
// Note: most of error handling code is omitted for brevity
//
var (
c *imap.Client
cmd *imap.Command
rsp *imap.Response
)
// Connect to the server
c, _ = imap.Dial("imap.example.com")
// Remember to log out and close the connection when finished
defer c.Logout(30 * time.Second)
// Print server greeting (first response in the unilateral server data queue)
fmt.Println("Server says hello:", c.Data[0].Info)
c.Data = nil
// Enable encryption, if supported by the server
if c.Caps["STARTTLS"] {
c.StartTLS(nil)
}
// Authenticate
if c.State() == imap.Login {
c.Login("*****@*****.**", "mysupersecretpassword")
}
// List all top-level mailboxes, wait for the command to finish
cmd, _ = imap.Wait(c.List("", "%"))
// Print mailbox information
fmt.Println("\nTop-level mailboxes:")
for _, rsp = range cmd.Data {
fmt.Println("|--", rsp.MailboxInfo())
}
// Check for new unilateral server data responses
for _, rsp = range c.Data {
fmt.Println("Server data:", rsp)
}
c.Data = nil
// Open a mailbox (synchronous command - no need for imap.Wait)
c.Select("INBOX", true)
fmt.Print("\nMailbox status:\n", c.Mailbox)
// Fetch the headers of the 10 most recent messages
set, _ := imap.NewSeqSet("")
if c.Mailbox.Messages >= 10 {
set.AddRange(c.Mailbox.Messages-9, c.Mailbox.Messages)
} else {
set.Add("1:*")
}
cmd, _ = c.Fetch(set, "RFC822.HEADER")
// Process responses while the command is running
fmt.Println("\nMost recent messages:")
for cmd.InProgress() {
// Wait for the next response (no timeout)
c.Recv(-1)
// Process command data
for _, rsp = range cmd.Data {
header := imap.AsBytes(rsp.MessageInfo().Attrs["RFC822.HEADER"])
if msg, _ := mail.ReadMessage(bytes.NewReader(header)); msg != nil {
fmt.Println("|--", msg.Header.Get("Subject"))
}
}
cmd.Data = nil
// Process unilateral server data
for _, rsp = range c.Data {
fmt.Println("Server data:", rsp)
}
c.Data = nil
}
// Check command completion status
if rsp, err := cmd.Result(imap.OK); err != nil {
if err == imap.ErrAborted {
fmt.Println("Fetch command aborted")
} else {
fmt.Println("Fetch error:", rsp.Info)
}
}
}
// Idle setup the processes to wait for notifications from the IMAP source connection.
// If an EXISTS or EXPUNGE command comes across the pipe, the appropriate actions will be
// taken to update the destinations. If the process decides the inboxes are out of sync,
// it will pass a bool to the requestPurge channel. It is expected that the requestPurge
// channel is setup to initiate a purge process when it receives the notificaiton.
func Idle(src *imap.Client, appendRequests []chan WorkRequest, requestPurge chan bool) (err error) {
var nextUID uint32
if nextUID, err = getNextUID(src); err != nil {
log.Printf("Unable to get UIDNext: %s", err.Error())
return err
}
// hold the size so we can determine how to react to commands
startSize := src.Mailbox.Messages
// setup interrupt signal channel to terminate the idle
interrupt := make(chan os.Signal, 1)
signal.Notify(interrupt, os.Interrupt, os.Kill)
// setup ticker to reset the idle every 20 minutes (RFC-2177 recommends 29 mins max)
timeout := time.NewTicker(idleTimeoutMinutes * time.Minute)
// setup poller signal for checking for data on the idle command
poll := make(chan bool, 1)
poll <- true
log.Print("beginning idle...")
_, idleErr := src.Idle()
if (idleErr != nil) && (idleErr != imap.ErrTimeout) {
log.Printf("Idle error: %s", idleErr.Error())
return
}
for {
select {
// if we receive a 'poll' we should check the pipe for new messages
case <-poll:
err = src.Recv(0)
if (idleErr != nil) && (idleErr != imap.ErrTimeout) {
log.Printf("Idle error: %s", idleErr.Error())
go sleep(poll)
continue
}
// cache the data so we dont mess it up while start/stopping idle
var tempData []*imap.Response
tempData = append(tempData, src.Data...)
src.Data = nil
for _, data := range tempData {
switch data.Type {
case imap.Data:
// len of 2 likely means its an EXPUNGE or EXISTS command...
if len(data.Fields) == 2 {
msgNum := imap.AsNumber(data.Fields[0])
switch data.Fields[1] {
case "EXPUNGE":
log.Printf("Received an EXPUNGE notification requesting purge - %d", msgNum)
startSize = msgNum
requestPurge <- true
case "EXISTS":
log.Printf("Received an EXISTS notification - %d", msgNum)
if startSize > msgNum {
log.Printf("Mailbox decreased in size %d --> %d. Requesting a purge. MAILBOX MAY NEED TO SYNC", startSize, msgNum)
requestPurge <- true
startSize = msgNum
continue
}
// temporarily term the idle so we can fetch the message
if _, err = src.IdleTerm(); err != nil {
log.Printf("error while temporarily terminating idle: %s", err.Error())
return
}
log.Printf("terminated idle. appending message.")
newMessages := msgNum - startSize
log.Printf("attempting to find/append %d new messages", newMessages)
for i := uint32(0); i < newMessages; i++ {
var request WorkRequest
if request, err = getMessageInfo(src, nextUID); err == nil {
log.Printf("creating %d append requests for %d", len(appendRequests), nextUID)
for _, requests := range appendRequests {
requests <- request
}
log.Printf("done creating append requests for %d", nextUID)
nextUID++
startSize++
} else {
log.Printf("Unable to find message for UID (%d): %s", nextUID, err.Error())
}
}
log.Printf("continuing idle...")
// turn idle back on
if _, err = src.Idle(); err != nil {
log.Printf("Unable to restart idle: %s", err.Error())
return
}
}
}
}
}
go sleep(poll)
case <-interrupt:
log.Printf("Received interrupt. Terminating idle...")
_, err = src.IdleTerm()
if err != nil {
log.Printf("error while terminating idle: %s", err.Error())
}
return
case <-timeout.C:
log.Printf("resetting idle...")
_, err = src.IdleTerm()
if err != nil {
log.Printf("error while temporarily terminating idle: %s", err.Error())
return
}
log.Printf("terminated idle.")
// turn idle back on
_, err = src.Idle()
if err != nil {
log.Printf("Unable to restart idle: %s", err.Error())
return
}
log.Printf("idle restarted.")
}
}
return
}
func GetMessage(folder, uid string, client *imap.Client) (messag *Message, e error) {
cmd, err := client.Select(folder, true)
if err != nil {
return nil, err
}
uidlist, _ := imap.NewSeqSet(uid)
//uidlist.Add(uid)
fmt.Println("get_mess", folder, uid)
mess := new(Message)
mess.Folder = folder
cmd, err = imap.Wait(client.UIDFetch(uidlist, "FLAGS", "INTERNALDATE", "RFC822.SIZE", "RFC822")) // "RFC822.HEADER", "BODY.PEEK[TEXT]") )
if err != nil {
return mess, err
}
fmt.Println(len(cmd.Data), cmd.Data)
rsp := cmd.Data[0]
minfo := rsp.MessageInfo()
mess.Uid = minfo.UID
msg, _ := mail.ReadMessage(bytes.NewReader(imap.AsBytes(minfo.Attrs["RFC822"])))
mime, mime_err := enmime.ParseMIMEBody(msg)
for flag, boo := range minfo.Flags {
if flag == "\\Seen" && boo {
mess.Seen = true
}
if flag == "\\Flagged" && boo {
mess.Flagged = true
}
}
/*
bites := imap.AsBytes(minfo.Attrs["RFC822"])
msg, msg_err := mail.ReadMessage(bytes.NewReader(bites))
if msg_err != nil {
return mess, msg_err
}
*/
//fmt.Println(msg.Header.Get("Content-type"))
//fmt.Println(msg.Header.Get("To"))
//fmt.Println(msg.Header.Get("Delivered-To"))
// From
from, fro_err := mail.ParseAddress(msg.Header.Get("From"))
if fro_err != nil {
fmt.Println("address ettot")
} else {
mess.FromName = from.Name
mess.FromEmail = from.Address
}
//for i, m := range minfo.Attrs {
//fmt.Println(i,m)
//}
// Date
dat := imap.AsDateTime(minfo.Attrs["INTERNALDATE"])
mess.Date = dat.Format("2006-01-02 15:04:05")
mess.Subject = msg.Header.Get("Subject")
mess.ContentType = msg.Header.Get("Content-Type")
//fmt.Println("body=", cmd.Data[0].String)
//bodyb := imap.AsBytes(minfo.Attrs["BODY[TEXT]"])
//bb := bytes.NewReader(bytes.NewReader(header + bodyb))
//fmt.Println("bodyb=", string(msg.Body))
//fmt.Printf("----\n%v\n", mime.Html == nil)
//mess.Body = mime.Text
if mime_err != nil {
fmt.Println("err=", mime_err, mime)
}
//*/
//fmt.Println("body=", body)
mess.BodyText = mime.Text
mess.BodyHtml = mime.Html //imap.AsString(minfo.Attrs["RFC822"])
return mess, nil
}
func AddDeletedFlag(conn *imap.Client, uid uint32) error {
seqSet, _ := imap.NewSeqSet("")
seqSet.AddNum(uid)
_, err := conn.UIDStore(seqSet, "+FLAGS", imap.NewFlagSet(`\Deleted`))
return err
}
func AppendMessage(conn *imap.Client, messageData MessageData) error {
_, err := imap.Wait(conn.Append("INBOX", imap.NewFlagSet("UnSeen"), &messageData.InternalDate, imap.NewLiteral(messageData.Body)))
return err
}