// checkAndStoreMessages will wait for WorkRequests to come acorss the pipe. When it receives a request, it will search
// the given destination inbox for the message. If it is not found, this method will attempt to pull the messages data
// from fetchRequests and then append it to the destination.
func CheckAndAppendMessages(dstConn *imap.Client, storeRequests chan WorkRequest, fetchRequests chan fetchRequest, wg *sync.WaitGroup) {
defer wg.Done()
// noop it every few to keep things alive
timeout := time.NewTicker(NoopMinutes * time.Minute)
done := false
for {
select {
case request, ok := <-storeRequests:
if !ok {
done = true
break
}
// search for in dst
cmd, err := imap.Wait(dstConn.UIDSearch([]imap.Field{"HEADER", request.Header, request.Value}))
if err != nil {
log.Printf("Unable to search for message (%s): %s. skippin!", request.Value, err.Error())
continue
}
results := cmd.Data[0].SearchResults()
// if not found, PULL from SRC and STORE in DST
if len(results) == 0 {
// only fetch if we dont have data already
if len(request.Msg.Body) == 0 {
// build and send fetch request
response := make(chan MessageData)
fr := fetchRequest{MessageId: request.Value, UID: request.UID, Response: response}
fetchRequests <- fr
// grab response from fetchers
request.Msg = <-response
}
if len(request.Msg.Body) == 0 {
log.Printf("No data found for from fetch request (%s). giving up", request.Value)
continue
}
err = AppendMessage(dstConn, request.Msg)
if err != nil {
log.Printf("Problems appending message to dst: %s. quitting.", err.Error())
return
}
}
case <-timeout.C:
imap.Wait(dstConn.Noop())
}
if done {
break
}
}
log.Print("storer complete!")
return
}
// FetchEmails will sit and wait for fetchRequests from the destination workers.
func fetchEmails(conn *imap.Client, requests chan fetchRequest, cache *Cache) {
// noop every few to keep things alive
timeout := time.NewTicker(NoopMinutes * time.Minute)
done := false
for {
select {
case request, ok := <-requests:
if !ok {
done = true
break
}
found := true
// check if the message body is in cache
data, err := cache.Get(request.MessageId)
if err != nil {
found = false
if err != ErrNotFound {
log.Printf("problems pulling message data from cache: %s. Pulling message from src...", err.Error())
}
data = MessageData{}
}
if found {
log.Print("cache success!")
request.Response <- data
continue
}
msgData, err := FetchMessage(conn, request.UID)
if err != nil {
if err == NotFound {
log.Printf("No data found for UID: %d", request.UID)
} else {
log.Printf("Problems fetching message (%s) data: %s. Passing request and quitting.", request.MessageId, err.Error())
requests <- request
return
}
}
request.Response <- msgData
err = cache.Put(request.MessageId, msgData)
if err != nil {
log.Printf("Unable to add message (%s) to cache: %s", request.MessageId, err.Error())
}
case <-timeout.C:
imap.Wait(conn.Noop())
}
if done {
break
}
}
}
func checkMessagesExist(srcConn *imap.Client, checkRequests chan checkExistsRequest, wg *sync.WaitGroup) {
defer wg.Done()
// get memcache client
cache := memcache.New(MemcacheServer)
timeout := time.NewTicker(NoopMinutes * time.Minute)
done := false
for {
select {
case request, ok := <-checkRequests:
if !ok {
done = true
break
}
// check if it exists in src
// search for in src
cmd, err := imap.Wait(srcConn.UIDSearch([]imap.Field{"HEADER", "Message-Id", request.MessageId}))
if err != nil {
log.Printf("Unable to search source: %s", err.Error())
request.Response <- true
continue
}
results := cmd.Data[0].SearchResults()
// if not found, mark for deletion in DST
found := (len(results) > 0)
// response with found bool
request.Response <- found
// if it doesnt exist, attempt to remove it from memcached
if !found {
cache.Delete(request.MessageId)
}
case <-timeout.C:
imap.Wait(srcConn.Noop())
}
if done {
break
}
}
}
func checkAndPurgeMessages(conn *imap.Client, requests chan WorkRequest, checkRequests chan checkExistsRequest, wg *sync.WaitGroup) {
defer wg.Done()
timeout := time.NewTicker(NoopMinutes * time.Minute)
done := false
for {
select {
case request, ok := <-requests:
if !ok {
done = true
break
}
// check and wait for response
response := make(chan bool)
cr := checkExistsRequest{UID: request.UID, MessageId: request.Value, Response: response}
checkRequests <- cr
// if response is false (does not exist), flag as Deleted
if exists := <-response; !exists {
log.Printf("not found in src. marking for deletion: %s", request.Value)
err := AddDeletedFlag(conn, request.UID)
if err != nil {
log.Printf("Problems removing message from dst: %s", err.Error())
}
}
case <-timeout.C:
imap.Wait(conn.Noop())
}
if done {
break
}
}
log.Printf("expunging...")
// expunge at the end
allMsgs, _ := imap.NewSeqSet("")
allMsgs.Add("1:*")
imap.Wait(conn.Expunge(allMsgs))
log.Printf("expunge complete.")
}