Beispiel #1
0
func main() {
	if len(os.Args) < 2 {
		log.Fatalf("Usage: %s TOPIC", os.Args[0])
	}
	url := "http://l1sp.org/" + os.Args[1]
	request, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Fatal(err)
	}
	client := &http.Client{}
	resp, err := client.Do(request)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	text, err := html2text.FromString(string(contents))
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(text)
}
func (bt *berthaTransformer) authorToPerson(a author) (person, error) {
	uuid := uuid.NewMD5(uuid.UUID{}, []byte(a.TmeIdentifier)).String()
	plainDescription, err := html2text.FromString(a.Biography)

	if err != nil {
		return person{}, err
	}

	altIds := alternativeIdentifiers{
		UUIDS: []string{uuid},
		TME:   []string{a.TmeIdentifier},
	}

	p := person{
		Uuid:                   uuid,
		Name:                   a.Name,
		PrefLabel:              a.Name,
		EmailAddress:           a.Email,
		TwitterHandle:          a.TwitterHandle,
		FacebookProfile:        a.FacebookProfile,
		LinkedinProfile:        a.LinkedinProfile,
		Description:            plainDescription,
		DescriptionXML:         a.Biography,
		ImageUrl:               a.ImageUrl,
		AlternativeIdentifiers: altIds,
	}

	return p, err
}
Beispiel #3
0
func (self *Wiki) IndexArticle(path string, article *Article) error {
	html, err := article.GetHtml()
	if err != nil {
		return err
	}
	text, err := html2text.FromString(html)
	if err != nil {
		return err
	}
	self.Index.Index(path, ArticleData{Name: article.Name, Content: text})
	return nil
}
Beispiel #4
0
func completeTags(episode *Episode) {

	logger.Debug.Println("Tag update : " + episode.Podcast.feedPodcast.Title + " - " + episode.feedEpisode.Title + " : " + episode.file())

	tag, err := taglib.Read(episode.file())
	if err != nil {
		logger.Warning.Println("Cannot complete episode tags for "+episode.Podcast.feedPodcast.Title+" - "+episode.feedEpisode.Title, err)
		return
	}
	defer tag.Close()

	var replaceArtist string
	if episode.feedEpisode.Author.Name != "" {
		replaceArtist = episode.feedEpisode.Author.Name
	} else {
		replaceArtist = episode.Podcast.feedPodcast.Title
	}

	//use the podcast title for now
	replaceArtist = episode.Podcast.feedPodcast.Title

	completeTag(taglib.Artist, replaceArtist, tag)
	completeTag(taglib.Album, episode.Podcast.feedPodcast.Title, tag)

	plaintextDescription, err := html2text.FromString(episode.feedEpisode.Description)
	if err == nil {
		episode.feedEpisode.Description = plaintextDescription
	}
	if len(episode.feedEpisode.Description) > maxCommentSize+5 {
		episode.feedEpisode.Description = episode.feedEpisode.Description[:maxCommentSize] + " ..."
	}

	completeTag(taglib.Comments, episode.feedEpisode.Description, tag)
	completeTag(taglib.Title, episode.feedEpisode.Title+" "+episode.formattedPubDate(dateFormat), tag)
	completeTag(taglib.Genre, "Podcast", tag)

	pubdate, err := episode.feedEpisode.ParsedPubDate()
	if err == nil {
		completeTag(taglib.Year, strconv.Itoa(pubdate.Year()), tag)
	}
	logger.Debug.Println("Tag Write Start for : " + episode.file())
	err = tag.Save()
	//setAlbumArtist(episode.Podcast.feedPodcast.Title, episode.file())

	logger.Debug.Println("Tag Write End for : " + episode.file())
	if err != nil {
		logger.Warning.Println(episode.Podcast.feedPodcast.Title+" - "+episode.feedEpisode.Title+" : Cannot save the modified tags", err)
	}
	logger.Debug.Println("Tag update END : " + episode.Podcast.feedPodcast.Title + " - " + episode.feedEpisode.Title)

}
Beispiel #5
0
//formatMessage make into text if contains html
func formatMessage(message string) string {

	isHTML := strings.Contains(message, "<html>")

	if isHTML {
		text, err := html2text.FromString(message)
		if err != nil {
			log.Println("failed to convert to text " + err.Error())
		}
		return text
	}

	return message
}
Beispiel #6
0
// NewMessageFrom creates new mail message object with custom From header.
func NewMessageFrom(to []string, from, subject, htmlBody string) *Message {
	msg := gomail.NewMessage()
	msg.SetHeader("From", from)
	msg.SetHeader("To", to...)
	msg.SetHeader("Subject", subject)
	msg.SetDateHeader("Date", time.Now())

	body, err := html2text.FromString(htmlBody)
	if err != nil {
		log.Error(4, "html2text.FromString: %v", err)
		msg.SetBody("text/html", htmlBody)
	} else {
		msg.SetBody("text/plain", body)
		if setting.MailService.EnableHTMLAlternative {
			msg.AddAlternative("text/html", htmlBody)
		}
	}

	return &Message{
		Message: msg,
	}
}
Beispiel #7
0
func (c *Context) View(i int) {
	var email protocol.Encrypted
	request, err := http.NewRequest("GET", fmt.Sprintf("%v/inbox/%v", c.host, i), nil)
	request.SetBasicAuth("user", c.password)
	response, err := c.client.Do(request)
	if err != nil {
		log.Fatal(err)
	}
	if response.StatusCode == http.StatusOK {
		data, err := ioutil.ReadAll(response.Body)
		response.Body.Close()
		if err != nil {
			log.Fatal(err)
		}
		err = proto.Unmarshal(data, &email)
		if err != nil {
			log.Fatal(err)
		}

		key, err := rsa.DecryptPKCS1v15(rand.Reader, &c.key, email.Key)
		if err != nil {
			log.Fatal(err)
		}
		cipher, err := aes.NewCipher(key)
		if err != nil {
			log.Fatal(err)
		}
		cipher.Decrypt(email.Data, email.Data)
		decrypted := protocol.Email{}
		err = proto.Unmarshal(email.Data, &decrypted)
		if err != nil {
			log.Fatal(err)
		}

		process := func(mediaType string, body []byte) {
			fmt.Println(mediaType)
			if mediaType == "text/plain" {
				fmt.Println(string(body))
			} else if mediaType == "text/html" {
				text, err := html2text.FromString(string(body))
				if err != nil {
					log.Fatal(err)
				}
				fmt.Println(text)
			}
		}
		message, err := goemail.ParseMessage(strings.NewReader(*decrypted.Mail))
		if err != nil {
			log.Fatal(err)
		}
		if message.HasBody() {
			mediaType, _, _ := message.Header.ContentType()
			process(mediaType, message.Body)
		} else {
			for _, part := range message.MessagesAll() {
				mediaType, _, err := part.Header.ContentType()
				if err != nil {
					log.Fatal(err)
				}
				process(mediaType, part.Body)
			}
		}
	}
}
Beispiel #8
0
// ParseMIMEBody parses the body of the message object into a  tree of MIMEPart
// objects, each of which is aware of its content type, filename and headers.
// If the part was encoded in quoted-printable or base64, it is decoded before
// being stored in the MIMEPart object.
func ParseMIMEBody(mailMsg *mail.Message) (*MIMEBody, error) {
	mimeMsg := &MIMEBody{
		IsTextFromHTML: false,
		header:         mailMsg.Header,
	}

	if !IsMultipartMessage(mailMsg) {
		// Attachment only?
		if IsBinaryBody(mailMsg) {
			return binMIME(mailMsg)
		}

		// Parse as text only
		bodyBytes, err := decodeSection(mailMsg.Header.Get("Content-Transfer-Encoding"),
			mailMsg.Body)
		if err != nil {
			return nil, fmt.Errorf("Error decoding text-only message: %v", err)
		}
		// Handle plain ASCII text, content-type unspecified
		mimeMsg.Text = string(bodyBytes)

		// Process top-level content-type
		ctype := mailMsg.Header.Get("Content-Type")
		if ctype != "" {
			if mediatype, mparams, err := mime.ParseMediaType(ctype); err == nil {
				if mparams["charset"] != "" {
					// Convert plain text to UTF8 if content type specified a charset
					newStr, err := ConvertToUTF8String(mparams["charset"], bodyBytes)
					if err != nil {
						return nil, err
					}
					mimeMsg.Text = newStr
				} else if mediatype == "text/html" {
					// charset is empty, look in HTML body for charset
					charset, err := charsetFromHTMLString(mimeMsg.Text)

					if charset != "" && err == nil {
						newStr, err := ConvertToUTF8String(charset, bodyBytes)
						if err == nil {
							mimeMsg.Text = newStr
						}
					}
				}
				if mediatype == "text/html" {
					mimeMsg.HTML = mimeMsg.Text
					// Empty Text will trigger html2text conversion below
					mimeMsg.Text = ""
				}
			}
		}
	} else {
		// Parse top-level multipart
		ctype := mailMsg.Header.Get("Content-Type")
		mediatype, params, err := mime.ParseMediaType(ctype)
		if err != nil {
			return nil, fmt.Errorf("Unable to parse media type: %v", err)
		}
		if !strings.HasPrefix(mediatype, "multipart/") {
			return nil, fmt.Errorf("Unknown mediatype: %v", mediatype)
		}
		boundary := params["boundary"]
		if boundary == "" {
			return nil, fmt.Errorf("Unable to locate boundary param in Content-Type header")
		}
		// Root Node of our tree
		root := NewMIMEPart(nil, mediatype)
		mimeMsg.Root = root
		err = parseParts(root, mailMsg.Body, boundary)
		if err != nil {
			return nil, err
		}

		// Locate text body
		if mediatype == "multipart/altern" {
			match := BreadthMatchFirst(root, func(p MIMEPart) bool {
				return p.ContentType() == "text/plain" && p.Disposition() != "attachment"
			})
			if match != nil {
				if match.Charset() != "" {
					newStr, err := ConvertToUTF8String(match.Charset(), match.Content())
					if err != nil {
						return nil, err
					}
					mimeMsg.Text += newStr
				} else {
					mimeMsg.Text += string(match.Content())
				}
			}
		} else {
			// multipart is of a mixed type
			match := DepthMatchAll(root, func(p MIMEPart) bool {
				return p.ContentType() == "text/plain" && p.Disposition() != "attachment"
			})
			for i, m := range match {
				if i > 0 {
					mimeMsg.Text += "\n--\n"
				}
				if m.Charset() != "" {
					newStr, err := ConvertToUTF8String(m.Charset(), m.Content())
					if err != nil {
						return nil, err
					}
					mimeMsg.Text += newStr
				} else {
					mimeMsg.Text += string(m.Content())
				}
			}
		}

		// Locate HTML body
		match := BreadthMatchFirst(root, func(p MIMEPart) bool {
			return p.ContentType() == "text/html" && p.Disposition() != "attachment"
		})
		if match != nil {
			if match.Charset() != "" {
				newStr, err := ConvertToUTF8String(match.Charset(), match.Content())
				if err != nil {
					return nil, err
				}
				mimeMsg.HTML += newStr
			} else {
				mimeMsg.HTML = string(match.Content())
			}

		}

		// Locate attachments
		mimeMsg.Attachments = BreadthMatchAll(root, func(p MIMEPart) bool {
			return p.Disposition() == "attachment" || p.ContentType() == "application/octet-stream"
		})

		// Locate inlines
		mimeMsg.Inlines = BreadthMatchAll(root, func(p MIMEPart) bool {
			return p.Disposition() == "inline"
		})

		// Locate others parts not handled in "Attachments" and "inlines"
		mimeMsg.OtherParts = BreadthMatchAll(root, func(p MIMEPart) bool {
			if strings.HasPrefix(p.ContentType(), "multipart/") {
				return false
			}

			if p.Disposition() != "" {
				return false
			}

			if p.ContentType() == "application/octet-stream" {
				return false
			}

			return p.ContentType() != "text/plain" && p.ContentType() != "text/html"
		})
	}

	// Down-convert HTML to text if necessary
	if mimeMsg.Text == "" && mimeMsg.HTML != "" {
		mimeMsg.IsTextFromHTML = true
		var err error
		if mimeMsg.Text, err = html2text.FromString(mimeMsg.HTML); err != nil {
			// Fail gently
			mimeMsg.Text = ""
			return mimeMsg, err
		}
	}

	return mimeMsg, nil
}
Beispiel #9
0
func main() {
	domain := flag.String("domain", "", "mailgun domain")
	privateKey := flag.String("private-key", "", "secret mailgun api key")
	publicKey := flag.String("public-key", "", "mailgun public api key")
	dry := flag.Bool("dry", false, "do not update cache & only print to stdout (no e-mail)")
	verbose := flag.Bool("verbose", false, "Print detailed output per monitored url.")
	flag.Parse()

	if *domain == "" || *privateKey == "" || *publicKey == "" {
		log.Fatalln("domain, private-key and public-key flags are required")
	}

	gun := m.NewMailgun(*domain, *privateKey, *publicKey)

	urls := loadUrls()
	for _, url := range urls {
		body, err := requestURL(url)
		if err != nil {
			log.Println("Skipped URL because of error requesting it:", url)
			continue
		}
		filename := getExecFolder() + "/cache/" + getMD5Hash(url) + ".html"

		cached, err := ioutil.ReadFile(filename)
		if err != nil {
			if *dry == false {
				updateCache(filename, body)
			}

			switch err := err.(type) {
			case *os.PathError:
				fmt.Printf("This URL will now be monitored: %s\n\n", url)
			default:
				log.Fatalf("Fatal errors type %T\n", err)
			}
		} else {
			if reflect.DeepEqual(cached, body) {
				if *verbose {
					fmt.Printf("This URL didn't change: %s\n\n", url)
				}
			} else {
				cachedDoc := getGoqueryDoc(cached)
				currentDoc := getGoqueryDoc(body)
				cachedContent, _ := cachedDoc.Find("#content").Html()
				currentContent, _ := currentDoc.Find("#content").Html()

				if cachedContent == currentContent {
					if *verbose {
						fmt.Println("The website changed, but the content stayed the same.")
					}
				} else {
					cachedText, _ := html2text.FromString(cachedContent)
					currentText, _ := html2text.FromString(currentContent)
					diff := difflib.Diff(strings.Split(cachedText, "\n"), strings.Split(currentText, "\n"))

					msg := createMessage(diff, url)
					htmlMsg := createHTMLMessage(diff, url)
					if *dry {
						fmt.Println(msg)
					} else {
						sendEmails(gun, msg, htmlMsg)
					}
				}
				if *dry == false {
					updateCache(filename, body)
				}
			}
		}
	}
}
Beispiel #10
0
Datei: mail.go Projekt: kaey/mail
// ReadMessage reads message from r.
// Using Send() on read messages can result in garbage in headers,
// make sure to remove unnecessary ones, before sending.
func ReadMessage(r io.Reader) (*Message, error) {
	rawmsg, err := mail.ReadMessage(r)
	if err != nil {
		return nil, err
	}

	m := new(Message)

	// MessageID
	if id := rawmsg.Header.Get("Message-Id"); id != "" {
		m.ID = id
	} else {
		m.ID = makeID()
	}

	// Date
	if date, err := rawmsg.Header.Date(); err == nil {
		m.Date = date
	} else {
		m.Date = time.Now()
	}

	// Subject
	if subject, err := decodeHeader(rawmsg.Header.Get("Subject")); err == nil && subject != "" {
		m.Subject = subject
	} else if err != nil {
		return nil, fmt.Errorf("decode header: %v", err)
	} else {
		m.Subject = "No subject"
	}

	// Return-Path
	if h := rawmsg.Header.Get("Return-Path"); h != "" {
		retpath, err := DecodeAddress(h)
		if err != nil {
			return nil, fmt.Errorf("parse return-path: %v", err)
		}
		if len(retpath) > 0 {
			m.ReturnPath = retpath[0]
		}
	}

	// From
	if h := rawmsg.Header.Get("From"); h != "" {
		from, err := DecodeAddress(h)
		if err != nil {
			return nil, fmt.Errorf("parse from: %v", err)
		}
		if len(from) > 0 {
			m.From = from[0]
		}
	}

	// To
	if h := rawmsg.Header.Get("To"); h != "" {
		to, err := DecodeAddress(h)
		if err != nil {
			return nil, fmt.Errorf("parse to: %v", err)
		}
		m.To = to
	} else {
		m.To = make([]string, 0)
	}

	// CC
	if h := rawmsg.Header.Get("Cc"); h != "" {
		cc, err := DecodeAddress(h)
		if err != nil {
			return nil, fmt.Errorf("parse cc: %v", err)
		}
		m.CC = cc
	} else {
		m.CC = make([]string, 0)
	}

	// If return-path is unset, set it using from.
	if m.ReturnPath == "" {
		m.ReturnPath = m.From
	}

	// Decode rest of the headers.
	headers := make(map[string]string)
	for k, v := range rawmsg.Header {
		switch k {
		case "Message-Id", "Subject", "Date", "Return-Path", "From", "To", "Cc":
			continue
		}
		for _, w := range v {
			decoded, err := decodeHeader(w)
			if err != nil {
				return nil, fmt.Errorf("decode header: %v", err)
			}
			str := headers[k]
			if str != "" {
				str += " "
			}
			headers[k] = str + decoded
		}
	}
	m.Headers = headers

	// Decode body.
	if err := m.decodeBody(rawmsg.Body, textproto.MIMEHeader(rawmsg.Header)); err != nil {
		return nil, fmt.Errorf("decode body: %v", err)
	}

	if len(m.HTML) > 0 {
		m.Body, err = html2text.FromString(m.HTML)
		if err != nil {
			return nil, err
		}
		m.IsHTML = true
	}

	return m, nil
}