// Adds a new part to the given multipart writer, containing the given revision.
// The revision will be written as a nested multipart body if it has attachments.
func (db *Database) WriteRevisionAsPart(revBody Body, isError bool, compressPart bool, writer *multipart.Writer) error {
partHeaders := textproto.MIMEHeader{}
docID, _ := revBody["_id"].(string)
revID, _ := revBody["_rev"].(string)
if len(docID) > 0 {
partHeaders.Set("X-Doc-ID", docID)
partHeaders.Set("X-Rev-ID", revID)
}
if hasInlineAttachments(revBody) {
// Write as multipart, including attachments:
// OPT: Find a way to do this w/o having to buffer the MIME body in memory!
var buffer bytes.Buffer
docWriter := multipart.NewWriter(&buffer)
contentType := fmt.Sprintf("multipart/related; boundary=%q",
docWriter.Boundary())
partHeaders.Set("Content-Type", contentType)
db.WriteMultipartDocument(revBody, docWriter, compressPart)
docWriter.Close()
content := bytes.TrimRight(buffer.Bytes(), "\r\n")
part, err := writer.CreatePart(partHeaders)
if err == nil {
_, err = part.Write(content)
}
return err
} else {
// Write as JSON:
contentType := "application/json"
if isError {
contentType += `; error="true"`
}
return writeJSONPart(writer, contentType, revBody, compressPart)
}
}
func writeJSONPart(writer *multipart.Writer, contentType string, body Body, compressed bool) (err error) {
bytes, err := json.Marshal(body)
if err != nil {
return err
}
if len(bytes) < kMinCompressedJSONSize {
compressed = false
}
partHeaders := textproto.MIMEHeader{}
partHeaders.Set("Content-Type", contentType)
if compressed {
partHeaders.Set("Content-Encoding", "gzip")
}
part, err := writer.CreatePart(partHeaders)
if err != nil {
return err
}
if compressed {
gz := gzip.NewWriter(part)
_, err = gz.Write(bytes)
gz.Close()
} else {
_, err = part.Write(bytes)
}
return
}
// CreateMailMessage creates a (plain) SMTP email with body and
// optional attachments.
func CreateMailMessage(body []byte, att []*MailAttachment) ([]byte, error) {
buf := new(bytes.Buffer)
wrt := multipart.NewWriter(buf)
buf.WriteString(
"MIME-Version: 1.0\n" +
"Content-Type: multipart/mixed;\n" +
" boundary=\"" + wrt.Boundary() + "\"\n\n" +
"This is a multi-part message in MIME format.\n")
hdr := textproto.MIMEHeader{}
hdr.Set("Content-Type", "text/plain; charset=ISO-8859-15")
hdr.Set("Content-Transfer-Encoding", "utf-8")
pw, err := wrt.CreatePart(hdr)
if err != nil {
return nil, err
}
pw.Write(body)
for _, a := range att {
pw, err = wrt.CreatePart(a.Header)
if err != nil {
return nil, err
}
pw.Write(a.Data)
}
wrt.Close()
return buf.Bytes(), nil
}
// Writes a revision to a MIME multipart writer, encoding large attachments as separate parts.
func (db *Database) WriteMultipartDocument(body Body, writer *multipart.Writer, compress bool) {
// First extract the attachments that should follow:
following := []attInfo{}
for name, value := range BodyAttachments(body) {
meta := value.(map[string]interface{})
var info attInfo
info.contentType, _ = meta["content_type"].(string)
info.data, _ = decodeAttachment(meta["data"])
if info.data != nil && len(info.data) > kMaxInlineAttachmentSize {
info.name = name
following = append(following, info)
delete(meta, "data")
meta["follows"] = true
}
}
// Write the main JSON body:
writeJSONPart(writer, "application/json", body, compress)
// Write the following attachments
for _, info := range following {
partHeaders := textproto.MIMEHeader{}
if info.contentType != "" {
partHeaders.Set("Content-Type", info.contentType)
}
partHeaders.Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", info.name))
part, _ := writer.CreatePart(partHeaders)
part.Write(info.data)
}
}
// parseMIMEParts will recursively walk a MIME entity and return a []mime.Part containing
// each (flattened) mime.Part found.
// It is important to note that there are no limits to the number of recursions, so be
// careful when parsing unknown MIME structures!
func parseMIMEParts(hs textproto.MIMEHeader, b io.Reader) ([]*part, error) {
var ps []*part
// If no content type is given, set it to the default
if _, ok := hs["Content-Type"]; !ok {
hs.Set("Content-Type", defaultContentType)
}
ct, params, err := mime.ParseMediaType(hs.Get("Content-Type"))
if err != nil {
return ps, err
}
// If it's a multipart email, recursively parse the parts
if strings.HasPrefix(ct, "multipart/") {
if _, ok := params["boundary"]; !ok {
return ps, ErrMissingBoundary
}
mr := multipart.NewReader(b, params["boundary"])
for {
var buf bytes.Buffer
p, err := mr.NextPart()
if err == io.EOF {
break
}
if err != nil {
return ps, err
}
if _, ok := p.Header["Content-Type"]; !ok {
p.Header.Set("Content-Type", defaultContentType)
}
subct, _, err := mime.ParseMediaType(p.Header.Get("Content-Type"))
if strings.HasPrefix(subct, "multipart/") {
sps, err := parseMIMEParts(p.Header, p)
if err != nil {
return ps, err
}
ps = append(ps, sps...)
} else {
// Otherwise, just append the part to the list
// Copy the part data into the buffer
if _, err := io.Copy(&buf, p); err != nil {
return ps, err
}
ps = append(ps, &part{body: buf.Bytes(), header: p.Header})
}
}
} else {
// If it is not a multipart email, parse the body content as a single "part"
var buf bytes.Buffer
if _, err := io.Copy(&buf, b); err != nil {
return ps, err
}
ps = append(ps, &part{body: buf.Bytes(), header: hs})
}
return ps, nil
}
func renderPartsToWriter(parts cloudInitParts, writer io.Writer) error {
mimeWriter := multipart.NewWriter(writer)
defer mimeWriter.Close()
writer.Write([]byte(fmt.Sprintf("Content-Type: multipart/mixed; boundary=\"%s\"\n", mimeWriter.Boundary())))
for _, part := range parts {
header := textproto.MIMEHeader{}
if part.ContentType == "" {
header.Set("Content-Type", "text/plain")
} else {
header.Set("Content-Type", part.ContentType)
}
if part.Filename != "" {
header.Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, part.Filename))
}
if part.MergeType != "" {
header.Set("X-Merge-Type", part.MergeType)
}
partWriter, err := mimeWriter.CreatePart(header)
if err != nil {
return err
}
_, err = partWriter.Write([]byte(part.Content))
if err != nil {
return err
}
}
return nil
}
func appendPart(w *multipart.Writer, headers func(h textproto.MIMEHeader), body string) {
if body == "" {
return
}
h := textproto.MIMEHeader{}
h.Set("Content-Transfer-Encoding", "quoted-printable")
headers(h)
partW, err := w.CreatePart(h)
log.Panice(err, "create MIME part")
quoW := quotedprintable.NewWriter(partW)
defer quoW.Close()
io.WriteString(quoW, body)
}
// store message, unpack attachments, register with daemon, send to daemon for federation
// in that order
func (self *nntpConnection) storeMessage(daemon *NNTPDaemon, hdr textproto.MIMEHeader, body io.Reader) (err error) {
var f io.WriteCloser
msgid := getMessageID(hdr)
if msgid == "" {
// drop, invalid header
log.Println(self.name, "dropping message with invalid mime header, no message-id")
_, err = io.Copy(Discard, body)
return
} else if ValidMessageID(msgid) {
f = daemon.store.CreateFile(msgid)
} else {
// invalid message-id
log.Println(self.name, "dropping message with invalid message-id", msgid)
_, err = io.Copy(Discard, body)
return
}
if f == nil {
// could not open file, probably already storing it from another connection
log.Println(self.name, "discarding duplicate message")
_, err = io.Copy(Discard, body)
return
}
path := hdr.Get("Path")
hdr.Set("Path", daemon.instance_name+"!"+path)
// now store attachments and article
err = writeMIMEHeader(f, hdr)
if err == nil {
err = daemon.store.ProcessMessageBody(f, hdr, body)
if err == nil {
// tell daemon
daemon.loadFromInfeed(msgid)
}
}
f.Close()
if err != nil {
// clean up
if ValidMessageID(msgid) {
DelFile(daemon.store.GetFilename(msgid))
}
}
return
}
func createFormFile(writer *multipart.Writer, fieldname, filename string) {
// Try to open the file.
file, err := os.Open(filename)
if err != nil {
panic(err)
}
defer file.Close()
// Create a new form-data header with the provided field name and file name.
// Determine Content-Type of the file by its extension.
h := textproto.MIMEHeader{}
h.Set("Content-Disposition", fmt.Sprintf(
`form-data; name="%s"; filename="%s"`,
escapeQuotes(fieldname),
escapeQuotes(filepath.Base(filename)),
))
h.Set("Content-Type", "application/octet-stream")
if ct := mime.TypeByExtension(filepath.Ext(filename)); ct != "" {
h.Set("Content-Type", ct)
}
part, err := writer.CreatePart(h)
if err != nil {
panic(err)
}
// Copy the content of the file we have opened not reading the whole
// file into memory.
_, err = io.Copy(part, file)
if err != nil {
panic(err)
}
}
// Send does what it is supposed to do.
func (m *Mail) Send(host string, port int, user, pass string) error {
// validate from address
from, err := mail.ParseAddress(m.From)
if err != nil {
return err
}
// validate to address
to, err := mail.ParseAddress(m.To)
if err != nil {
return err
}
// set headers for html email
header := textproto.MIMEHeader{}
header.Set(textproto.CanonicalMIMEHeaderKey("from"), from.Address)
header.Set(textproto.CanonicalMIMEHeaderKey("to"), to.Address)
header.Set(textproto.CanonicalMIMEHeaderKey("content-type"), "text/html; charset=UTF-8")
header.Set(textproto.CanonicalMIMEHeaderKey("mime-version"), "1.0")
header.Set(textproto.CanonicalMIMEHeaderKey("subject"), m.Subject)
// init empty message
var buffer bytes.Buffer
// write header
for key, value := range header {
buffer.WriteString(fmt.Sprintf("%s: %s\r\n", key, value[0]))
}
// write body
buffer.WriteString(fmt.Sprintf("\r\n%s", m.HTML))
// send email
addr := fmt.Sprintf("%s:%d", host, port)
auth := smtp.PlainAuth("", user, pass, host)
return smtp.SendMail(addr, auth, from.Address, []string{to.Address}, buffer.Bytes())
}
// Bytes converts the Email object to a []byte representation, including all needed MIMEHeaders, boundaries, etc.
func (m *Message) Bytes() ([]byte, error) {
// TODO: better guess buffer size
buff := bytes.NewBuffer(make([]byte, 0, 4096))
headers := m.msgHeaders()
w := multipart.NewWriter(buff)
// TODO: determine the content type based on message/attachment mix.
headers.Set("Content-Type", "multipart/mixed;\r\n boundary="+w.Boundary())
headerToBytes(buff, headers)
io.WriteString(buff, "\r\n")
// Start the multipart/mixed part
fmt.Fprintf(buff, "--%s\r\n", w.Boundary())
header := textproto.MIMEHeader{}
// Check to see if there is a Text or HTML field
if len(m.Content) > 0 {
subWriter := multipart.NewWriter(buff)
// Create the multipart alternative part
header.Set("Content-Type", fmt.Sprintf("multipart/alternative;\r\n boundary=%s\r\n", subWriter.Boundary()))
// Write the header
headerToBytes(buff, header)
// Create the body sections
if len(m.Content) > 0 {
header.Set("Content-Type", fmt.Sprintf("%s; charset=UTF-8", m.ContentType))
header.Set("Content-Transfer-Encoding", "quoted-printable")
if _, err := subWriter.CreatePart(header); err != nil {
return nil, err
}
// Write the text
if err := quotePrintEncode(buff, []byte(m.Content)); err != nil {
return nil, err
}
}
if err := subWriter.Close(); err != nil {
return nil, err
}
}
// Create attachment part, if necessary
for _, a := range m.Attachments {
ap, err := w.CreatePart(a.Header)
if err != nil {
return nil, err
}
// Write the base64Wrapped content to the part
base64Wrap(ap, a.Content)
}
if err := w.Close(); err != nil {
return nil, err
}
return buff.Bytes(), nil
}
func renderPartsToWriter(parts cloudInitParts, writer io.Writer) error {
mimeWriter := multipart.NewWriter(writer)
defer mimeWriter.Close()
// we need to set the boundary explictly, otherwise the boundary is random
// and this causes terraform to complain about the resource being different
if err := mimeWriter.SetBoundary("MIMEBOUNDARY"); err != nil {
return err
}
writer.Write([]byte(fmt.Sprintf("Content-Type: multipart/mixed; boundary=\"%s\"\n", mimeWriter.Boundary())))
writer.Write([]byte("MIME-Version: 1.0\r\n"))
for _, part := range parts {
header := textproto.MIMEHeader{}
if part.ContentType == "" {
header.Set("Content-Type", "text/plain")
} else {
header.Set("Content-Type", part.ContentType)
}
header.Set("MIME-Version", "1.0")
header.Set("Content-Transfer-Encoding", "7bit")
if part.Filename != "" {
header.Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, part.Filename))
}
if part.MergeType != "" {
header.Set("X-Merge-Type", part.MergeType)
}
partWriter, err := mimeWriter.CreatePart(header)
if err != nil {
return err
}
_, err = partWriter.Write([]byte(part.Content))
if err != nil {
return err
}
}
return nil
}
// Bytes converts the Email object to a []byte representation, including all needed MIMEHeaders, boundaries, etc.
func (e *Email) Bytes() ([]byte, error) {
buff := &bytes.Buffer{}
w := multipart.NewWriter(buff)
// Set the appropriate headers (overwriting any conflicts)
// Leave out Bcc (only included in envelope headers)
e.Headers.Set("To", strings.Join(e.To, ","))
if e.Cc != nil {
e.Headers.Set("Cc", strings.Join(e.Cc, ","))
}
e.Headers.Set("From", e.From)
e.Headers.Set("Subject", e.Subject)
if len(e.ReadReceipt) != 0 {
e.Headers.Set("Disposition-Notification-To", strings.Join(e.ReadReceipt, ","))
}
e.Headers.Set("MIME-Version", "1.0")
e.Headers.Set("Content-Type", fmt.Sprintf("multipart/mixed;\r\n boundary=%s\r\n", w.Boundary()))
// Write the envelope headers (including any custom headers)
if err := headerToBytes(buff, e.Headers); err != nil {
return nil, fmt.Errorf("Failed to render message headers: %s", err)
}
// Start the multipart/mixed part
fmt.Fprintf(buff, "--%s\r\n", w.Boundary())
header := textproto.MIMEHeader{}
// Check to see if there is a Text or HTML field
if e.Text != "" || e.HTML != "" {
subWriter := multipart.NewWriter(buff)
// Create the multipart alternative part
header.Set("Content-Type", fmt.Sprintf("multipart/alternative;\r\n boundary=%s\r\n", subWriter.Boundary()))
// Write the header
if err := headerToBytes(buff, header); err != nil {
return nil, fmt.Errorf("Failed to render multipart message headers: %s", err)
}
// Create the body sections
if e.Text != "" {
header.Set("Content-Type", fmt.Sprintf("text/plain; charset=UTF-8"))
header.Set("Content-Transfer-Encoding", "quoted-printable")
if _, err := subWriter.CreatePart(header); err != nil {
return nil, err
}
// Write the text
if err := quotePrintEncode(buff, e.Text); err != nil {
return nil, err
}
}
if e.HTML != "" {
header.Set("Content-Type", fmt.Sprintf("text/html; charset=UTF-8"))
header.Set("Content-Transfer-Encoding", "quoted-printable")
if _, err := subWriter.CreatePart(header); err != nil {
return nil, err
}
// Write the text
if err := quotePrintEncode(buff, e.HTML); err != nil {
return nil, err
}
}
if err := subWriter.Close(); err != nil {
return nil, err
}
}
// Create attachment part, if necessary
for _, a := range e.Attachments {
ap, err := w.CreatePart(a.Header)
if err != nil {
return nil, err
}
// Write the base64Wrapped content to the part
base64Wrap(ap, a.Content)
}
if err := w.Close(); err != nil {
return nil, err
}
return buff.Bytes(), nil
}
// Send a multipart/related POST request to OpenOCR API endpoint to decode image data
func (c HttpClient) DecodeImageReader(imageReader io.Reader, ocrRequest OcrRequest) (string, error) {
// create JSON for POST reqeust
jsonBytes, err := json.Marshal(ocrRequest)
if err != nil {
return "", err
}
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
mimeHeader := textproto.MIMEHeader{}
mimeHeader.Set("Content-Type", "application/json")
part, err := writer.CreatePart(mimeHeader)
if err != nil {
logg.LogTo("OCRCLIENT", "Unable to create json multipart part")
return "", err
}
_, err = part.Write(jsonBytes)
if err != nil {
logg.LogTo("OCRCLIENT", "Unable to write json multipart part")
return "", err
}
partHeaders := textproto.MIMEHeader{}
// TODO: pass these vals in instead of hardcoding
partHeaders.Set("Content-Type", "image/png")
partHeaders.Set("Content-Disposition", "attachment; filename=\"attachment.txt\".")
partAttach, err := writer.CreatePart(partHeaders)
if err != nil {
logg.LogTo("OCRCLIENT", "Unable to create image multipart part")
return "", err
}
_, err = io.Copy(partAttach, imageReader)
if err != nil {
logg.LogTo("OCRCLIENT", "Unable to write image multipart part")
return "", err
}
err = writer.Close()
if err != nil {
logg.LogTo("OCRCLIENT", "Error closing writer")
return "", err
}
// create a client
client := &http.Client{}
// create POST request
apiUrl := c.OcrApiFileUploadEndpointUrl()
req, err := http.NewRequest("POST", apiUrl, bytes.NewReader(body.Bytes()))
if err != nil {
logg.LogTo("OCRCLIENT", "Error creating POST request")
return "", err
}
// set content type
contentType := fmt.Sprintf("multipart/related; boundary=%q", writer.Boundary())
req.Header.Set("Content-Type", contentType)
// TODO: code below is all duplicated with DecodeImageUrl()
// send POST request
resp, err := client.Do(req)
if err != nil {
logg.LogTo("OCRCLIENT", "Error sending POST request")
return "", err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return "", fmt.Errorf("Got error status response: %d", resp.StatusCode)
}
respBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
logg.LogTo("OCRCLIENT", "Error reading response")
return "", err
}
return string(respBytes), nil
}
func (e *Email) WriteTo(w io.Writer) (int64, error) {
wc := &writeCounter{w: w}
w = wc
mpw := multipart.NewWriter(wc)
// Write top-level headers.
headers := e.Header
headers.Set("Content-Type", fmt.Sprintf(`multipart/alternative; boundary="%s"`, mpw.Boundary()))
headers.Set("MIME-Version", "1.0")
for k, vv := range headers {
for _, v := range vv {
if _, err := fmt.Fprintf(w, "%s: %s\n", k, v); err != nil {
return wc.n, err
}
}
}
if _, err := fmt.Fprintln(w); err != nil {
return wc.n, err
}
// Write text part.
textHeader := textproto.MIMEHeader{}
textHeader.Set("Content-Type", `text/plain; charset="utf-8"; format="fixed"`)
textHeader.Set("Content-Transfer-Encoding", "quoted-printable")
pw, err := mpw.CreatePart(textHeader)
if err != nil {
return wc.n, fmt.Errorf("create text part: %s", err)
}
qpw := quotedprintable.NewWriter(pw)
if _, err := qpw.Write(e.Text); err != nil {
return wc.n, fmt.Errorf("write text part: %s", err)
}
if err := qpw.Close(); err != nil {
return wc.n, fmt.Errorf("close text part: %s", err)
}
// Write top-level HTML multipart/related headers.
// Chicken-and-egg: we need the inner multipart boundary to write the outer
// part headers, but we can't get the boundary until we've constructed
// multipart.Writer with a part writer. We'll generate the boundary ahead
// of time and then manually supply it to the inner multipart.Writer.
htmlRelatedBoundary := multipart.NewWriter(nil).Boundary()
htmlRelatedHeader := textproto.MIMEHeader{}
htmlRelatedHeader.Set("Content-Type", fmt.Sprintf(`multipart/related; boundary="%s"`, htmlRelatedBoundary))
htmlRelatedHeader.Set("MIME-Version", "1.0")
pw, err = mpw.CreatePart(htmlRelatedHeader)
if err != nil {
return wc.n, fmt.Errorf("create html multipart part: %s", err)
}
// Create inner multipart data for HTML with attachments.
htmlmpw := multipart.NewWriter(pw)
htmlmpw.SetBoundary(htmlRelatedBoundary)
if err != nil {
return wc.n, fmt.Errorf("set html multipart boundary: %s", err)
}
// Write html part.
htmlHeader := textproto.MIMEHeader{}
htmlHeader.Set("Content-Type", `text/html; charset="utf-8"`)
htmlHeader.Set("Content-Transfer-Encoding", "quoted-printable")
htmlpw, err := htmlmpw.CreatePart(htmlHeader)
if err != nil {
return wc.n, fmt.Errorf("create html part: %s", err)
}
qpw = quotedprintable.NewWriter(htmlpw)
if _, err := qpw.Write(e.HTML); err != nil {
return wc.n, fmt.Errorf("write html part: %s", err)
}
if err := qpw.Close(); err != nil {
return wc.n, fmt.Errorf("close html part: %s", err)
}
// Write attachments.
for _, att := range e.Attachments {
attHeader := textproto.MIMEHeader{}
attHeader.Set("Content-ID", fmt.Sprintf("<%s>", att.ContentID))
attHeader.Set("Content-Type", mime.TypeByExtension(filepath.Ext(att.Name)))
attHeader.Set("Content-Transfer-Encoding", "base64")
attHeader.Set("Content-Disposition", fmt.Sprintf(`inline; filename="%s"`, att.Name))
htmlpw, err := htmlmpw.CreatePart(attHeader)
if err != nil {
return wc.n, fmt.Errorf("create attachment %s: %s", att.Name, err)
}
b64w := base64.NewEncoder(base64.StdEncoding, htmlpw)
if _, err := b64w.Write(att.Content); err != nil {
return wc.n, fmt.Errorf("write attachment %s: %s", att.Name, err)
}
if err := b64w.Close(); err != nil {
return wc.n, fmt.Errorf("close attachment %s: %s", att.Name, err)
}
}
// Finalize HTML multipart/related.
if err := htmlmpw.Close(); err != nil {
return wc.n, fmt.Errorf("multipart close: %s", err)
}
// Finalize.
if err := mpw.Close(); err != nil {
return wc.n, fmt.Errorf("multipart close: %s", err)
}
return wc.n, nil
}
// HTTP handler for a GET of a document
func (h *handler) handleGetDoc() error {
docid := h.PathVars()["docid"]
revid := h.getQuery("rev")
includeRevs := h.getBoolQuery("revs")
openRevs := h.getQuery("open_revs")
// What attachment bodies should be included?
var attachmentsSince []string = nil
if h.getBoolQuery("attachments") {
atts := h.getQuery("atts_since")
if atts != "" {
err := json.Unmarshal([]byte(atts), &attachmentsSince)
if err != nil {
return &base.HTTPError{http.StatusBadRequest, "bad atts_since"}
}
} else {
attachmentsSince = []string{}
}
}
if openRevs == "" {
// Single-revision GET:
value, err := h.db.GetRev(docid, revid, includeRevs, attachmentsSince)
if err != nil {
return err
}
if value == nil {
return kNotFoundError
}
h.setHeader("Etag", value["_rev"].(string))
if h.requestAccepts("application/json") {
h.writeJSON(value)
} else {
return h.writeMultipart(func(writer *multipart.Writer) error {
h.db.WriteMultipartDocument(value, writer)
return nil
})
}
} else if openRevs == "all" {
return &base.HTTPError{http.StatusNotImplemented, "open_revs=all unimplemented"} // TODO
} else {
var revids []string
err := json.Unmarshal([]byte(openRevs), &revids)
if err != nil {
return &base.HTTPError{http.StatusBadRequest, "bad open_revs"}
}
err = h.writeMultipart(func(writer *multipart.Writer) error {
for _, revid := range revids {
contentType := "application/json"
value, err := h.db.GetRev(docid, revid, includeRevs, attachmentsSince)
if err != nil {
value = db.Body{"missing": revid} //TODO: More specific error
contentType += `; error="true"`
}
jsonOut, _ := json.Marshal(value)
partHeaders := textproto.MIMEHeader{}
partHeaders.Set("Content-Type", contentType)
part, _ := writer.CreatePart(partHeaders)
part.Write(jsonOut)
}
return nil
})
return err
}
return nil
}
// BodyString converts the Email object to a string representation, excluding the headers
func (e *Email) BodyString(boundary string) (string, error) {
// TODO: better guess buffer size
buff := bytes.NewBuffer(make([]byte, 0, 4096))
w := multipart.NewWriter(buff)
if boundary != "" {
w.SetBoundary(boundary)
}
// Start the multipart/mixed part
//fmt.Fprintf(buff, "--%s\r\n", w.Boundary())
header := textproto.MIMEHeader{}
// Check to see if there is a Text or HTML field
if len(e.Text) > 0 || len(e.HTML) > 0 {
//subWriter := multipart.NewWriter(buff)
// Create the multipart alternative part
//header.Set("Content-Type", fmt.Sprintf("multipart/alternative;\r\n boundary=%s\r\n", subWriter.Boundary()))
// Write the header
//headerToBytes(buff, header)
// Create the body sections
if len(e.Text) > 0 {
header.Set("Content-Type", fmt.Sprintf("text/plain; charset=UTF-8"))
header.Set("Content-Transfer-Encoding", "quoted-printable")
if _, err := w.CreatePart(header); err != nil {
return "", err
}
qp := quotedprintable.NewWriter(buff)
// Write the text
if _, err := qp.Write(e.Text); err != nil {
return "", err
}
if err := qp.Close(); err != nil {
return "", err
}
}
if len(e.HTML) > 0 {
header.Set("Content-Type", fmt.Sprintf("text/html; charset=UTF-8"))
header.Set("Content-Transfer-Encoding", "quoted-printable")
if _, err := w.CreatePart(header); err != nil {
return "", err
}
qp := quotedprintable.NewWriter(buff)
// Write the HTML
if _, err := qp.Write(e.HTML); err != nil {
return "", err
}
if err := qp.Close(); err != nil {
return "", err
}
}
if err := w.Close(); err != nil {
return "", err
}
}
// Create attachment part, if necessary
for _, a := range e.Attachments {
ap, err := w.CreatePart(a.Header)
if err != nil {
return "", err
}
// Write the base64Wrapped content to the part
base64Wrap(ap, a.Content)
}
if err := w.Close(); err != nil {
return "", err
}
return buff.String(), nil
}
// HTTP handler for a GET of a document
func (h *handler) handleGetDoc() error {
docid := h.PathVars()["docid"]
revid := h.getQuery("rev")
includeRevs := h.getBoolQuery("revs")
openRevs := h.getQuery("open_revs")
// What attachment bodies should be included?
var attachmentsSince []string = nil
if h.getBoolQuery("attachments") {
atts := h.getQuery("atts_since")
if atts != "" {
err := json.Unmarshal([]byte(atts), &attachmentsSince)
if err != nil {
return &base.HTTPError{http.StatusBadRequest, "bad atts_since"}
}
} else {
attachmentsSince = []string{}
}
}
if openRevs == "" {
// Single-revision GET:
value, err := h.db.GetRev(docid, revid, includeRevs, attachmentsSince)
if err != nil {
return err
}
if value == nil {
return kNotFoundError
}
h.setHeader("Etag", value["_rev"].(string))
hasBodies := (attachmentsSince != nil && value["_attachments"] != nil)
if h.requestAccepts("multipart/") && (hasBodies || !h.requestAccepts("application/json")) {
return h.writeMultipart(func(writer *multipart.Writer) error {
h.db.WriteMultipartDocument(value, writer)
return nil
})
} else {
h.writeJSON(value)
}
} else if openRevs == "all" {
return &base.HTTPError{http.StatusNotImplemented, "open_revs=all unimplemented"} // TODO
} else {
attachmentsSince = []string{}
var revids []string
err := json.Unmarshal([]byte(openRevs), &revids)
if err != nil {
return &base.HTTPError{http.StatusBadRequest, "bad open_revs"}
}
err = h.writeMultipart(func(writer *multipart.Writer) error {
for _, revid := range revids {
var content []byte
var contentType string
revBody, err := h.db.GetRev(docid, revid, includeRevs, attachmentsSince)
if err == nil && len(db.BodyAttachments(revBody)) > 0 {
// Write as multipart, including attachments:
var buffer bytes.Buffer
docWriter := multipart.NewWriter(&buffer)
contentType = fmt.Sprintf("multipart/related; boundary=%q",
docWriter.Boundary())
h.db.WriteMultipartDocument(revBody, docWriter)
docWriter.Close()
content = bytes.TrimRight(buffer.Bytes(), "\r\n")
} else {
// Write as JSON:
contentType = "application/json"
if err != nil {
revBody = db.Body{"missing": revid} //TODO: More specific error
contentType += `; error="true"`
}
content, _ = json.Marshal(revBody)
}
partHeaders := textproto.MIMEHeader{}
partHeaders.Set("Content-Type", contentType)
part, _ := writer.CreatePart(partHeaders)
part.Write(content)
}
return nil
})
return err
}
return nil
}
func (self *nntpConnection) handleLine(daemon *NNTPDaemon, code int, line string, conn *textproto.Conn) (err error) {
parts := strings.Split(line, " ")
var msgid string
if code == 0 && len(parts) > 1 {
msgid = parts[1]
} else {
msgid = parts[0]
}
if code == 238 {
if ValidMessageID(msgid) {
self.messageSetPendingState(msgid, "takethis")
// they want this article
self.takethis <- msgid
}
return
} else if code == 239 {
// successful TAKETHIS
log.Println(msgid, "sent via", self.name)
self.messageSetProcessed(msgid)
return
// TODO: remember success
} else if code == 431 {
// CHECK said we would like this article later
self.messageSetProcessed(msgid)
} else if code == 439 {
// TAKETHIS failed
log.Println(msgid, "was not sent to", self.name, "denied:", line)
self.messageSetProcessed(msgid)
// TODO: remember denial
} else if code == 438 {
// they don't want the article
// TODO: remeber rejection
self.messageSetProcessed(msgid)
} else {
// handle command
parts := strings.Split(line, " ")
if len(parts) > 1 {
cmd := strings.ToUpper(parts[0])
if cmd == "MODE" {
mode := strings.ToUpper(parts[1])
if mode == "READER" {
// reader mode
self.mode = "READER"
log.Println(self.name, "switched to reader mode")
if self.authenticated {
conn.PrintfLine("200 Posting Permitted")
} else {
conn.PrintfLine("201 No posting Permitted")
}
} else if mode == "STREAM" && self.authenticated {
// wut? we're already in streaming mode
log.Println(self.name, "already in streaming mode")
conn.PrintfLine("203 Streaming enabled brah")
} else {
// invalid
log.Println(self.name, "got invalid mode request", parts[1])
conn.PrintfLine("501 invalid mode variant:", parts[1])
}
} else if cmd == "QUIT" {
// quit command
conn.PrintfLine("")
// close our connection and return
conn.Close()
return
} else if cmd == "AUTHINFO" {
if len(parts) > 1 {
auth_cmd := strings.ToUpper(parts[1])
if auth_cmd == "USER" {
// first part
self.username = parts[2]
// next phase is PASS
conn.PrintfLine("381 Password required")
} else if auth_cmd == "PASS" {
if len(self.username) == 0 {
conn.PrintfLine("482 Authentication commands issued out of sequence")
} else {
// try login
var valid bool
valid, err = daemon.database.CheckNNTPUserExists(self.username)
if valid {
valid, err = daemon.database.CheckNNTPLogin(self.username, line[14:])
}
if valid {
// valid login
self.authenticated = true
conn.PrintfLine("281 Authentication accepted")
} else if err == nil {
// invalid login
conn.PrintfLine("481 Authentication rejected")
} else {
// there was an error
// logit
log.Println(self.name, "error while logging in as", self.username, err)
conn.PrintfLine("501 error while logging in")
}
}
}
} else {
// wut ?
// wrong legnth of parametrs
}
} else if cmd == "CHECK" {
// handle check command
msgid := parts[1]
if self.mode != "STREAM" {
// we can't we are not in streaming mode
conn.PrintfLine("431 %s", msgid)
return
}
// have we seen this article?
if daemon.database.HasArticle(msgid) {
// yeh don't want it
conn.PrintfLine("438 %s", msgid)
} else if daemon.database.ArticleBanned(msgid) {
// it's banned we don't want it
conn.PrintfLine("438 %s", msgid)
} else {
// yes we do want it and we don't have it
conn.PrintfLine("238 %s", msgid)
}
} else if cmd == "TAKETHIS" {
// handle takethis command
var hdr textproto.MIMEHeader
var reason string
var ban bool
// read the article header
r := bufio.NewReader(conn.DotReader())
hdr, err = readMIMEHeader(r)
if err == nil {
// check the header
reason, ban, err = self.checkMIMEHeader(daemon, hdr)
if len(reason) > 0 {
// discard, we do not want
code = 439
log.Println(self.name, "rejected", msgid, reason)
_, err = io.Copy(ioutil.Discard, r)
if ban {
err = daemon.database.BanArticle(msgid, reason)
}
} else if err == nil {
// check if we don't have the rootpost
reference := hdr.Get("References")
newsgroup := hdr.Get("Newsgroups")
if reference != "" && ValidMessageID(reference) && !daemon.store.HasArticle(reference) && !daemon.database.IsExpired(reference) {
log.Println(self.name, "got reply to", reference, "but we don't have it")
go daemon.askForArticle(ArticleEntry{reference, newsgroup})
}
// store message
err = self.storeMessage(daemon, hdr, r)
if err == nil {
code = 239
reason = "gotten"
} else {
code = 439
reason = err.Error()
}
} else {
// error?
// discard, we do not want
code = 439
log.Println(self.name, "rejected", msgid, reason)
_, err = io.Copy(ioutil.Discard, r)
if ban {
err = daemon.database.BanArticle(msgid, reason)
}
}
} else {
log.Println(self.name, "error reading mime header:", err)
code = 439
reason = "error reading mime header"
}
conn.PrintfLine("%d %s %s", code, msgid, reason)
} else if cmd == "ARTICLE" {
if !ValidMessageID(msgid) {
if len(self.group) > 0 {
n, err := strconv.Atoi(msgid)
if err == nil {
msgid, err = daemon.database.GetMessageIDForNNTPID(self.group, int64(n))
}
}
}
if ValidMessageID(msgid) && daemon.store.HasArticle(msgid) {
// we have it yeh
f, err := daemon.store.OpenMessage(msgid)
if err == nil {
conn.PrintfLine("220 %s", msgid)
dw := conn.DotWriter()
_, err = io.Copy(dw, f)
dw.Close()
f.Close()
} else {
// wtf?!
conn.PrintfLine("503 idkwtf happened: %s", err.Error())
}
} else {
// we dont got it
conn.PrintfLine("430 %s", msgid)
}
} else if cmd == "IHAVE" {
if !self.authenticated {
conn.PrintfLine("483 You have not authenticated")
} else {
// handle IHAVE command
msgid := parts[1]
if daemon.database.HasArticleLocal(msgid) || daemon.database.HasArticle(msgid) || daemon.database.ArticleBanned(msgid) {
// we don't want it
conn.PrintfLine("435 Article Not Wanted")
} else {
// gib we want
conn.PrintfLine("335 Send it plz")
r := bufio.NewReader(conn.DotReader())
hdr, err := readMIMEHeader(r)
if err == nil {
// check the header
var reason string
var ban bool
reason, ban, err = self.checkMIMEHeader(daemon, hdr)
if len(reason) > 0 {
// discard, we do not want
log.Println(self.name, "rejected", msgid, reason)
_, err = io.Copy(ioutil.Discard, r)
if ban {
_ = daemon.database.BanArticle(msgid, reason)
}
conn.PrintfLine("437 Rejected do not send again bro")
} else {
// check if we don't have the rootpost
reference := hdr.Get("References")
newsgroup := hdr.Get("Newsgroups")
if reference != "" && ValidMessageID(reference) && !daemon.store.HasArticle(reference) && !daemon.database.IsExpired(reference) {
log.Println(self.name, "got reply to", reference, "but we don't have it")
go daemon.askForArticle(ArticleEntry{reference, newsgroup})
}
err = self.storeMessage(daemon, hdr, r)
if err == nil {
conn.PrintfLine("235 We got it")
} else {
conn.PrintfLine("437 Transfer Failed %s", err.Error())
}
}
} else {
// error here
conn.PrintfLine("436 Transfer failed: " + err.Error())
}
}
}
} else if cmd == "LISTGROUP" {
// handle LISTGROUP
var group string
if len(parts) > 1 {
// parameters
group = parts[1]
} else {
group = self.group
}
if len(group) > 0 && newsgroupValidFormat(group) {
if daemon.database.HasNewsgroup(group) {
// we has newsgroup
var hi, lo int64
count, err := daemon.database.CountAllArticlesInGroup(group)
if err == nil {
hi, lo, err = daemon.database.GetLastAndFirstForGroup(group)
if err == nil {
conn.PrintfLine("211 %d %d %d %s list follows", count, lo, hi, group)
dw := conn.DotWriter()
idx := lo
for idx <= hi {
fmt.Fprintf(dw, "%d\r\n", idx)
idx++
}
dw.Close()
}
}
if err != nil {
log.Println("LISTGROUP fail", err)
conn.PrintfLine("500 error in LISTGROUP: %s", err.Error())
}
} else {
// don't has newsgroup
conn.PrintfLine("411 no such newsgroup")
}
} else {
conn.PrintfLine("412 no newsgroup selected")
}
} else if cmd == "NEWSGROUPS" {
// handle NEWSGROUPS
conn.PrintfLine("231 List of newsgroups follow")
dw := conn.DotWriter()
// get a list of every newsgroup
groups := daemon.database.GetAllNewsgroups()
// for each group
for _, group := range groups {
// get low/high water mark
lo, hi, err := daemon.database.GetLastAndFirstForGroup(group)
if err == nil {
// XXX: we ignore errors here :\
_, _ = io.WriteString(dw, fmt.Sprintf("%s %d %d y\n", group, lo, hi))
} else {
log.Println(self.name, "could not get low/high water mark for", group, err)
}
}
// flush dotwriter
dw.Close()
} else if cmd == "XOVER" {
// handle XOVER
if self.group == "" {
conn.PrintfLine("412 No newsgroup selected")
} else {
// handle xover command
// right now it's every article in group
models, err := daemon.database.GetPostsInGroup(self.group)
if err == nil {
conn.PrintfLine("224 Overview information follows")
dw := conn.DotWriter()
for idx, model := range models {
io.WriteString(dw, fmt.Sprintf("%.6d\t%s\t\"%s\" <%s@%s>\t%s\t%s\t%s\r\n", idx+1, model.Subject(), model.Name(), model.Name(), model.Frontend(), model.Date(), model.MessageID(), model.Reference()))
}
dw.Close()
} else {
log.Println(self.name, "error when getting posts in", self.group, err)
conn.PrintfLine("500 error, %s", err.Error())
}
}
} else if cmd == "HEAD" {
if len(self.group) == 0 {
// no group selected
conn.PrintfLine("412 No newsgroup slected")
} else {
// newsgroup is selected
// handle HEAD command
if len(parts) == 0 {
// we have no parameters
if len(self.selected_article) > 0 {
// we have a selected article
} else {
// no selected article
conn.PrintfLine("420 current article number is invalid")
}
} else {
// head command has 1 or more paramters
var n int64
var msgid string
var has bool
var code int
n, err = strconv.ParseInt(parts[1], 10, 64)
if err == nil {
// is a number
msgid, err = daemon.database.GetMessageIDForNNTPID(self.group, n)
if err == nil && len(msgid) > 0 {
has = daemon.database.HasArticleLocal(msgid)
}
if !has {
code = 423
}
} else if ValidMessageID(parts[1]) {
msgid = parts[1]
has = daemon.database.HasArticleLocal(msgid)
if has {
n, err = daemon.database.GetNNTPIDForMessageID(self.group, parts[1])
} else {
code = 430
}
}
if err == nil {
if has {
// we has
hdrs := daemon.store.GetHeaders(msgid)
if hdrs == nil {
// wtf can't load?
conn.PrintfLine("500 cannot load headers")
} else {
// headers loaded, send them
conn.PrintfLine("221 %d %s", n, msgid)
dw := conn.DotWriter()
err = writeMIMEHeader(dw, hdrs)
dw.Close()
hdrs = nil
}
} else if code > 0 {
// don't has
conn.PrintfLine("%d don't have article", code)
} else {
// invalid state
conn.PrintfLine("500 invalid state in HEAD, should have article but we don't")
}
} else {
// error occured
conn.PrintfLine("500 error in HEAD: %s", err.Error())
}
}
}
} else if cmd == "GROUP" {
// handle GROUP command
group := parts[1]
// check for newsgroup
if daemon.database.HasNewsgroup(group) {
// we have the group
self.group = group
// count posts
number := daemon.database.CountPostsInGroup(group, 0)
// get hi/low water marks
hi, low, err := daemon.database.GetLastAndFirstForGroup(group)
if err == nil {
// we gud
conn.PrintfLine("211 %d %d %d %s", number, low, hi, group)
} else {
// wtf error
log.Println(self.name, "error in GROUP command", err)
// still have to reply, send it bogus low/hi
conn.PrintfLine("211 %d 0 1 %s", number, group)
}
} else {
// no such group
conn.PrintfLine("411 No Such Newsgroup")
}
} else if cmd == "LIST" && parts[1] == "NEWSGROUPS" {
conn.PrintfLine("215 list of newsgroups follows")
// handle list command
groups := daemon.database.GetAllNewsgroups()
dw := conn.DotWriter()
for _, group := range groups {
last, first, err := daemon.database.GetLastAndFirstForGroup(group)
if err == nil {
io.WriteString(dw, fmt.Sprintf("%s %d %d y\r\n", group, first, last))
} else {
log.Println("cannot get last/first ids for group", group, err)
}
}
dw.Close()
} else if cmd == "STAT" {
if len(self.group) == 0 {
if len(parts) == 2 {
// parameter given
msgid := parts[1]
// check for article
if ValidMessageID(msgid) && daemon.database.HasArticleLocal(msgid) {
// valid message id
var n int64
n, err = daemon.database.GetNNTPIDForMessageID(self.group, msgid)
// exists
conn.PrintfLine("223 %d %s", n, msgid)
err = nil
} else {
conn.PrintfLine("430 No article with that message-id")
}
} else {
conn.PrintfLine("412 No newsgroup selected")
}
} else if daemon.database.HasNewsgroup(self.group) {
// group specified
if len(parts) == 2 {
// parameter specified
var msgid string
var n int64
n, err = strconv.ParseInt(parts[1], 10, 64)
if err == nil {
msgid, err = daemon.database.GetMessageIDForNNTPID(self.group, n)
if err != nil {
// error getting id
conn.PrintfLine("500 error getting nntp article id: %s", err.Error())
return
}
} else {
// message id
msgid = parts[1]
}
if ValidMessageID(msgid) && daemon.database.HasArticleLocal(msgid) {
conn.PrintfLine("223 %d %s", n, msgid)
} else if n == 0 {
// was a message id
conn.PrintfLine("430 no such article")
} else {
// was an article number
conn.PrintfLine("423 no article with that number")
}
} else {
conn.PrintfLine("420 Current article number is invalid")
}
} else {
conn.PrintfLine("500 invalid daemon state, got STAT with group set but we don't have that group now?")
}
} else {
log.Println(self.name, "invalid command recv'd", cmd)
conn.PrintfLine("500 Invalid command: %s", cmd)
}
} else {
if line == "LIST" {
conn.PrintfLine("215 list of newsgroups follows")
// handle list command
groups := daemon.database.GetAllNewsgroups()
dw := conn.DotWriter()
for _, group := range groups {
last, first, err := daemon.database.GetLastAndFirstForGroup(group)
if err == nil {
io.WriteString(dw, fmt.Sprintf("%s %d %d y\r\n", group, first, last))
} else {
log.Println("cannot get last/first ids for group", group, err)
}
}
dw.Close()
} else if line == "POST" {
if !self.authenticated {
// needs tls to work if not logged in
conn.PrintfLine("440 Posting Not Allowed")
} else {
// handle POST command
conn.PrintfLine("340 Yeeeh postit yo; end with <CR-LF>.<CR-LF>")
var hdr textproto.MIMEHeader
hdr, err = readMIMEHeader(conn.R)
var success, gotten bool
var reason string
if err == nil {
if getMessageID(hdr) == "" {
hdr.Set("Message-ID", genMessageID(daemon.instance_name))
}
msgid = getMessageID(hdr)
hdr.Set("Date", timeNowStr())
ipaddr, _, _ := net.SplitHostPort(self.addr.String())
if len(ipaddr) > 0 {
// inject encrypted ip for poster
encaddr, err := daemon.database.GetEncAddress(ipaddr)
if err == nil {
hdr.Set("X-Encrypted-Ip", encaddr)
}
}
reason, _, err = self.checkMIMEHeader(daemon, hdr)
success = reason == "" && err == nil
if success {
r := bufio.NewReader(conn.DotReader())
reference := hdr.Get("References")
newsgroup := hdr.Get("Newsgroups")
if reference != "" && ValidMessageID(reference) {
if !daemon.store.HasArticle(reference) && !daemon.database.IsExpired(reference) {
log.Println(self.name, "got reply to", reference, "but we don't have it")
go daemon.askForArticle(ArticleEntry{reference, newsgroup})
}
} else if reference != "" {
// bad message id
reason = "cannot reply with invalid reference, maybe you are replying to a reply?"
success = false
}
if success && daemon.database.HasNewsgroup(newsgroup) {
err = self.storeMessage(daemon, hdr, r)
}
}
}
if success && gotten && err == nil {
// all gud
conn.PrintfLine("240 We got it, thnkxbai")
} else {
// failed posting
if err != nil {
log.Println(self.name, "failed nntp POST", err)
}
conn.PrintfLine("441 Posting Failed %s", reason)
}
}
} else {
conn.PrintfLine("500 wut?")
}
}
}
return
}