func Fuzz(data []byte) int { sdata := string(data) mt, params, err := mime.ParseMediaType(sdata) if err != nil { return 0 } sdata1 := mime.FormatMediaType(mt, params) mt1, params1, err := mime.ParseMediaType(sdata1) if err != nil { if err.Error() == "mime: no media type" { // https://github.com/golang/go/issues/11289 return 0 } if err.Error() == "mime: invalid media parameter" { // https://github.com/golang/go/issues/11290 return 0 } fmt.Printf("%q(%q, %+v) -> %q\n", sdata, mt, params, sdata1) panic(err) } if !fuzz.DeepEqual(mt, mt1) { fmt.Printf("%q -> %q\n", mt, mt1) panic("mediatype changed") } if !fuzz.DeepEqual(params, params1) { fmt.Printf("%+v -> %+v\n", params, params1) panic("params changed") } return 1 }
func init() { m = New() m.AddFunc("dummy/copy", func(m Minifier, mediatype string, w io.Writer, r io.Reader) error { io.Copy(w, r) return nil }) m.AddFunc("dummy/nil", func(m Minifier, mediatype string, w io.Writer, r io.Reader) error { return nil }) m.AddFunc("dummy/err", func(m Minifier, mediatype string, w io.Writer, r io.Reader) error { return errDummy }) m.AddFunc("dummy/charset", func(m Minifier, mediatype string, w io.Writer, r io.Reader) error { _, param, _ := mime.ParseMediaType(mediatype) w.Write([]byte(param["charset"])) return nil }) m.AddFunc("dummy/params", func(m Minifier, mediatype string, w io.Writer, r io.Reader) error { _, param, _ := mime.ParseMediaType(mediatype) return m.Minify(param["type"]+"/"+param["sub"], w, r) }) m.AddFunc("type/sub", func(m Minifier, mediatype string, w io.Writer, r io.Reader) error { w.Write([]byte("type/sub")) return nil }) m.AddFuncRegexp(regexp.MustCompile("^type/.+$"), func(m Minifier, mediatype string, w io.Writer, r io.Reader) error { w.Write([]byte("type/*")) return nil }) m.AddFuncRegexp(regexp.MustCompile("^.+/.+$"), func(m Minifier, mediatype string, w io.Writer, r io.Reader) error { w.Write([]byte("*/*")) return nil }) }
// fixContentType checks resp's Content-Type header. If it is missing or too // general, it sniffs the body content to find the content type, and updates // the Content-Type header. func fixContentType(resp *http.Response) { ct, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) if err != nil || !strings.Contains(ct, "/") { ct = "" } switch ct { case "unknown/unknown", "application/unknown", "*/*", "", "application/octet-stream": // These types tend to be used for content whose type is unknown, // so we should try to second-guess them. preview := make([]byte, 512) n, _ := resp.Body.Read(preview) preview = preview[:n] if n > 0 { ct, _, _ = mime.ParseMediaType(http.DetectContentType(preview)) if ct != "application/octet-stream" { resp.Header.Set("Content-Type", ct) } // Make the preview data available for re-reading. var rc struct { io.Reader io.Closer } rc.Reader = io.MultiReader(bytes.NewBuffer(preview), resp.Body) rc.Closer = resp.Body resp.Body = rc } } }
// parseMIMEParts will recursively walk a MIME entity and return a []mime.Part containing // each (flattened) mime.Part found. // It is important to note that there are no limits to the number of recursions, so be // careful when parsing unknown MIME structures! func parseMIMEParts(hs textproto.MIMEHeader, b io.Reader) ([]*part, error) { var ps []*part // If no content type is given, set it to the default if _, ok := hs["Content-Type"]; !ok { hs.Set("Content-Type", defaultContentType) } ct, params, err := mime.ParseMediaType(hs.Get("Content-Type")) if err != nil { return ps, err } // If it's a multipart email, recursively parse the parts if strings.HasPrefix(ct, "multipart/") { if _, ok := params["boundary"]; !ok { return ps, ErrMissingBoundary } mr := multipart.NewReader(b, params["boundary"]) for { var buf bytes.Buffer p, err := mr.NextPart() if err == io.EOF { break } if err != nil { return ps, err } if _, ok := p.Header["Content-Type"]; !ok { p.Header.Set("Content-Type", defaultContentType) } subct, _, err := mime.ParseMediaType(p.Header.Get("Content-Type")) if strings.HasPrefix(subct, "multipart/") { sps, err := parseMIMEParts(p.Header, p) if err != nil { return ps, err } ps = append(ps, sps...) } else { // Otherwise, just append the part to the list // Copy the part data into the buffer if _, err := io.Copy(&buf, p); err != nil { return ps, err } ps = append(ps, &part{body: buf.Bytes(), header: p.Header}) } } } else { // If it is not a multipart email, parse the body content as a single "part" var buf bytes.Buffer if _, err := io.Copy(&buf, b); err != nil { return ps, err } ps = append(ps, &part{body: buf.Bytes(), header: hs}) } return ps, nil }
// GetBodyStr returns plain string crawled. func (self *Context) initText() { // 采用surf内核下载时,尝试自动转码 if self.Request.DownloaderID == request.SURF_ID { var contentType, pageEncode string // 优先从响应头读取编码类型 contentType = self.Response.Header.Get("Content-Type") if _, params, err := mime.ParseMediaType(contentType); err == nil { if cs, ok := params["charset"]; ok { pageEncode = strings.ToLower(strings.TrimSpace(cs)) } } // 响应头未指定编码类型时,从请求头读取 if len(pageEncode) == 0 { contentType = self.Request.Header.Get("Content-Type") if _, params, err := mime.ParseMediaType(contentType); err == nil { if cs, ok := params["charset"]; ok { pageEncode = strings.ToLower(strings.TrimSpace(cs)) } } } switch pageEncode { // 不做转码处理 case "", "utf8", "utf-8", "unicode-1-1-utf-8": default: // 指定了编码类型,但不是utf8时,自动转码为utf8 // get converter to utf-8 // Charset auto determine. Use golang.org/x/net/html/charset. Get response body and change it to utf-8 destReader, err := charset.NewReaderLabel(pageEncode, self.Response.Body) if err == nil { self.text, err = ioutil.ReadAll(destReader) if err == nil { self.Response.Body.Close() return } else { logs.Log.Warning(" * [convert][%v]: %v (ignore transcoding)\n", self.GetUrl(), err) } } else { logs.Log.Warning(" * [convert][%v]: %v (ignore transcoding)\n", self.GetUrl(), err) } } } // 不做转码处理 var err error self.text, err = ioutil.ReadAll(self.Response.Body) self.Response.Body.Close() if err != nil { panic(err.Error()) return } }
// binMIME handles the special case where the only content of the message is an // attachment. It is called by ParseMIME when needed. func binMIME(mailMsg *mail.Message) (*MIMEBody, error) { // Determine mediatype ctype := mailMsg.Header.Get("Content-Type") mediatype, mparams, err := mime.ParseMediaType(ctype) if err != nil { mediatype = "attachment" } // Build the MIME part representing most of this message p := NewMIMEPart(nil, mediatype) content, err := decodeSection(mailMsg.Header.Get("Content-Transfer-Encoding"), mailMsg.Body) if err != nil { return nil, err } p.SetContent(content) p.SetHeader(make(textproto.MIMEHeader, 4)) // Determine and set headers for: content disposition, filename and // character set disposition, dparams, err := mime.ParseMediaType(mailMsg.Header.Get("Content-Disposition")) if err == nil { // Disposition is optional p.SetDisposition(disposition) p.SetFileName(DecodeHeader(dparams["filename"])) } if p.FileName() == "" && mparams["name"] != "" { p.SetFileName(DecodeHeader(mparams["name"])) } if p.FileName() == "" && mparams["file"] != "" { p.SetFileName(DecodeHeader(mparams["file"])) } if p.Charset() == "" { p.SetCharset(mparams["charset"]) } p.Header().Set("Content-Type", mailMsg.Header.Get("Content-Type")) p.Header().Set("Content-Disposition", mailMsg.Header.Get("Content-Disposition")) // Add our part to the appropriate section of MIMEBody m := &MIMEBody{ header: mailMsg.Header, Root: NewMIMEPart(nil, mediatype), IsTextFromHTML: false, } if disposition == "inline" { m.Inlines = append(m.Inlines, p) } else { m.Attachments = append(m.Attachments, p) } return m, err }
func (msg Jmessage) DecBody() (mailbody []byte, err error) { contentType := msg.Header.Get("Content-Type") if contentType == "" { return readPlainText(map[string][]string(msg.Header), msg.Body) } mediaType, params, err := mime.ParseMediaType(contentType) debug.Printf("MediaType: %s, %v\n", mediaType, params) if err != nil { debug.Printf("Error: %v", err) return } mailbody = make([]byte, 0) if strings.HasPrefix(mediaType, MEDIATYPE_MULTI) { // multipart/... mr := multipart.NewReader(msg.Body, params["boundary"]) for { p, err := mr.NextPart() if err == io.EOF { return mailbody, err } if err != nil { debug.Printf("Error: %v", err) } mt, _, err := mime.ParseMediaType(p.Header.Get("Content-Type")) if err != nil { debug.Printf("Error: %v", err) return nil, err } debug.Printf("MediaType-inner: %s\n", mt) if strings.HasPrefix(mt, MEDIATYPE_TEXT) { // text/plain return readPlainText(p.Header, p) } if strings.HasPrefix(mt, MEDIATYPE_MULTI_ALT) { // multipart/alternative return readAlternative(p) } // slurp, err := ioutil.ReadAll(p) // if err != nil { // debug.Printf("Error: %v", err) // } // for key, values := range p.Header { // debug.Printf("%s:%v", key, values) // } // fmt.Printf("Slurping...: %q\n", slurp) } } else { // text/plain, text/html return readPlainText(map[string][]string(msg.Header), msg.Body) } return }
// IsAttachment returns true, if the given header defines an attachment. // First it checks, if the Content-Disposition header defines an attachement. // If this test is false, the Content-Type header is checked. // // Valid Attachment-Headers: // // Content-Disposition: attachment; filename="frog.jpg" // Content-Type: attachment; filename="frog.jpg" // func IsAttachment(header mail.Header) bool { mediatype, _, _ := mime.ParseMediaType(header.Get("Content-Disposition")) if strings.ToLower(mediatype) == "attachment" { return true } mediatype, _, _ = mime.ParseMediaType(header.Get("Content-Type")) if strings.ToLower(mediatype) == "attachment" { return true } return false }
// parseBody will accept a a raw body, break it into all its parts and then convert the // message to UTF-8 from whatever charset it may have. func parseBody(header mail.Header, body []byte) (html []byte, text []byte, isMultipart bool, err error) { var mediaType string var params map[string]string mediaType, params, err = mime.ParseMediaType(header.Get("Content-Type")) if err != nil { return } if strings.HasPrefix(mediaType, "multipart/") { isMultipart = true mr := multipart.NewReader(bytes.NewReader(body), params["boundary"]) for { p, err := mr.NextPart() if err == io.EOF { break } if err != nil { break } slurp, err := ioutil.ReadAll(p) if err != nil { break } partMediaType, partParams, err := mime.ParseMediaType(p.Header.Get("Content-Type")) if err != nil { break } var htmlT, textT []byte htmlT, textT, err = parsePart(partMediaType, partParams["charset"], p.Header.Get("Content-Transfer-Encoding"), slurp) if len(htmlT) > 0 { html = htmlT } else { text = textT } } } else { splitBody := bytes.SplitN(body, headerSplitter, 2) if len(splitBody) < 2 { err = errors.New("unexpected email format. (single part and no \\r\\n\\r\\n separating headers/body") return } body = splitBody[1] html, text, err = parsePart(mediaType, params["charset"], header.Get("Content-Transfer-Encoding"), body) } return }
func (m *meta) parseContentType(ct string) error { if ct == "" { m.MediaType = "application/octet-stream" return nil } mediatype, params, err := mime.ParseMediaType(ct) if err != nil { return err } if mediatype == "multipart/form-data" { boundary, ok := params["boundary"] if !ok { return errors.New("meta: boundary not defined") } m.MediaType = mediatype m.Boundary = boundary } else { m.MediaType = "application/octet-stream" } return nil }
// GetContentType returns content-type of the message func (m *Email) GetContentType() (contentType string, params map[string]string, err error) { hdrCt, err := m.GetHeader("Content-Type") if err != nil { return } return mime.ParseMediaType(hdrCt) }
// also from goiardi calc and encodebody data func calcBodyHash(r *http.Request) (string, error) { var bodyStr string var err error if r.Body == nil { bodyStr = "" } else { save := r.Body save, r.Body, err = drainBody(r.Body) if err != nil { return "", err } mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) if err != nil { return "", err } if strings.HasPrefix(mediaType, "multipart/form-data") { bodyStr, err = readFileFromRequest(r, params["boundary"]) if err != nil { return "", err } } else { buf := new(bytes.Buffer) buf.ReadFrom(r.Body) bodyStr = buf.String() } r.Body = save } chkHash := hashStr(bodyStr) return chkHash, err }
func matchesContentType(contentType, expectedType string) bool { mimetype, _, err := mime.ParseMediaType(contentType) if err != nil { utils.Errorf("Error parsing media type: %s error: %s", contentType, err.Error()) } return err == nil && mimetype == expectedType }
func (t *Transport) updateToken(tok *Token, v url.Values) error { v.Set("client_id", t.ClientId) v.Set("client_secret", t.ClientSecret) r, err := (&http.Client{Transport: t.transport()}).PostForm(t.TokenURL, v) if err != nil { return err } defer r.Body.Close() if r.StatusCode != 200 { return OAuthError{"updateToken", r.Status} } var b struct { Access string `json:"access_token"` Refresh string `json:"refresh_token"` ExpiresIn time.Duration `json:"expires_in"` Id string `json:"id_token"` } content, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type")) switch content { case "application/x-www-form-urlencoded", "text/plain": body, err := ioutil.ReadAll(r.Body) if err != nil { return err } vals, err := url.ParseQuery(string(body)) if err != nil { return err } b.Access = vals.Get("access_token") b.Refresh = vals.Get("refresh_token") b.ExpiresIn, _ = time.ParseDuration(vals.Get("expires_in") + "s") b.Id = vals.Get("id_token") default: if err = json.NewDecoder(r.Body).Decode(&b); err != nil { return err } // The JSON parser treats the unitless ExpiresIn like 'ns' instead of 's' as above, // so compensate here. b.ExpiresIn *= time.Second } tok.AccessToken = b.Access // Don't overwrite `RefreshToken` with an empty value if len(b.Refresh) > 0 { tok.RefreshToken = b.Refresh } if b.ExpiresIn == 0 { tok.Expiry = time.Time{} } else { tok.Expiry = time.Now().Add(b.ExpiresIn) } if b.Id != "" { if tok.Extra == nil { tok.Extra = make(map[string]string) } tok.Extra["id_token"] = b.Id } return nil }
func getEncoding(resp *gohttp.Response, body []byte) int { charsetTable := map[string]int{ "gb18030": CP_GBK, "gb2312": CP_GBK, "hz": CP_GBK, "big5": CP_BIG5, "shift_jis": CP_SHIFT_JIS, "euc-jp": CP_SHIFT_JIS, "utf-8": CP_UTF8, } ctype := resp.Header.Get("Content-Type") if len(ctype) == 0 { return CP_UTF8 } _, params, err := mime.ParseMediaType(ctype) if err != nil { return CP_UTF8 } charset, ok := params["charset"] if ok == false { return CP_UTF8 } encoding, ok := charsetTable[string(String(charset).ToLower())] if ok == false { encoding = CP_UTF8 } return encoding }
func MatchesContentType(contentType, expectedType string) bool { mimetype, _, err := mime.ParseMediaType(contentType) if err != nil { fmt.Printf("Error parsing media type: %s error: %v", contentType, err) } return err == nil && mimetype == expectedType }
// Reads & parses the request body, handling either JSON or multipart. func (h *handler) readDocument() (db.Body, error) { contentType, attrs, _ := mime.ParseMediaType(h.rq.Header.Get("Content-Type")) switch contentType { case "", "application/json": return h.readJSON() case "multipart/related": if DebugMultipart { raw, err := h.readBody() if err != nil { return nil, err } reader := multipart.NewReader(bytes.NewReader(raw), attrs["boundary"]) body, err := db.ReadMultipartDocument(reader) if err != nil { ioutil.WriteFile("GatewayPUT.mime", raw, 0600) base.Warn("Error reading MIME data: copied to file GatewayPUT.mime") } return body, err } else { reader := multipart.NewReader(h.requestBody, attrs["boundary"]) return db.ReadMultipartDocument(reader) } default: return nil, base.HTTPErrorf(http.StatusUnsupportedMediaType, "Invalid content type %s", contentType) } }
// isMime returns true if the string is a properly formed mime-type func isMime(smime string) bool { _, _, err := mime.ParseMediaType(smime) if err != nil { return false } return true }
// extractBoundary extract boundary string in contentType. // It returns empty string if no valid boundary found func extractBoundary(contentType string) string { _, params, err := mime.ParseMediaType(contentType) if err == nil { return params["boundary"] } return "" }
// boudary string found in the Content-Type header. func (m *Message) boundary() string { _, params, _ := mime.ParseMediaType(m.ContentType()) if value, exists := params["boundary"]; exists { return value } return "" }
func NewFileFromPart(part *multipart.Part) (File, error) { f := &MultipartFile{ Part: part, } contentType := part.Header.Get(contentTypeHeader) if contentType == applicationSymlink { out, err := ioutil.ReadAll(part) if err != nil { return nil, err } return &Symlink{ Target: string(out), name: f.FileName(), }, nil } var err error f.Mediatype, _, err = mime.ParseMediaType(contentType) if err != nil { return nil, err } return f, nil }
func logAccess(req *http.Request, resp *http.Response, contentLength int, pruned bool, user string, tally map[rule]int, scores map[string]int, rule ACLActionRule, title string, ignored []string, userAgent string) { modified := "" if pruned { modified = "pruned" } status := 0 if resp != nil { status = resp.StatusCode } if rule.Action == "" { rule.Action = "allow" } var contentType string if resp != nil { contentType = resp.Header.Get("Content-Type") } if ct2, _, err := mime.ParseMediaType(contentType); err == nil { contentType = ct2 } accessLogChan <- toStrings(time.Now().Format("2006-01-02 15:04:05"), user, rule.Action, req.URL, req.Method, status, contentType, contentLength, modified, listTally(stringTally(tally)), listTally(scores), rule.Conditions(), title, strings.Join(ignored, ","), userAgent) }
func (m *Message) parse(r io.Reader) error { msg, err := mail.ReadMessage(r) if err != nil { return err } m.Subject = msg.Header.Get("Subject") m.To = msg.Header.Get("To") mediaType, params, err := mime.ParseMediaType(msg.Header.Get("Content-Type")) if err != nil || !strings.HasPrefix(mediaType, "multipart/") { slurp, _ := ioutil.ReadAll(msg.Body) m.Body = string(slurp) return nil } if err := m.parseMultipart(msg.Body, params["boundary"]); err != nil { return err } // If we didn't find a text/plain body, pick the first body we did find. if m.Body == "" { for _, body := range m.bodies { if body != "" { m.Body = body break } } } return nil }
func NewDecoderFromResponse(res *http.Response) (*Decoder, error) { _, param, err := mime.ParseMediaType(res.Header.Get("Content-Type")) if err != nil { return nil, err } return NewDecoder(res.Body, param["boundary"]), nil }
func resolveHTTP(uri *URI) (*e3x.Identity, error) { resp, err := http.Get("http://" + uri.Canonical + "/.well-known/mesh.json") if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != 200 { return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) } typ := resp.Header.Get("Content-Type") typ, _, err = mime.ParseMediaType(typ) if err != nil { return nil, err } if typ != "application/json" && typ != "text/json" { return nil, fmt.Errorf("unexpected content type: %q", typ) } var ident *e3x.Identity err = json.NewDecoder(resp.Body).Decode(&ident) if err != nil { return nil, err } return ident, nil }
// ForEach ... func (r *GetObjectResponse) ForEach(result GetObjectResult) error { resp := r.response defer resp.Body.Close() mediaType, params, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) if err != nil { return err } // its not multipart, just leave if !strings.HasPrefix(mediaType, "multipart/") { return result(NewObjectFromStream(textproto.MIMEHeader(resp.Header), resp.Body)) } // its multipart, need to break it up partsReader := multipart.NewReader(resp.Body, params["boundary"]) for { part, err := partsReader.NextPart() switch { case err == io.EOF: return nil case err != nil: return err } err = result(NewObjectFromStream(part.Header, part)) if err != nil { return err } } // return nil }
func selectMultiPart(msg *mail.Message, boundary, preferType string) (*mail.Message, error) { reader := multipart.NewReader(msg.Body, boundary) parts := make(map[string]*mail.Message) for part, err := reader.NextPart(); err != io.EOF; part, err = reader.NextPart() { if err != nil { return nil, err } header := mail.Header(part.Header) mediatype, _, err := mime.ParseMediaType(header.Get("Content-Type")) if err != nil { continue } if mediatype == preferType { return &mail.Message{header, part}, nil } types := strings.Split(mediatype, "/") if len(types) == 0 { continue } if _, ok := parts[types[0]]; !ok { body, err := ioutil.ReadAll(part) if err == nil { parts[types[0]] = &mail.Message{header, bytes.NewBuffer(body)} } } } types := strings.Split(preferType, "/") if part, ok := parts[types[0]]; ok { return part, nil } if part, ok := parts["multipart"]; ok { return part, nil } return nil, fmt.Errorf("No prefered part") }
// assumes base64'd func createAttachment(content_type, fname string, body io.Reader) NNTPAttachment { media_type, _, err := mime.ParseMediaType(content_type) if err == nil { a := new(nntpAttachment) dec := base64.NewDecoder(base64.StdEncoding, body) _, err = io.Copy(a, dec) if err == nil { a.header = make(textproto.MIMEHeader) a.mime = media_type + "; charset=UTF-8" idx := strings.LastIndex(fname, ".") a.ext = ".txt" if idx > 0 { a.ext = fname[idx:] } a.header.Set("Content-Disposition", `form-data; filename="`+fname+`"; name="attachment"`) a.header.Set("Content-Type", a.mime) a.header.Set("Content-Transfer-Encoding", "base64") h := a.Hash() hashstr := base32.StdEncoding.EncodeToString(h[:]) a.hash = h[:] a.filepath = hashstr + a.ext a.filename = fname return a } } return nil }
func (p *Part) parseContentDisposition() { v := p.Header.Get("Content-Disposition") p.disposition, p.dispositionParams = mime.ParseMediaType(v) if p.dispositionParams == nil { p.dispositionParams = emptyParams } }
func downloadFile(link, name string) { var file *os.File defer file.Close() check := http.Client{ CheckRedirect: func(r *http.Request, via []*http.Request) error { r.URL.Opaque = r.URL.Path return nil }, } resp, err := check.Get(link) checkErr(err) defer resp.Body.Close() disposition, ok := resp.Header["Content-Disposition"] if ok { _, params, err := mime.ParseMediaType(disposition[0]) checkErr(err) filename := strings.Split(name, "/")[0] + "/" + params["filename"] file, err = os.Create(filename) checkErr(err) } else { file, err = os.Create(name) checkErr(err) } _, err = io.Copy(file, resp.Body) checkErr(err) }