// ScaledCached reads the scaled version of the image in file, // if it is in cache and writes it to buf. // // On successful read and population of buf, the returned format is non-empty. // Almost all errors are not interesting. Real errors will be logged. func (ih *ImageHandler) scaledCached(buf *bytes.Buffer, file blob.Ref) (format string) { key := cacheKey(file.String(), ih.MaxWidth, ih.MaxHeight) br, err := ih.thumbMeta.Get(key) if err == errCacheMiss { return } if err != nil { log.Printf("Warning: thumbnail cachekey(%q)->meta lookup error: %v", key, err) return } fr, err := ih.cached(br) if err != nil { return } defer fr.Close() _, err = io.Copy(buf, fr) if err != nil { return } mime := magic.MIMEType(buf.Bytes()) if format = strings.TrimPrefix(mime, "image/"); format == mime { log.Printf("Warning: unescaped MIME type %q of %v file for thumbnail %q", mime, br, key) return } return format }
// ScaledCached reads the scaled version of the image in file, // if it is in cache. On success, the image format is returned. func (ih *ImageHandler) scaledCached(buf *bytes.Buffer, file blob.Ref) (format string, err error) { name := cacheKey(file.String(), ih.MaxWidth, ih.MaxHeight) br, err := ih.sc.Get(name) if err != nil { return format, fmt.Errorf("%v: %v", name, err) } fr, err := ih.cached(br) if err != nil { return format, fmt.Errorf("No cache hit for %v: %v", br, err) } defer fr.Close() _, err = io.Copy(buf, fr) if err != nil { return format, fmt.Errorf("error reading cached thumbnail %v: %v", name, err) } mime := magic.MIMEType(buf.Bytes()) if mime == "" { return format, fmt.Errorf("error with cached thumbnail %v: unknown mime type", name) } pieces := strings.Split(mime, "/") if len(pieces) < 2 { return format, fmt.Errorf("error with cached thumbnail %v: bogus mime type", name) } if pieces[0] != "image" { return format, fmt.Errorf("error with cached thumbnail %v: not an image", name) } return pieces[1], nil }
func (sn *BlobSniffer) Parse() { if sn.bufferIsCamliJSON() { sn.mimeType = "application/json; camliType=" + sn.meta.Type() } else { sn.mimeType = magic.MIMEType(sn.header) } }
// cached returns a FileReader for the given blobref, which may // point to either a blob representing the entire thumbnail (max // 16MB) or a file schema blob. // // The ReadCloser should be closed when done reading. func (ih *ImageHandler) cached(br blob.Ref) (io.ReadCloser, error) { rsc, _, err := ih.Cache.FetchStreaming(br) if err != nil { return nil, err } slurp, err := ioutil.ReadAll(rsc) rsc.Close() if err != nil { return nil, err } // In the common case, when the scaled image itself is less than 16 MB, it's // all together in one blob. if strings.HasPrefix(magic.MIMEType(slurp), "image/") { thumbCacheHitFull.Add(1) if imageDebug { log.Printf("Image Cache: hit: %v\n", br) } return ioutil.NopCloser(bytes.NewReader(slurp)), nil } // For large scaled images, the cached blob is a file schema blob referencing // the sub-chunks. fileBlob, err := schema.BlobFromReader(br, bytes.NewReader(slurp)) if err != nil { log.Printf("Failed to parse non-image thumbnail cache blob %v: %v", br, err) return nil, err } fetchSeeker := blob.SeekerFromStreamingFetcher(ih.Cache) fr, err := fileBlob.NewFileReader(fetchSeeker) if err != nil { log.Printf("cached(%d) NewFileReader = %v", br, err) return nil, err } thumbCacheHitFile.Add(1) if imageDebug { log.Printf("Image Cache: fileref hit: %v\n", br) } return fr, nil }
func main() { server := os.Getenv("AMQP_URL") if server == "" { server = os.ExpandEnv("amqp://$AMQP_USER:[email protected]:5672") } timeout := 5 * time.Second clientID, _ := os.Hostname() queue := "scanner" mainCmd := &cobra.Command{ Use: "amqpc", } p := mainCmd.PersistentFlags() p.StringVarP(&server, "server", "S", server, "server address") p.DurationVarP(&timeout, "timeout", "", timeout, "timeout for commands") p.StringVarP(&clientID, "id", "", clientID, "client ID") p.StringVarP(&queue, "queue", "q", queue, "queue name to publish") appID := queue var noCompress bool pubCmd := &cobra.Command{ Use: "pub", Aliases: []string{"publish", "send", "write"}, Run: func(_ *cobra.Command, args []string) { c, err := newClient(server, queue) if err != nil { log.Fatal(err) } defer c.Close() if err = c.Confirm(false); err != nil { log.Fatal(err) } confirms := c.NotifyPublish(make(chan amqp.Confirmation, 1)) returns := c.NotifyReturn(make(chan amqp.Return, 1)) var sendCount int tbl := make(map[string]interface{}, 1) for _, arg := range args { for k := range tbl { delete(tbl, k) } var r io.ReadCloser mimeType, contentEncoding := "text/plain", "" if strings.HasPrefix(arg, "@") { arg = arg[1:] if arg == "-" { r = os.Stdin } else if fh, err := os.Open(arg); err != nil { log.Fatal(err) } else { if noCompress { r = fh } else { pr, pw := io.Pipe() go func() { defer fh.Close() gw := gzip.NewWriter(pw) if _, err := io.Copy(gw, fh); err != nil { pw.CloseWithError(err) return } pw.CloseWithError(gw.Close()) }() r = pr contentEncoding = "application/gzip" } tbl["FileName"] = arg if err := amqp.Table(tbl).Validate(); err != nil { log.Fatal(err) } mimeType = mime.TypeByExtension(filepath.Ext(arg)) } } else { r = ioutil.NopCloser(strings.NewReader(arg)) } b, err := ioutil.ReadAll(&io.LimitedReader{R: r, N: 256 << 20}) r.Close() if err != nil { log.Fatal(err) } if mimeType == "" { if mimeType = magic.MIMEType(b); mimeType == "" { mimeType = "application/octet-stream" } } if err := c.Publish("", c.Queue.Name, false, false, amqp.Publishing{ Headers: tbl, DeliveryMode: amqp.Persistent, ContentType: mimeType, ContentEncoding: contentEncoding, AppId: appID, Body: b, }, ); err != nil { log.Fatalf("Publish: %v", err) } log.Printf("Sent %q", arg) sendCount++ } Loop: for i := 0; i < sendCount; { select { case c, ok := <-confirms: if !ok { break Loop } if !c.Ack { log.Printf("couldn't deliver %d", c.DeliveryTag) } else { log.Printf("Delivered %d.", c.DeliveryTag) i++ } case r, ok := <-returns: if !ok { break Loop } log.Printf("RETURN: %#v", r) i++ } } }, } f := pubCmd.Flags() f.StringVarP(&appID, "app-id", "", appID, "appID") f.BoolVarP(&noCompress, "no-compress", "", noCompress, "disable file data compression (for slow devices)") var keepFiles bool subCmd := &cobra.Command{ Use: "sub", Aliases: []string{"subscribe", "recv", "receive", "read"}, Run: func(_ *cobra.Command, args []string) { c, err := newClient(server, queue) if err != nil { log.Fatal(err) } defer c.Close() d, err := c.Consume(c.Queue.Name, clientID, false, false, false, false, nil) if err != nil { log.Fatal(err) } tempDir, err := ioutil.TempDir("", "amqpc-") if err != nil { log.Fatal(err) } defer os.RemoveAll(tempDir) var i uint64 for msg := range d { i++ log.Printf("Received %s with %q from %s@%s.", msg.MessageId, msg.Headers, msg.UserId, msg.AppId) var fn string if fnI := msg.Headers["FileName"]; fnI != nil { if fn, _ = fnI.(string); fn != "" { fn = filepath.Base(fn) } } if fn == "" { var ext string if exts, err := mime.ExtensionsByType(msg.ContentType); err != nil { log.Printf("Extension for %q: %v", msg.ContentType) } else if len(ext) > 0 { ext = exts[0] } fn = fmt.Sprintf("%09d%s", i, ext) } fn = filepath.Join(tempDir, fn) err = receive(fn, msg, args) if !keepFiles { os.Remove(fn) } if err != nil { msg.Nack(false, true) log.Fatal(err) } if err := msg.Ack(false); err != nil { log.Printf("cannot ACK %q: %v", msg, err) } } }, } subCmd.Flags().BoolVarP(&keepFiles, "keep-files", "x", keepFiles, "keep temporary files") mainCmd.AddCommand(pubCmd, subCmd) mainCmd.Execute() }