// ScaledCached reads the scaled version of the image in file,
// if it is in cache and writes it to buf.
//
// On successful read and population of buf, the returned format is non-empty.
// Almost all errors are not interesting. Real errors will be logged.
func (ih *ImageHandler) scaledCached(buf *bytes.Buffer, file blob.Ref) (format string) {
key := cacheKey(file.String(), ih.MaxWidth, ih.MaxHeight)
br, err := ih.thumbMeta.Get(key)
if err == errCacheMiss {
return
}
if err != nil {
log.Printf("Warning: thumbnail cachekey(%q)->meta lookup error: %v", key, err)
return
}
fr, err := ih.cached(br)
if err != nil {
return
}
defer fr.Close()
_, err = io.Copy(buf, fr)
if err != nil {
return
}
mime := magic.MIMEType(buf.Bytes())
if format = strings.TrimPrefix(mime, "image/"); format == mime {
log.Printf("Warning: unescaped MIME type %q of %v file for thumbnail %q", mime, br, key)
return
}
return format
}
// ScaledCached reads the scaled version of the image in file,
// if it is in cache. On success, the image format is returned.
func (ih *ImageHandler) scaledCached(buf *bytes.Buffer, file blob.Ref) (format string, err error) {
name := cacheKey(file.String(), ih.MaxWidth, ih.MaxHeight)
br, err := ih.sc.Get(name)
if err != nil {
return format, fmt.Errorf("%v: %v", name, err)
}
fr, err := ih.cached(br)
if err != nil {
return format, fmt.Errorf("No cache hit for %v: %v", br, err)
}
defer fr.Close()
_, err = io.Copy(buf, fr)
if err != nil {
return format, fmt.Errorf("error reading cached thumbnail %v: %v", name, err)
}
mime := magic.MIMEType(buf.Bytes())
if mime == "" {
return format, fmt.Errorf("error with cached thumbnail %v: unknown mime type", name)
}
pieces := strings.Split(mime, "/")
if len(pieces) < 2 {
return format, fmt.Errorf("error with cached thumbnail %v: bogus mime type", name)
}
if pieces[0] != "image" {
return format, fmt.Errorf("error with cached thumbnail %v: not an image", name)
}
return pieces[1], nil
}
func (sn *BlobSniffer) Parse() {
if sn.bufferIsCamliJSON() {
sn.mimeType = "application/json; camliType=" + sn.meta.Type()
} else {
sn.mimeType = magic.MIMEType(sn.header)
}
}
// cached returns a FileReader for the given blobref, which may
// point to either a blob representing the entire thumbnail (max
// 16MB) or a file schema blob.
//
// The ReadCloser should be closed when done reading.
func (ih *ImageHandler) cached(br blob.Ref) (io.ReadCloser, error) {
rsc, _, err := ih.Cache.FetchStreaming(br)
if err != nil {
return nil, err
}
slurp, err := ioutil.ReadAll(rsc)
rsc.Close()
if err != nil {
return nil, err
}
// In the common case, when the scaled image itself is less than 16 MB, it's
// all together in one blob.
if strings.HasPrefix(magic.MIMEType(slurp), "image/") {
thumbCacheHitFull.Add(1)
if imageDebug {
log.Printf("Image Cache: hit: %v\n", br)
}
return ioutil.NopCloser(bytes.NewReader(slurp)), nil
}
// For large scaled images, the cached blob is a file schema blob referencing
// the sub-chunks.
fileBlob, err := schema.BlobFromReader(br, bytes.NewReader(slurp))
if err != nil {
log.Printf("Failed to parse non-image thumbnail cache blob %v: %v", br, err)
return nil, err
}
fetchSeeker := blob.SeekerFromStreamingFetcher(ih.Cache)
fr, err := fileBlob.NewFileReader(fetchSeeker)
if err != nil {
log.Printf("cached(%d) NewFileReader = %v", br, err)
return nil, err
}
thumbCacheHitFile.Add(1)
if imageDebug {
log.Printf("Image Cache: fileref hit: %v\n", br)
}
return fr, nil
}
func main() {
server := os.Getenv("AMQP_URL")
if server == "" {
server = os.ExpandEnv("amqp://$AMQP_USER:[email protected]:5672")
}
timeout := 5 * time.Second
clientID, _ := os.Hostname()
queue := "scanner"
mainCmd := &cobra.Command{
Use: "amqpc",
}
p := mainCmd.PersistentFlags()
p.StringVarP(&server, "server", "S", server, "server address")
p.DurationVarP(&timeout, "timeout", "", timeout, "timeout for commands")
p.StringVarP(&clientID, "id", "", clientID, "client ID")
p.StringVarP(&queue, "queue", "q", queue, "queue name to publish")
appID := queue
var noCompress bool
pubCmd := &cobra.Command{
Use: "pub",
Aliases: []string{"publish", "send", "write"},
Run: func(_ *cobra.Command, args []string) {
c, err := newClient(server, queue)
if err != nil {
log.Fatal(err)
}
defer c.Close()
if err = c.Confirm(false); err != nil {
log.Fatal(err)
}
confirms := c.NotifyPublish(make(chan amqp.Confirmation, 1))
returns := c.NotifyReturn(make(chan amqp.Return, 1))
var sendCount int
tbl := make(map[string]interface{}, 1)
for _, arg := range args {
for k := range tbl {
delete(tbl, k)
}
var r io.ReadCloser
mimeType, contentEncoding := "text/plain", ""
if strings.HasPrefix(arg, "@") {
arg = arg[1:]
if arg == "-" {
r = os.Stdin
} else if fh, err := os.Open(arg); err != nil {
log.Fatal(err)
} else {
if noCompress {
r = fh
} else {
pr, pw := io.Pipe()
go func() {
defer fh.Close()
gw := gzip.NewWriter(pw)
if _, err := io.Copy(gw, fh); err != nil {
pw.CloseWithError(err)
return
}
pw.CloseWithError(gw.Close())
}()
r = pr
contentEncoding = "application/gzip"
}
tbl["FileName"] = arg
if err := amqp.Table(tbl).Validate(); err != nil {
log.Fatal(err)
}
mimeType = mime.TypeByExtension(filepath.Ext(arg))
}
} else {
r = ioutil.NopCloser(strings.NewReader(arg))
}
b, err := ioutil.ReadAll(&io.LimitedReader{R: r, N: 256 << 20})
r.Close()
if err != nil {
log.Fatal(err)
}
if mimeType == "" {
if mimeType = magic.MIMEType(b); mimeType == "" {
mimeType = "application/octet-stream"
}
}
if err := c.Publish("", c.Queue.Name, false, false,
amqp.Publishing{
Headers: tbl,
DeliveryMode: amqp.Persistent,
ContentType: mimeType,
ContentEncoding: contentEncoding,
AppId: appID,
Body: b,
},
); err != nil {
log.Fatalf("Publish: %v", err)
}
log.Printf("Sent %q", arg)
sendCount++
}
Loop:
for i := 0; i < sendCount; {
select {
case c, ok := <-confirms:
if !ok {
break Loop
}
if !c.Ack {
log.Printf("couldn't deliver %d", c.DeliveryTag)
} else {
log.Printf("Delivered %d.", c.DeliveryTag)
i++
}
case r, ok := <-returns:
if !ok {
break Loop
}
log.Printf("RETURN: %#v", r)
i++
}
}
},
}
f := pubCmd.Flags()
f.StringVarP(&appID, "app-id", "", appID, "appID")
f.BoolVarP(&noCompress, "no-compress", "", noCompress, "disable file data compression (for slow devices)")
var keepFiles bool
subCmd := &cobra.Command{
Use: "sub",
Aliases: []string{"subscribe", "recv", "receive", "read"},
Run: func(_ *cobra.Command, args []string) {
c, err := newClient(server, queue)
if err != nil {
log.Fatal(err)
}
defer c.Close()
d, err := c.Consume(c.Queue.Name, clientID, false, false, false, false, nil)
if err != nil {
log.Fatal(err)
}
tempDir, err := ioutil.TempDir("", "amqpc-")
if err != nil {
log.Fatal(err)
}
defer os.RemoveAll(tempDir)
var i uint64
for msg := range d {
i++
log.Printf("Received %s with %q from %s@%s.",
msg.MessageId, msg.Headers, msg.UserId, msg.AppId)
var fn string
if fnI := msg.Headers["FileName"]; fnI != nil {
if fn, _ = fnI.(string); fn != "" {
fn = filepath.Base(fn)
}
}
if fn == "" {
var ext string
if exts, err := mime.ExtensionsByType(msg.ContentType); err != nil {
log.Printf("Extension for %q: %v", msg.ContentType)
} else if len(ext) > 0 {
ext = exts[0]
}
fn = fmt.Sprintf("%09d%s", i, ext)
}
fn = filepath.Join(tempDir, fn)
err = receive(fn, msg, args)
if !keepFiles {
os.Remove(fn)
}
if err != nil {
msg.Nack(false, true)
log.Fatal(err)
}
if err := msg.Ack(false); err != nil {
log.Printf("cannot ACK %q: %v", msg, err)
}
}
},
}
subCmd.Flags().BoolVarP(&keepFiles, "keep-files", "x", keepFiles, "keep temporary files")
mainCmd.AddCommand(pubCmd, subCmd)
mainCmd.Execute()
}
