// blobref: of the file or schema blob // ss: the parsed file schema blob // bm: keys to populate func (ix *Index) populateFile(blobRef *blobref.BlobRef, ss *schema.Superset, bm BatchMutation) error { seekFetcher, err := blobref.SeekerFromStreamingFetcher(ix.BlobSource) if err != nil { return err } sha1 := sha1.New() fr, err := ss.NewFileReader(seekFetcher) if err != nil { // TODO(bradfitz): propagate up a transient failure // error type, so we can retry indexing files in the // future if blobs are only temporarily unavailable. // Basically the same as the TODO just below. log.Printf("index: error indexing file, creating NewFileReader %s: %v", blobRef, err) return nil } mime, reader := magic.MimeTypeFromReader(fr) size, err := io.Copy(sha1, reader) if err != nil { // TODO: job scheduling system to retry this spaced // out max n times. Right now our options are // ignoring this error (forever) or returning the // error and making the indexing try again (likely // forever failing). Both options suck. For now just // log and act like all's okay. log.Printf("index: error indexing file %s: %v", blobRef, err) return nil } wholeRef := blobref.FromHash("sha1", sha1) bm.Set(keyWholeToFileRef.Key(wholeRef, blobRef), "1") bm.Set(keyFileInfo.Key(blobRef), keyFileInfo.Val(size, ss.FileName, mime)) return nil }
// blobref: of the file or schema blob // ss: the parsed file schema blob // bm: keys to populate func (ix *Index) populateFile(blobRef *blobref.BlobRef, ss *schema.Superset, bm BatchMutation) error { seekFetcher := blobref.SeekerFromStreamingFetcher(ix.BlobSource) fr, err := ss.NewFileReader(seekFetcher) if err != nil { // TODO(bradfitz): propagate up a transient failure // error type, so we can retry indexing files in the // future if blobs are only temporarily unavailable. // Basically the same as the TODO just below. log.Printf("index: error indexing file, creating NewFileReader %s: %v", blobRef, err) return nil } defer fr.Close() mime, reader := magic.MimeTypeFromReader(fr) sha1 := sha1.New() var copyDest io.Writer = sha1 var withCopyErr func(error) // or nil if strings.HasPrefix(mime, "image/") { pr, pw := io.Pipe() copyDest = io.MultiWriter(copyDest, pw) confc := make(chan *image.Config, 1) go func() { conf, _, err := image.DecodeConfig(pr) defer io.Copy(ioutil.Discard, pr) if err == nil { confc <- &conf } else { confc <- nil } }() withCopyErr = func(err error) { pw.CloseWithError(err) if conf := <-confc; conf != nil { bm.Set(keyImageSize.Key(blobRef), keyImageSize.Val(fmt.Sprint(conf.Width), fmt.Sprint(conf.Height))) } } } size, err := io.Copy(copyDest, reader) if f := withCopyErr; f != nil { f(err) } if err != nil { // TODO: job scheduling system to retry this spaced // out max n times. Right now our options are // ignoring this error (forever) or returning the // error and making the indexing try again (likely // forever failing). Both options suck. For now just // log and act like all's okay. log.Printf("index: error indexing file %s: %v", blobRef, err) return nil } wholeRef := blobref.FromHash("sha1", sha1) bm.Set(keyWholeToFileRef.Key(wholeRef, blobRef), "1") bm.Set(keyFileInfo.Key(blobRef), keyFileInfo.Val(size, ss.FileName, mime)) return nil }