func (mi *Indexer) ReceiveBlob(blobRef *blobref.BlobRef, source io.Reader) (retsb blobref.SizedBlobRef, err os.Error) { sniffer := new(blobSniffer) hash := blobRef.Hash() var written int64 written, err = io.Copy(io.MultiWriter(hash, sniffer), source) log.Printf("mysqlindexer: hashed+sniffed %d bytes; err %v", written, err) if err != nil { return } if !blobRef.HashMatches(hash) { err = blobserver.ErrCorruptBlob return } sniffer.Parse() mimeType := sniffer.MimeType() log.Printf("mysqlindexer: type=%v; truncated=%v", mimeType, sniffer.IsTruncated()) var client *mysql.Client if client, err = mi.getConnection(); err != nil { return } defer mi.releaseConnection(client) var stmt *mysql.Statement if stmt, err = client.Prepare("INSERT IGNORE INTO blobs (blobref, size, type) VALUES (?, ?, ?)"); err != nil { log.Printf("mysqlindexer: prepare error: %v", err) return } if err = stmt.BindParams(blobRef.String(), written, mimeType); err != nil { log.Printf("mysqlindexer: bind error: %v", err) return } if err = stmt.Execute(); err != nil { log.Printf("mysqlindexer: execute error: %v", err) return } if camli := sniffer.camli; camli != nil { switch camli.Type { case "claim": if err = mi.populateClaim(client, blobRef, camli, sniffer); err != nil { return } case "permanode": if err = mi.populatePermanode(client, blobRef, camli); err != nil { return } case "file": if err = mi.populateFile(client, blobRef, camli); err != nil { return } } } retsb = blobref.SizedBlobRef{BlobRef: blobRef, Size: written} return }
func (sto *appengineStorage) ReceiveBlob(br *blobref.BlobRef, in io.Reader) (sb blobref.SizedBlobRef, err os.Error) { if sto.ctx == nil { err = errNoContext return } var b bytes.Buffer hash := br.Hash() written, err := io.Copy(io.MultiWriter(hash, &b), in) if err != nil { return } if !br.HashMatches(hash) { err = blobserver.ErrCorruptBlob return } mimeType := "application/octet-stream" bw, err := blobstore.Create(sto.ctx, mimeType) if err != nil { return } written, err = io.Copy(bw, &b) if err != nil { // TODO(bradfitz): try to clean up; close it, see if we can find the key, delete it. return } err = bw.Close() if err != nil { // TODO(bradfitz): try to clean up; see if we can find the key, delete it. return } bkey, err := bw.Key() if err != nil { return } var ent blobEnt ent.BlobRefStr = br.String() ent.Size = written ent.BlobKey = bkey dkey := datastore.NewKey(sto.ctx, blobKind, br.String(), 0, nil) _, err = datastore.Put(sto.ctx, dkey, &ent) if err != nil { blobstore.Delete(sto.ctx, bkey) // TODO: insert into task queue on error to try later? return } return blobref.SizedBlobRef{br, written}, nil }
func (mi *Indexer) ReceiveBlob(blobRef *blobref.BlobRef, source io.Reader) (retsb blobref.SizedBlobRef, err os.Error) { sniffer := new(blobSniffer) hash := blobRef.Hash() var written int64 written, err = io.Copy(io.MultiWriter(hash, sniffer), source) log.Printf("mysqlindexer: hashed+sniffed %d bytes; err %v", written, err) if err != nil { return } if !blobRef.HashMatches(hash) { err = blobserver.ErrCorruptBlob return } sniffer.Parse() mimeType := sniffer.MimeType() log.Printf("mysqlindexer: type=%v; truncated=%v", mimeType, sniffer.IsTruncated()) if camli := sniffer.camli; camli != nil { switch camli.Type { case "claim": if err = mi.populateClaim(blobRef, camli, sniffer); err != nil { return } case "permanode": if err = mi.populatePermanode(blobRef, camli); err != nil { return } case "file": if err = mi.populateFile(blobRef, camli); err != nil { return } } } if err = mi.db.Execute("INSERT IGNORE INTO blobs (blobref, size, type) VALUES (?, ?, ?)", blobRef.String(), written, mimeType); err != nil { log.Printf("mysqlindexer: insert into blobs: %v", err) return } retsb = blobref.SizedBlobRef{BlobRef: blobRef, Size: written} return }
func (ds *DiskStorage) ReceiveBlob(blobRef *blobref.BlobRef, source io.Reader) (blobGot blobref.SizedBlobRef, err os.Error) { pname := ds.partition if pname != "" { err = fmt.Errorf("refusing upload directly to queue partition %q", pname) return } hashedDirectory := ds.blobDirectory(pname, blobRef) err = os.MkdirAll(hashedDirectory, 0700) if err != nil { return } tempFile, err := ioutil.TempFile(hashedDirectory, BlobFileBaseName(blobRef)+".tmp") if err != nil { return } success := false // set true later defer func() { if !success { log.Println("Removing temp file: ", tempFile.Name()) os.Remove(tempFile.Name()) } }() hash := blobRef.Hash() written, err := io.Copy(io.MultiWriter(hash, tempFile), source) if err != nil { return } if err = tempFile.Sync(); err != nil { return } if err = tempFile.Close(); err != nil { return } if !blobRef.HashMatches(hash) { err = blobserver.ErrCorruptBlob return } fileName := ds.blobPath("", blobRef) if err = os.Rename(tempFile.Name(), fileName); err != nil { return } stat, err := os.Lstat(fileName) if err != nil { return } if !stat.IsRegular() || stat.Size != written { err = os.NewError("Written size didn't match.") return } for _, mirror := range ds.mirrorPartitions { pname := mirror.partition if pname == "" { panic("expected partition name") } partitionDir := ds.blobDirectory(pname, blobRef) if err = os.MkdirAll(partitionDir, 0700); err != nil { return } partitionFileName := ds.blobPath(pname, blobRef) pfi, err := os.Stat(partitionFileName) if err == nil && pfi.IsRegular() { log.Printf("Skipped dup on partition %q", pname) } else { if err = os.Link(fileName, partitionFileName); err != nil && !linkAlreadyExists(err) { log.Fatalf("got link error %T %#v", err, err) return } log.Printf("Mirrored to partition %q", pname) } } blobGot = blobref.SizedBlobRef{BlobRef: blobRef, Size: stat.Size} success = true if os.Getenv("CAMLI_HACK_OPEN_IMAGES") == "1" { exec.Command("eog", fileName).Run() } hub := ds.GetBlobHub() hub.NotifyBlobReceived(blobRef) for _, mirror := range ds.mirrorPartitions { mirror.GetBlobHub().NotifyBlobReceived(blobRef) } return }
func receiveBlob(blobRef *blobref.BlobRef, source io.Reader) (blobGot *receivedBlob, err os.Error) { hashedDirectory := BlobDirectoryName(blobRef) err = os.MkdirAll(hashedDirectory, 0700) if err != nil { return } var tempFile *os.File tempFile, err = ioutil.TempFile(hashedDirectory, BlobFileBaseName(blobRef)+".tmp") if err != nil { return } success := false // set true later defer func() { if !success { log.Println("Removing temp file: ", tempFile.Name()) os.Remove(tempFile.Name()) } }() hash := blobRef.Hash() var written int64 written, err = io.Copy(io.MultiWriter(hash, tempFile), source) if err != nil { return } // TODO: fsync before close. if err = tempFile.Close(); err != nil { return } if !blobRef.HashMatches(hash) { err = CorruptBlobError return } fileName := BlobFileName(blobRef) if err = os.Rename(tempFile.Name(), fileName); err != nil { return } stat, err := os.Lstat(fileName) if err != nil { return } if !stat.IsRegular() || stat.Size != written { err = os.NewError("Written size didn't match.") return } blobGot = &receivedBlob{blobRef: blobRef, size: stat.Size} success = true if *flagOpenImages { exec.Run("/usr/bin/eog", []string{"/usr/bin/eog", fileName}, os.Environ(), "/", exec.DevNull, exec.DevNull, exec.MergeWithStdout) } return }