func (s *TarballSuite) TestNewTarball(c *C) { tarball, err := NewTarball(config.Value("TARBALL_PREFIX").(string)) s.tarball = tarball c.Assert(tarball, NotNil) c.Assert(err, IsNil) // now lets make sure that we have set the parameters properly c.Assert(tarball.Id, Equals, tarballCounter) c.Assert(tarball.Key, Equals, config.Value("TARBALL_PREFIX").(string)+strconv.Itoa(int(tarballCounter))+".tar.gz") c.Assert(tarball.Full, Equals, false) c.Assert(tarball.file, NotNil) c.Assert(tarball.gz, NotNil) c.Assert(tarball.tw, NotNil) }
func ErrorHandler() { var operation CommunicationOperation // initialize our fatal errors associative arry fatalErrors := config.Value("FATAL_ERRORS").(map[int]bool) for { operation = <-errorComm // check if there is an error and if it is fatal if operation.err != nil && fatalErrors[operation.code] { log.Fatal(operation.err) } } }
func (t *Tarball) Upload() error { // make sure the tarball is completed etc t.close() file, err := os.Open(t.Key) if err != nil { return err } defer file.Close() stat, err := file.Stat() if err != nil { return err } // now set up the bucket and prepare for the upload s3Conn := s3.New(config.Value("AWS_AUTH").(aws.Auth), config.Value("AWS_REGION").(aws.Region)) bucket := s3Conn.Bucket(config.Value("BUCKET_NAME").(string)) // now lets upload the file if err := bucket.PutReader(t.Key, file, stat.Size(), contentType, permissions); err != nil { return err } return nil }
/* 1.) check to see if new file will put us over 25% threshold yes: create a new tarball with only the new large file no: add file to the current tarball 2.) guess which tarball of the two we should upload 3.) if we are uploading the old tarball, then go ahead and return the new tarball */ func (t *Tarball) AddFile(file *File) (*Tarball, error) { var newTarball *Tarball maxSize := int64(config.Value("MAX_TARBALL_SIZE").(int)) maxThresholdedSize := int64(float64(maxSize) * float64(1+threshold)) // handle the adding of the file into the tarball // if file pushes current tarball over threshold or is big enough for its own tarball if file.size > maxSize || file.size+t.size > maxThresholdedSize { // create a new tarball tarball, err := NewTarball(t.Prefix) if err != nil { return nil, err } if err := tarball.addFile(file); err != nil { return nil, err } newTarball = tarball } else { //file size is small enough to add to the current tarball if err := t.addFile(file); err != nil { return nil, err } } // handle uploading / returning of the tarball if newTarball == nil { if t.size > maxSize { if err := t.Upload(); err != nil { return nil, err } return NewTarball(t.Prefix) } return t, nil } else if t.size > maxSize && newTarball.size > maxSize { // upload both and return a new tarball (empty) if err := t.Upload(); err != nil { return nil, err } if err := newTarball.Upload(); err != nil { return nil, err } return NewTarball(t.Prefix) } else if t.size > newTarball.size { // upload t, return newTarball if err := t.Upload(); err != nil { return nil, err } return newTarball, nil } else if newTarball.size > t.size { // upload newTarball return t or nil if err := newTarball.Upload(); err != nil { return nil, err } return t, nil } return nil, nil }
func (s *TarballSuite) TestUpload(c *C) { tarball, err := NewTarball(config.Value("TARBALL_PREFIX").(string)) s.tarball = tarball c.Assert(err, IsNil) c.Assert(tarball, NotNil) // add a file to the archive newTarball, err := tarball.AddFile(s.files[0]) c.Assert(newTarball, Equals, tarball) c.Assert(err, IsNil) // now we can upload the archive err = tarball.Upload() c.Assert(err, IsNil) }
func Worker(waitGroup *sync.WaitGroup, commChannel chan CommunicationOperation) { defer waitGroup.Done() tarball, err := NewTarball(config.Value("TARBALL_PREFIX").(string)) if err != nil { commChannel <- CommunicationOperation{err: err} } finished := false popReciever := make(chan PopResponseOperation, 5) // finalize tarball as needed etc closeTarball := func() { finished = true // try to upload the tarball if err := tarball.Upload(); err != nil { commChannel <- CommunicationOperation{err: err} } } for { select { case _ = <-commChannel: closeTarball() default: //pop a new file // submit our reciever channel to the pop as needed pop <- PopOperation{channel: popReciever} response := <-popReciever if response.err != nil { closeTarball() } else { newTarball, err := tarball.AddFile(response.file) if err != nil || newTarball == nil { finished = true commChannel <- CommunicationOperation{err: err} } tarball = newTarball } } if finished { break } } }