func (s *TarballSuite) TestNewTarball(c *C) {
tarball, err := NewTarball(config.Value("TARBALL_PREFIX").(string))
s.tarball = tarball
c.Assert(tarball, NotNil)
c.Assert(err, IsNil)
// now lets make sure that we have set the parameters properly
c.Assert(tarball.Id, Equals, tarballCounter)
c.Assert(tarball.Key, Equals, config.Value("TARBALL_PREFIX").(string)+strconv.Itoa(int(tarballCounter))+".tar.gz")
c.Assert(tarball.Full, Equals, false)
c.Assert(tarball.file, NotNil)
c.Assert(tarball.gz, NotNil)
c.Assert(tarball.tw, NotNil)
}
func ErrorHandler() {
var operation CommunicationOperation
// initialize our fatal errors associative arry
fatalErrors := config.Value("FATAL_ERRORS").(map[int]bool)
for {
operation = <-errorComm
// check if there is an error and if it is fatal
if operation.err != nil && fatalErrors[operation.code] {
log.Fatal(operation.err)
}
}
}
func (t *Tarball) Upload() error {
// make sure the tarball is completed etc
t.close()
file, err := os.Open(t.Key)
if err != nil {
return err
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
return err
}
// now set up the bucket and prepare for the upload
s3Conn := s3.New(config.Value("AWS_AUTH").(aws.Auth), config.Value("AWS_REGION").(aws.Region))
bucket := s3Conn.Bucket(config.Value("BUCKET_NAME").(string))
// now lets upload the file
if err := bucket.PutReader(t.Key, file, stat.Size(), contentType, permissions); err != nil {
return err
}
return nil
}
/*
1.) check to see if new file will put us over 25% threshold
yes: create a new tarball with only the new large file
no: add file to the current tarball
2.) guess which tarball of the two we should upload
3.) if we are uploading the old tarball, then go ahead and return the new tarball
*/
func (t *Tarball) AddFile(file *File) (*Tarball, error) {
var newTarball *Tarball
maxSize := int64(config.Value("MAX_TARBALL_SIZE").(int))
maxThresholdedSize := int64(float64(maxSize) * float64(1+threshold))
// handle the adding of the file into the tarball
// if file pushes current tarball over threshold or is big enough for its own tarball
if file.size > maxSize || file.size+t.size > maxThresholdedSize {
// create a new tarball
tarball, err := NewTarball(t.Prefix)
if err != nil {
return nil, err
}
if err := tarball.addFile(file); err != nil {
return nil, err
}
newTarball = tarball
} else { //file size is small enough to add to the current tarball
if err := t.addFile(file); err != nil {
return nil, err
}
}
// handle uploading / returning of the tarball
if newTarball == nil {
if t.size > maxSize {
if err := t.Upload(); err != nil {
return nil, err
}
return NewTarball(t.Prefix)
}
return t, nil
} else if t.size > maxSize && newTarball.size > maxSize { // upload both and return a new tarball (empty)
if err := t.Upload(); err != nil {
return nil, err
}
if err := newTarball.Upload(); err != nil {
return nil, err
}
return NewTarball(t.Prefix)
} else if t.size > newTarball.size { // upload t, return newTarball
if err := t.Upload(); err != nil {
return nil, err
}
return newTarball, nil
} else if newTarball.size > t.size { // upload newTarball return t or nil
if err := newTarball.Upload(); err != nil {
return nil, err
}
return t, nil
}
return nil, nil
}
func (s *TarballSuite) TestUpload(c *C) {
tarball, err := NewTarball(config.Value("TARBALL_PREFIX").(string))
s.tarball = tarball
c.Assert(err, IsNil)
c.Assert(tarball, NotNil)
// add a file to the archive
newTarball, err := tarball.AddFile(s.files[0])
c.Assert(newTarball, Equals, tarball)
c.Assert(err, IsNil)
// now we can upload the archive
err = tarball.Upload()
c.Assert(err, IsNil)
}
func Worker(waitGroup *sync.WaitGroup, commChannel chan CommunicationOperation) {
defer waitGroup.Done()
tarball, err := NewTarball(config.Value("TARBALL_PREFIX").(string))
if err != nil {
commChannel <- CommunicationOperation{err: err}
}
finished := false
popReciever := make(chan PopResponseOperation, 5)
// finalize tarball as needed etc
closeTarball := func() {
finished = true
// try to upload the tarball
if err := tarball.Upload(); err != nil {
commChannel <- CommunicationOperation{err: err}
}
}
for {
select {
case _ = <-commChannel:
closeTarball()
default: //pop a new file
// submit our reciever channel to the pop as needed
pop <- PopOperation{channel: popReciever}
response := <-popReciever
if response.err != nil {
closeTarball()
} else {
newTarball, err := tarball.AddFile(response.file)
if err != nil || newTarball == nil {
finished = true
commChannel <- CommunicationOperation{err: err}
}
tarball = newTarball
}
}
if finished {
break
}
}
}