Exemplo n.º 1
0
func (s *TarballSuite) TestNewTarball(c *C) {
	tarball, err := NewTarball(config.Value("TARBALL_PREFIX").(string))
	s.tarball = tarball
	c.Assert(tarball, NotNil)
	c.Assert(err, IsNil)
	// now lets make sure that we have set the parameters properly
	c.Assert(tarball.Id, Equals, tarballCounter)
	c.Assert(tarball.Key, Equals, config.Value("TARBALL_PREFIX").(string)+strconv.Itoa(int(tarballCounter))+".tar.gz")
	c.Assert(tarball.Full, Equals, false)
	c.Assert(tarball.file, NotNil)
	c.Assert(tarball.gz, NotNil)
	c.Assert(tarball.tw, NotNil)
}
Exemplo n.º 2
0
func ErrorHandler() {
	var operation CommunicationOperation
	// initialize our fatal errors associative arry
	fatalErrors := config.Value("FATAL_ERRORS").(map[int]bool)
	for {
		operation = <-errorComm
		// check if there is an error and if it is fatal
		if operation.err != nil && fatalErrors[operation.code] {
			log.Fatal(operation.err)
		}
	}
}
Exemplo n.º 3
0
func (t *Tarball) Upload() error {
	// make sure the tarball is completed etc
	t.close()
	file, err := os.Open(t.Key)
	if err != nil {
		return err
	}
	defer file.Close()
	stat, err := file.Stat()
	if err != nil {
		return err
	}
	// now set up the bucket and prepare for the upload
	s3Conn := s3.New(config.Value("AWS_AUTH").(aws.Auth), config.Value("AWS_REGION").(aws.Region))
	bucket := s3Conn.Bucket(config.Value("BUCKET_NAME").(string))
	// now lets upload the file
	if err := bucket.PutReader(t.Key, file, stat.Size(), contentType, permissions); err != nil {
		return err
	}
	return nil
}
Exemplo n.º 4
0
/*
	1.) check to see if new file will put us over 25% threshold
		yes: create a new tarball with only the new large file
		no: add file to the current tarball
	2.) guess which tarball of the two we should upload
	3.) if we are uploading the old tarball, then go ahead and return the new tarball

*/
func (t *Tarball) AddFile(file *File) (*Tarball, error) {

	var newTarball *Tarball
	maxSize := int64(config.Value("MAX_TARBALL_SIZE").(int))
	maxThresholdedSize := int64(float64(maxSize) * float64(1+threshold))
	// handle the adding of the file into the tarball
	// if file pushes current tarball over threshold or is big enough for its own tarball
	if file.size > maxSize || file.size+t.size > maxThresholdedSize {
		// create a new tarball
		tarball, err := NewTarball(t.Prefix)
		if err != nil {
			return nil, err
		}
		if err := tarball.addFile(file); err != nil {
			return nil, err
		}
		newTarball = tarball
	} else { //file size is small enough to add to the current tarball
		if err := t.addFile(file); err != nil {
			return nil, err
		}
	}
	// handle uploading / returning of the tarball
	if newTarball == nil {
		if t.size > maxSize {
			if err := t.Upload(); err != nil {
				return nil, err
			}
			return NewTarball(t.Prefix)
		}
		return t, nil
	} else if t.size > maxSize && newTarball.size > maxSize { // upload both and return a new tarball (empty)
		if err := t.Upload(); err != nil {
			return nil, err
		}
		if err := newTarball.Upload(); err != nil {
			return nil, err
		}
		return NewTarball(t.Prefix)
	} else if t.size > newTarball.size { // upload t, return newTarball
		if err := t.Upload(); err != nil {
			return nil, err
		}
		return newTarball, nil
	} else if newTarball.size > t.size { // upload newTarball return t or nil
		if err := newTarball.Upload(); err != nil {
			return nil, err
		}
		return t, nil
	}
	return nil, nil
}
Exemplo n.º 5
0
func (s *TarballSuite) TestUpload(c *C) {

	tarball, err := NewTarball(config.Value("TARBALL_PREFIX").(string))
	s.tarball = tarball
	c.Assert(err, IsNil)
	c.Assert(tarball, NotNil)
	// add a file to the archive
	newTarball, err := tarball.AddFile(s.files[0])
	c.Assert(newTarball, Equals, tarball)
	c.Assert(err, IsNil)
	// now we can upload the archive
	err = tarball.Upload()
	c.Assert(err, IsNil)
}
Exemplo n.º 6
0
func Worker(waitGroup *sync.WaitGroup, commChannel chan CommunicationOperation) {

	defer waitGroup.Done()
	tarball, err := NewTarball(config.Value("TARBALL_PREFIX").(string))
	if err != nil {
		commChannel <- CommunicationOperation{err: err}
	}
	finished := false
	popReciever := make(chan PopResponseOperation, 5)
	// finalize tarball as needed etc
	closeTarball := func() {
		finished = true
		// try to upload the tarball
		if err := tarball.Upload(); err != nil {
			commChannel <- CommunicationOperation{err: err}
		}
	}
	for {
		select {
		case _ = <-commChannel:
			closeTarball()
		default: //pop a new file
			// submit our reciever channel to the pop as needed
			pop <- PopOperation{channel: popReciever}
			response := <-popReciever
			if response.err != nil {
				closeTarball()
			} else {
				newTarball, err := tarball.AddFile(response.file)
				if err != nil || newTarball == nil {
					finished = true
					commChannel <- CommunicationOperation{err: err}
				}
				tarball = newTarball
			}
		}
		if finished {
			break
		}
	}
}