func testSize(t *testing.T, drv string) {
count := 10
if os.Getenv("BLOBSTORE_TEST_ALL") != "" {
count = -1
}
if testing.Short() {
t.SkipNow()
}
dir, err := ioutil.TempDir("", "pool-benchmark")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
cfg := drv + "://" + dir
u, err := config.ParseURL(cfg)
if err != nil {
t.Fatal(err)
}
store, err := New(u)
if err != nil {
t.Fatal(err)
}
defer store.Close()
start1 := time.Now()
stored := storeBinaries(t, store, nil, count)
stored += storeBinaries(t, store, make([]byte, 32), count)
t.Logf("took %s to store", time.Since(start1))
t.Logf("%s size is %s", drv, formatutil.Size(uint64(totalSize(dir))))
start2 := time.Now()
// Verify that all files are OK
iter, err := store.Iter()
if err != nil {
t.Fatal(err)
}
var id string
found := 0
for iter.Next(&id) {
found++
f, err := store.Open(id)
if err != nil {
t.Error(err)
continue
}
if err := f.Check(); err != nil {
t.Error(err)
}
f.Close()
}
if found != stored {
t.Errorf("stored %d files, %d found", stored, found)
}
t.Logf("took %s to verify", time.Since(start2))
}
func Bundle(groups []*Group, opts Options) (*Asset, error) {
assetType := Type(-1)
var names []string
for _, group := range groups {
for _, v := range group.Assets {
if v.Type == TypeOther {
return nil, fmt.Errorf("asset %q does not specify a Type and can't be bundled", v.Name)
}
if assetType < 0 {
assetType = v.Type
} else if assetType != v.Type {
return nil, fmt.Errorf("asset %q has different code type %s (first asset is of type %s)", v.Name, v.Type, assetType)
}
names = append(names, v.Name)
}
}
bundler := bundlers[assetType]
if bundler == nil {
return nil, fmt.Errorf("no bundler for %s", assetType)
}
// Prepare the code, changing relative paths if required
name, err := bundleName(groups, assetType.Ext(), opts)
if err != nil {
return nil, err
}
// The bundle is output to the first manager
m := groups[0].Manager
// Check if the code has been already bundled
if m.Has(name) {
log.Debugf("%s already bundled into %s and up to date", names, name)
} else {
dir := path.Dir(name)
log.Debugf("bundling %v", names)
var code []string
for _, group := range groups {
for _, v := range group.Assets {
c, err := v.Code(group.Manager)
if err != nil {
return nil, fmt.Errorf("error getting code for asset %q: %s", v.Name, err)
}
if vd := path.Dir(v.Name); vd != dir {
if assetType == TypeCSS {
log.Debugf("asset %q will move from %v to %v, rewriting relative paths...", v.Name, vd, dir)
c = replaceRelativePaths(c, vd, dir)
} else {
log.Warningf("asset %q will move from %v to %v, relative paths might not work", v.Name, vd, dir)
}
}
code = append(code, c)
}
}
// Bundle to a buf first. We don't want to create
// the file if the bundling fails.
var buf bytes.Buffer
allCode := strings.Join(code, "\n\n")
reader := strings.NewReader(allCode)
if err := bundler.Bundle(&buf, reader, opts); err != nil {
return nil, err
}
s := makeLinksCacheable(m, dir, buf.Bytes())
initial := len(allCode)
final := len(s)
var percent float64
if initial != 0 {
percent = float64(final) / float64(initial) * 100
}
log.Debugf("reduced size from %s to %s (%.2f%%)", formatutil.Size(uint64(initial)),
formatutil.Size(uint64(final)), percent)
w, err := m.Create(name, true)
if err == nil {
if _, err := io.Copy(w, strings.NewReader(s)); err != nil {
w.Close()
return nil, err
}
if err := w.Close(); err != nil {
return nil, err
}
} else {
// If the file exists, is up to date
if !os.IsExist(err) {
return nil, err
}
}
}
return &Asset{
Name: name,
Type: assetType,
Attributes: groups[0].Assets[0].Attributes,
Position: groups[0].Assets[0].Position,
}, nil
}