func indexPackage(pkg string) {
log.Printf("Indexing %s\n", pkg)
unpacked := filepath.Join(tmpdir, pkg, pkg)
if err := os.MkdirAll(*unpackedPath, os.FileMode(0755)); err != nil {
log.Fatalf("Could not create directory: %v\n", err)
}
// Write to a temporary file first so that merges can happen at the same
// time. If we don’t do that, merges will try to use incomplete index
// files, which are interpreted as corrupted.
tmpIndexPath := filepath.Join(*unpackedPath, pkg+".tmp")
index := index.Create(tmpIndexPath)
// +1 because of the / that should not be included in the index.
stripLen := len(filepath.Join(tmpdir, pkg)) + 1
filepath.Walk(unpacked,
func(path string, info os.FileInfo, err error) error {
if dir, filename := filepath.Split(path); filename != "" {
skip := ignored(info, dir, filename)
if *debugSkip && skip != nil {
log.Printf("Skipping %q: %v", path, skip)
}
if skip != nil && info.IsDir() {
if err := os.RemoveAll(path); err != nil {
log.Fatalf("Could not remove directory %q: %v\n", path, err)
}
return filepath.SkipDir
}
if skip != nil && !info.IsDir() {
if err := os.Remove(path); err != nil {
log.Fatalf("Could not remove file %q: %v\n", path, err)
}
return nil
}
}
if info == nil || !info.Mode().IsRegular() {
return nil
}
// Some filenames (e.g.
// "xblast-tnt-levels_20050106-2/reconstruct\xeeon2.xal") contain
// invalid UTF-8 and will break when sending them via JSON later
// on. Filter those out early to avoid breakage.
if !utf8.ValidString(path) {
log.Printf("Skipping due to invalid UTF-8: %s\n", path)
return nil
}
if err := index.AddFile(path, path[stripLen:]); err != nil {
log.Printf("Could not index %q: %v\n", path, err)
if err := os.Remove(path); err != nil {
log.Fatalf("Could not remove file %q: %v\n", path, err)
}
} else {
// Copy this file out of /tmp to our unpacked directory.
outputPath := filepath.Join(*unpackedPath, path[stripLen:])
if err := os.MkdirAll(filepath.Dir(outputPath), os.FileMode(0755)); err != nil {
log.Fatalf("Could not create directory: %v\n", err)
}
output, err := os.Create(outputPath)
if err != nil {
log.Fatalf("Could not create output file %q: %v\n", outputPath, err)
}
defer output.Close()
input, err := os.Open(path)
if err != nil {
log.Fatalf("Could not open input file %q: %v\n", path, err)
}
defer input.Close()
if _, err := io.Copy(output, input); err != nil {
log.Fatalf("Could not copy %q to %q: %v\n", path, outputPath, err)
}
}
return nil
})
index.Flush()
finalIndexPath := filepath.Join(*unpackedPath, pkg+".idx")
if err := os.Rename(tmpIndexPath, finalIndexPath); err != nil {
log.Fatal(err)
}
successfulPackageIndexes.Inc()
}
func indexPackage(pkg string) {
unpacked := filepath.Join(tmpdir, pkg, pkg)
index := index.Create(filepath.Join(*unpackedPath, pkg+".idx"))
stripLen := len(filepath.Join(tmpdir, pkg))
filepath.Walk(unpacked,
func(path string, info os.FileInfo, err error) error {
if dir, filename := filepath.Split(path); filename != "" {
skip := ignored(info, dir, filename)
if skip && info.IsDir() {
if err := os.RemoveAll(path); err != nil {
log.Fatalf("Could not remove directory %q: %v\n", path, err)
}
return filepath.SkipDir
}
if skip && !info.IsDir() {
if err := os.Remove(path); err != nil {
log.Fatalf("Could not remove file %q: %v\n", path, err)
}
return nil
}
}
if info == nil || !info.Mode().IsRegular() {
return nil
}
// Some filenames (e.g.
// "xblast-tnt-levels_20050106-2/reconstruct\xeeon2.xal") contain
// invalid UTF-8 and will break when sending them via JSON later
// on. Filter those out early to avoid breakage.
if !utf8.ValidString(path) {
log.Printf("Skipping due to invalid UTF-8: %s\n", path)
return nil
}
if err := index.AddFile(path, path[stripLen:]); err != nil {
if err := os.Remove(path); err != nil {
log.Fatalf("Could not remove file %q: %v\n", path, err)
}
} else {
// Copy this file out of /tmp to our unpacked directory.
outputPath := filepath.Join(*unpackedPath, path[stripLen:])
if err := os.MkdirAll(filepath.Dir(outputPath), os.FileMode(0755)); err != nil {
log.Fatalf("Could not create directory: %v\n", err)
}
output, err := os.Create(outputPath)
if err != nil {
log.Fatalf("Could not create output file %q: %v\n", outputPath, err)
}
defer output.Close()
input, err := os.Open(path)
if err != nil {
log.Fatalf("Could not open input file %q: %v\n", path, err)
}
defer input.Close()
if _, err := io.Copy(output, input); err != nil {
log.Fatalf("Could not copy %q to %q: %v\n", path, outputPath, err)
}
}
return nil
})
index.Flush()
}