Example #1
0
func indexPackage(pkg string) {
	log.Printf("Indexing %s\n", pkg)
	unpacked := filepath.Join(tmpdir, pkg, pkg)
	if err := os.MkdirAll(*unpackedPath, os.FileMode(0755)); err != nil {
		log.Fatalf("Could not create directory: %v\n", err)
	}

	// Write to a temporary file first so that merges can happen at the same
	// time. If we don’t do that, merges will try to use incomplete index
	// files, which are interpreted as corrupted.
	tmpIndexPath := filepath.Join(*unpackedPath, pkg+".tmp")
	index := index.Create(tmpIndexPath)
	// +1 because of the / that should not be included in the index.
	stripLen := len(filepath.Join(tmpdir, pkg)) + 1

	filepath.Walk(unpacked,
		func(path string, info os.FileInfo, err error) error {
			if dir, filename := filepath.Split(path); filename != "" {
				skip := ignored(info, dir, filename)
				if *debugSkip && skip != nil {
					log.Printf("Skipping %q: %v", path, skip)
				}
				if skip != nil && info.IsDir() {
					if err := os.RemoveAll(path); err != nil {
						log.Fatalf("Could not remove directory %q: %v\n", path, err)
					}
					return filepath.SkipDir
				}
				if skip != nil && !info.IsDir() {
					if err := os.Remove(path); err != nil {
						log.Fatalf("Could not remove file %q: %v\n", path, err)
					}
					return nil
				}
			}

			if info == nil || !info.Mode().IsRegular() {
				return nil
			}

			// Some filenames (e.g.
			// "xblast-tnt-levels_20050106-2/reconstruct\xeeon2.xal") contain
			// invalid UTF-8 and will break when sending them via JSON later
			// on. Filter those out early to avoid breakage.
			if !utf8.ValidString(path) {
				log.Printf("Skipping due to invalid UTF-8: %s\n", path)
				return nil
			}

			if err := index.AddFile(path, path[stripLen:]); err != nil {
				log.Printf("Could not index %q: %v\n", path, err)
				if err := os.Remove(path); err != nil {
					log.Fatalf("Could not remove file %q: %v\n", path, err)
				}
			} else {
				// Copy this file out of /tmp to our unpacked directory.
				outputPath := filepath.Join(*unpackedPath, path[stripLen:])
				if err := os.MkdirAll(filepath.Dir(outputPath), os.FileMode(0755)); err != nil {
					log.Fatalf("Could not create directory: %v\n", err)
				}
				output, err := os.Create(outputPath)
				if err != nil {
					log.Fatalf("Could not create output file %q: %v\n", outputPath, err)
				}
				defer output.Close()
				input, err := os.Open(path)
				if err != nil {
					log.Fatalf("Could not open input file %q: %v\n", path, err)
				}
				defer input.Close()
				if _, err := io.Copy(output, input); err != nil {
					log.Fatalf("Could not copy %q to %q: %v\n", path, outputPath, err)
				}
			}
			return nil
		})

	index.Flush()

	finalIndexPath := filepath.Join(*unpackedPath, pkg+".idx")
	if err := os.Rename(tmpIndexPath, finalIndexPath); err != nil {
		log.Fatal(err)
	}
	successfulPackageIndexes.Inc()
}
Example #2
0
func indexPackage(pkg string) {
	unpacked := filepath.Join(tmpdir, pkg, pkg)
	index := index.Create(filepath.Join(*unpackedPath, pkg+".idx"))
	stripLen := len(filepath.Join(tmpdir, pkg))

	filepath.Walk(unpacked,
		func(path string, info os.FileInfo, err error) error {
			if dir, filename := filepath.Split(path); filename != "" {
				skip := ignored(info, dir, filename)
				if skip && info.IsDir() {
					if err := os.RemoveAll(path); err != nil {
						log.Fatalf("Could not remove directory %q: %v\n", path, err)
					}
					return filepath.SkipDir
				}
				if skip && !info.IsDir() {
					if err := os.Remove(path); err != nil {
						log.Fatalf("Could not remove file %q: %v\n", path, err)
					}
					return nil
				}
			}

			if info == nil || !info.Mode().IsRegular() {
				return nil
			}

			// Some filenames (e.g.
			// "xblast-tnt-levels_20050106-2/reconstruct\xeeon2.xal") contain
			// invalid UTF-8 and will break when sending them via JSON later
			// on. Filter those out early to avoid breakage.
			if !utf8.ValidString(path) {
				log.Printf("Skipping due to invalid UTF-8: %s\n", path)
				return nil
			}

			if err := index.AddFile(path, path[stripLen:]); err != nil {
				if err := os.Remove(path); err != nil {
					log.Fatalf("Could not remove file %q: %v\n", path, err)
				}
			} else {
				// Copy this file out of /tmp to our unpacked directory.
				outputPath := filepath.Join(*unpackedPath, path[stripLen:])
				if err := os.MkdirAll(filepath.Dir(outputPath), os.FileMode(0755)); err != nil {
					log.Fatalf("Could not create directory: %v\n", err)
				}
				output, err := os.Create(outputPath)
				if err != nil {
					log.Fatalf("Could not create output file %q: %v\n", outputPath, err)
				}
				defer output.Close()
				input, err := os.Open(path)
				if err != nil {
					log.Fatalf("Could not open input file %q: %v\n", path, err)
				}
				defer input.Close()
				if _, err := io.Copy(output, input); err != nil {
					log.Fatalf("Could not copy %q to %q: %v\n", path, outputPath, err)
				}
			}
			return nil
		})

	index.Flush()
}