Beispiel #1
0
func walkDir(client *hdfs.Client, dir string, visit walkFunc) error {
	dirReader, err := client.Open(dir)
	if err != nil {
		return err
	}

	var partial []os.FileInfo
	for ; err != io.EOF; partial, err = dirReader.Readdir(100) {
		if err != nil {
			return err
		}

		for _, child := range partial {
			childPath := path.Join(dir, child.Name())
			visit(childPath, child)

			if child.IsDir() {
				err = walkDir(client, childPath, visit)
				if err != nil {
					return err
				}
			}
		}
	}

	return nil
}
Beispiel #2
0
// expandGlobs recursively expands globs in a filepath. It assumes the paths
// are already cleaned and normalize (ie, absolute).
func expandGlobs(client *hdfs.Client, globbedPath string) ([]string, error) {
	parts := strings.Split(globbedPath, "/")[1:]
	var res []string
	var splitAt int

	for splitAt = range parts {
		if hasGlob(parts[splitAt]) {
			break
		}
	}

	var base, glob, next, remainder string
	base = "/" + path.Join(parts[:splitAt]...)
	glob = parts[splitAt]

	if len(parts) > splitAt+1 {
		next = parts[splitAt+1]
		remainder = path.Join(parts[splitAt+2:]...)
	} else {
		next = ""
		remainder = ""
	}

	list, err := client.ReadDir(base)
	if err != nil {
		return nil, err
	}

	for _, fi := range list {
		match, _ := path.Match(glob, fi.Name())
		if !match {
			continue
		}

		if !hasGlob(next) {
			_, err := client.Stat(path.Join(base, fi.Name(), next))
			if err != nil && !os.IsNotExist(err) {
				return nil, err
			} else if os.IsNotExist(err) {
				continue
			}
		}

		newPath := path.Join(base, fi.Name(), next, remainder)
		if hasGlob(newPath) {
			children, err := expandGlobs(client, newPath)
			if err != nil {
				return nil, err
			}

			res = append(res, children...)
		} else {
			res = append(res, newPath)
		}
	}

	return res, nil
}
Beispiel #3
0
func moveTo(client *hdfs.Client, source, dest string, force bool) {
	if force {
		err := client.Remove(dest)
		if err != nil && !os.IsNotExist(err) {
			fatal(err)
		}
	}

	err := client.Rename(source, dest)
	if err != nil {
		fatal(err)
	}
}
Beispiel #4
0
func walk(client *hdfs.Client, root string, visit walkFunc) error {
	rootInfo, err := client.Stat(root)
	if err != nil {
		return err
	}

	visit(root, rootInfo)
	if rootInfo.IsDir() {
		err = walkDir(client, root, visit)
		if err != nil {
			return err
		}
	}

	return nil
}
Beispiel #5
0
func duDir(client *hdfs.Client, tw *tabwriter.Writer, dir string, humanReadable bool) int64 {
	dirReader, err := client.Open(dir)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return 0
	}

	var partial []os.FileInfo
	var dirSize int64
	for ; err != io.EOF; partial, err = dirReader.Readdir(100) {
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			return dirSize
		}

		for _, child := range partial {
			childPath := path.Join(dir, child.Name())
			info, err := client.Stat(childPath)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				return 0
			}

			var size int64
			if info.IsDir() {
				size = duDir(client, tw, childPath, humanReadable)
			} else {
				size = info.Size()
			}

			printSize(tw, size, childPath, humanReadable)
			dirSize += size
		}
	}

	return dirSize
}
Beispiel #6
0
func printDir(client *hdfs.Client, dir string, long, all, humanReadable bool) {
	dirReader, err := client.Open(dir)
	if err != nil {
		fatal(err)
	}

	var tw *tabwriter.Writer
	if long {
		tw = lsTabWriter()
		defer tw.Flush()
	}

	if all {
		if long {
			dirInfo, err := client.Stat(dir)
			if err != nil {
				fatal(err)
			}

			parentPath := path.Join(dir, "..")
			parentInfo, err := client.Stat(parentPath)
			if err != nil {
				fatal(err)
			}

			printLong(tw, ".", dirInfo, humanReadable)
			printLong(tw, "..", parentInfo, humanReadable)
		} else {
			fmt.Println(".")
			fmt.Println("..")
		}
	}

	var partial []os.FileInfo
	for ; err != io.EOF; partial, err = dirReader.Readdir(100) {
		if err != nil {
			fatal(err)
		}

		printFiles(tw, partial, long, all, humanReadable)
	}

	if long {
		tw.Flush()
	}
}