func main() {
sourceDir, destDir, destSubfolder, fileType, mode := "", "", "", "", ""
log.SetPrefix(fmt.Sprintf("cache-monitor (PID %d) ", os.Getpid()))
var debug bool
flag.StringVar(&sourceDir, "sourcedir", "", "The source directory where this cache-monitor should pull files from. The cache manager automatically watches files in base/, mobile/, and desktop/, calling the handler for each modified file.")
flag.StringVar(&destDir, "destdir", "", "The cache directory where this cache-monitor should place files in. If this directory does not exist, it is created. Subfolders for each different layout are generated automatically.")
flag.StringVar(&destSubfolder, "destsubfolder", "", "Do you want the files to end up in a subdirectory? This is different from changing the destdir in that the subfolder is applied to the path after the layout. E.g. setting the subfolder to 'events' would make the files end up in $sourcedir/$layout/$events/$file for each layout.")
flag.StringVar(&fileType, "filetype", "", "What filetype is this cache-monitor monitoring? This generally, but not always, corresponds to the file extension. For example, 'js', 'css', 'img', and 'tmpl' are all valid types.")
flag.StringVar(&mode, "mode", "sync", "What mode should the monitor run in? Valid options include:\n\tsync: Runs once, performing a one-way sync of all files.\n\tdeamon: Runs in the background, updating files as they are changed. Currently only supported on Linux.")
flag.BoolVar(&debug, "debug", false, "Run in debug mode? Outputs a lot more garbage.")
flag.Parse()
if debug {
dlog = log.New(os.Stderr, "DEBUG: ", log.Ltime|log.Lshortfile)
} else {
nul, err := os.OpenFile(os.DevNull, os.O_WRONLY, 0)
if err != nil {
log.Fatal("Could not open null device? Wtf?")
}
dlog = log.New(nul, "", 0)
}
if mode == "deamon" {
dlog.Println("Warning: Deamon mode is still beta. Currently it only works on linux systems with inotify support.")
}
var v Visitor
v.source = new(Path)
v.dest = new(Path)
v.source.root = sourceDir
v.dest.root = destDir
v.dest.subfolder = destSubfolder
v.fileType = fileType
var err error
if mode == "deamon" {
v.deamon = true
v.watcher, err = inotify.NewWatcher()
if err != nil {
log.Fatal("Error creating inotify instance; deamon mode cannot possibly work.")
}
}
if !osutil.FileExists(sourceDir) {
dlog.Printf("Warning: Given source directory of %s does not exist, exiting...", sourceDir)
os.Exit(0)
}
filepath.Walk(sourceDir, filepath.WalkFunc(func(pat string, fi os.FileInfo, err error) error { return v.Visit(pat, fi, err) }))
if v.deamon {
v.InotifyLoop()
}
}
func main() {
flag.Parse()
root := flag.Arg(0)
fmt.Println("Root参数： ", root)
err := filepath.Walk(root, filepath.WalkFunc(myftw()))
if err != nil {
fmt.Errorf("遍历目录报错: %s", err)
}
fmt.Println("-----------------------")
fmt.Printf("目录总数：%d\n", ndir)
fmt.Printf("文件总数：%d\n", nreg)
}
func LoadEventsFromDisk() chan *Event {
events := make(chan *Event)
f := func(path string, info os.FileInfo, err error) error {
if !info.IsDir() {
ev := LoadEventFile(path)
events <- ev
}
return nil
}
go func() {
filepath.Walk("events", filepath.WalkFunc(f))
close(events)
}()
return events
}
// Create a zip of all files in a directory recursively, return a byte array and
// the number of files archived.
func ZipDir(path string) ([]byte, int, error) {
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
zipWalker := ZipWalker(zw)
numberManifests := 0
err := filepath.Walk(path, filepath.WalkFunc(func(path string, info os.FileInfo, err error) error {
if !info.IsDir() {
numberManifests++
}
return zipWalker(path, info, err)
}))
if err != nil {
return nil, 0, err
} else if err = zw.Close(); err != nil {
return nil, 0, err
}
return buf.Bytes(), numberManifests, nil
}
// Create a zip of all files in a directory recursively, return a byte array and
// the number of files archived.
func ZipDir(path string) ([]byte, []string, error) {
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
zipWalker := ZipWalker(zw)
paths := []string{}
err := filepath.Walk(path, filepath.WalkFunc(func(path string, info os.FileInfo, err error) error {
if !info.IsDir() {
paths = append(paths, path)
}
return zipWalker(path, info, err)
}))
if err != nil {
return nil, nil, err
} else if err = zw.Close(); err != nil {
return nil, nil, err
}
return buf.Bytes(), paths, nil
}
// The coordinating function to show the different entries.
func viewEntry() {
entries = make([]string, 0)
err := filepath.Walk(".", filepath.WalkFunc(getFilenames))
if err != nil {
return
}
for {
clear()
listFilenames()
fmt.Printf("\nKies hier welk bericht u graag zou willen lezen, toets 0 om terug te keren naar het menu: ")
var i int
fmt.Scan(&i)
if i > 0 && i <= len(entries) {
showFile(i - 1)
} else {
return
}
}
}
func buildApp() {
buf := new(bytes.Buffer)
tw := tar.NewWriter(buf)
filepath.Walk(".", filepath.WalkFunc(func(path string, info os.FileInfo, err error) error {
if info.Mode().IsDir() {
return nil
}
if strings.HasPrefix(path, ".git") {
return nil
}
log.Printf("transferring %s", path)
fr, err := os.Open(path)
ensureNoError(err, "Unable to open file "+path)
defer fr.Close()
h, err := tar.FileInfoHeader(info, path)
ensureNoError(err, "Unable to construct tar info header for "+path)
err = tw.WriteHeader(h)
ensureNoError(err, "Unable to write tar headder for "+path)
_, err = io.Copy(tw, fr)
ensureNoError(err, "Unable to write file contents to tar "+path)
return nil
}))
tw.Close()
err := dockerClient.BuildImage(docker.BuildImageOptions{
Name: serviceName(),
InputStream: buf,
OutputStream: os.Stdout,
})
ensureNoError(err, "Unable to build image")
}
func (rl *RecursiveListing) get_walk_func() filepath.WalkFunc {
return filepath.WalkFunc(
func(path string, info os.FileInfo, err error) error {
if err == nil && !info.IsDir() && backend.IsVideo(filepath.Base(path)) {
rl.lock.Lock()
var new_file = backend.FileEntry{
Name: filepath.Base(path),
AbsPath: path,
IsDir: false,
IsAccessible: true,
IsVideo: true,
}
rl.video_files.PushBack(&new_file)
rl.lock.Unlock()
rl.update_chan <- 1
}
return nil
})
}
func logDirContents(t *testing.T, dirPath string) {
logWalkedPaths := filepath.WalkFunc(func(path string, info os.FileInfo, err error) error {
if err != nil {
t.Errorf("stat error for path %q: %s", path, err)
return nil
}
if info.IsDir() {
path = joinTrailingSep(path)
}
t.Logf("\t%s", path)
return nil
})
t.Logf("logging directory contents: %q", dirPath)
if err := filepath.Walk(dirPath, logWalkedPaths); err != nil {
t.Fatal(err)
}
}
func logDirContents(dirPath string) {
logWalkedPaths := filepath.WalkFunc(func(path string, info os.FileInfo, err error) error {
if err != nil {
fmt.Printf("stat error for path %q: %s\n", path, err)
return nil
}
if info.IsDir() {
path = joinTrailingSep(path)
}
fmt.Printf("\t%s\n", path)
return nil
})
fmt.Printf("logging directory contents: %q\n", dirPath)
if err := filepath.Walk(dirPath, logWalkedPaths); err != nil {
fmt.Printf("ERROR: %s\n", err)
}
}
func processAlbum(c *smugmug.Conn, album *smugmug.AlbumInfo) error {
path := album.Category.Name
if album.SubCategory != nil {
path = filepath.Join(path, album.SubCategory.Name)
}
path = filepath.Join(path, album.Title)
fullpath := filepath.Join(dir, path)
updated, err := time.ParseInLocation("2006-01-02 15:04:05", album.LastUpdated, time.Local)
if err != nil {
return fmt.Errorf("Unable to parse timestamp %q: %v", album.LastUpdated, err)
}
// see if we can skip this based on a time stamp
if fast {
info, err := os.Stat(fullpath)
if err == nil && info.IsDir() && info.ModTime().Equal(updated) {
log.Printf("Skipping %s [%s], timestamp of %s matches", path, album.URL, album.LastUpdated)
return nil
}
}
log.Printf("Processing %s [%s] (updated %s)", path, album.URL, album.LastUpdated)
// scan the local directory: map path to md5sum
localFiles := make(map[string]string)
if info, err := os.Stat(fullpath); err == nil && info.IsDir() {
if err := filepath.Walk(fullpath, filepath.WalkFunc(func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
suffix := path
if strings.HasPrefix(path, dir+"/") {
suffix = path[len(dir)+1:]
}
if info.IsDir() {
localFiles[suffix] = "directory"
return nil
}
// get an MD5 hash
h := md5.New()
f, err := os.Open(path)
if err != nil {
log.Printf("error opening %s: %v", path, err)
return err
}
defer f.Close()
if _, err = io.Copy(h, f); err != nil {
log.Printf("error reading %s: %v", path, err)
return err
}
sum := h.Sum(nil)
s := hex.EncodeToString(sum)
localFiles[suffix] = s
return nil
})); err != nil && err != os.ErrNotExist {
return fmt.Errorf("error walking local file system: %v", err)
}
}
// get full list of images from this album
images, err := c.Images(album)
if err != nil {
return fmt.Errorf("Images error: %v", err)
}
// process each image
for _, img := range images {
if err := syncFile(album, img, localFiles, dir); err != nil {
return fmt.Errorf("Error processing image %s from album %s in category %s: %v",
img.FileName, album.Title, album.Category.Name, err)
}
}
// delete extra files
if err = cleanup(localFiles, dir); err != nil {
return fmt.Errorf("Error cleaning up: %v", err)
}
// update the directory timestamp to match
if !dry {
if err = os.Chtimes(fullpath, updated, updated); err != nil {
return fmt.Errorf("failed to set timestamp on directory %s: %v", fullpath, err)
}
}
return nil
}
func makeTar(w io.Writer, workdir string) error {
zout := gzip.NewWriter(w)
tw := tar.NewWriter(zout)
err := filepath.Walk(workdir, filepath.WalkFunc(func(path string, fi os.FileInfo, err error) error {
if err != nil {
log.Printf("Error walking path %q: %v", path, err)
}
if fi == nil {
log.Printf("Odd: nil os.Fileinfo for path %q", path)
return nil
}
if !strings.HasPrefix(path, workdir) {
log.Panicf("walked filename %q doesn't begin with workdir %q", path, workdir)
}
name := path[len(workdir):]
// Chop of any leading / from filename, leftover from removing workdir.
if strings.HasPrefix(name, "/") {
name = name[1:]
}
if name == modtimeFile {
return nil
}
if fi.IsDir() {
if name != "" {
// Just return the top-level files in the directory.
return filepath.SkipDir
}
return nil
}
if !strings.HasSuffix(name, ".go") && fi.Size() > 10<<10 {
// Skip non-go files over some threshold
return nil
}
if fi.Size() > 1<<20 {
// Skip all files over some other threshold.
return nil
}
hdr, err := tarFileInfoHeader(fi, path)
if err != nil {
log.Printf("error making header of %q: %v", path, err)
return err
}
hdr.Name = name
hdr.Uname = "root"
hdr.Gname = "root"
hdr.Uid = 0
hdr.Gid = 0
// Force permissions to 0755 for executables, 0644 for everything else.
if fi.Mode().Perm()&0111 != 0 {
hdr.Mode = hdr.Mode&^0777 | 0755
} else {
hdr.Mode = hdr.Mode&^0777 | 0644
}
err = tw.WriteHeader(hdr)
if err != nil {
log.Printf("WriteHeader: %v", err)
return fmt.Errorf("Error writing file %q: %v", name, err)
}
r, err := os.Open(path)
if err != nil {
log.Printf("Open: %v", err)
return err
}
defer r.Close()
_, err = io.Copy(tw, r)
return err
}))
if err != nil {
return err
}
if err := tw.Close(); err != nil {
return err
}
if err := zout.Close(); err != nil {
return err
}
return nil
}
func NewAPIServiceListInspectPodsTest() testutils.Test {
return testutils.TestFunc(func(t *testing.T) {
ctx := testutils.NewRktRunCtx()
defer ctx.Cleanup()
svc := startAPIService(t, ctx)
defer stopAPIService(t, svc)
c, conn := newAPIClientOrFail(t, "localhost:15441")
defer conn.Close()
resp, err := c.ListPods(context.Background(), &v1alpha.ListPodsRequest{})
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if len(resp.Pods) != 0 {
t.Errorf("Unexpected result: %v, should see zero pods", resp.Pods)
}
patches := []string{"--exec=/inspect --print-msg=HELLO_API --exit-code=0"}
imageHash, err := patchImportAndFetchHash("rkt-inspect-print.aci", patches, t, ctx)
if err != nil {
t.Fatalf("%v", err)
}
imgID, err := types.NewHash(imageHash)
if err != nil {
t.Fatalf("Cannot generate types.Hash from %v: %v", imageHash, err)
}
podManifests := []struct {
mfst schema.PodManifest
net string
expectedExitCode int
}{
{
// 1, Good pod.
schema.PodManifest{
ACKind: schema.PodManifestKind,
ACVersion: schema.AppContainerVersion,
Apps: []schema.RuntimeApp{
{
Name: types.ACName("rkt-inspect"),
Image: schema.RuntimeImage{
Name: types.MustACIdentifier("coreos.com/rkt-inspect"),
ID: *imgID,
},
Annotations: []types.Annotation{{Name: types.ACIdentifier("app-test"), Value: "app-test"}},
},
},
Annotations: []types.Annotation{
{Name: types.ACIdentifier("test"), Value: "test"},
},
},
"default",
0,
},
{
// 2, Bad pod, won't be launched correctly.
schema.PodManifest{
ACKind: schema.PodManifestKind,
ACVersion: schema.AppContainerVersion,
Apps: []schema.RuntimeApp{
{
Name: types.ACName("rkt-inspect"),
Image: schema.RuntimeImage{
Name: types.MustACIdentifier("coreos.com/rkt-inspect"),
ID: *imgID,
},
},
},
},
"non-existent-network",
254,
},
}
// Launch the pods.
for _, entry := range podManifests {
manifestFile := generatePodManifestFile(t, &entry.mfst)
defer os.Remove(manifestFile)
runCmd := fmt.Sprintf("%s run --net=%s --pod-manifest=%s", ctx.Cmd(), entry.net, manifestFile)
waitOrFail(t, spawnOrFail(t, runCmd), entry.expectedExitCode)
}
time.Sleep(delta)
gcCmd := fmt.Sprintf("%s gc --mark-only=true", ctx.Cmd())
waitOrFail(t, spawnOrFail(t, gcCmd), 0)
gcTime := time.Now()
// ListPods(detail=false).
resp, err = c.ListPods(context.Background(), &v1alpha.ListPodsRequest{})
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if len(resp.Pods) != len(podManifests) {
t.Errorf("Unexpected result: %v, should see %v pods", len(resp.Pods), len(podManifests))
}
for _, p := range resp.Pods {
checkPodBasicsWithGCTime(t, ctx, p, gcTime)
// Test InspectPod().
inspectResp, err := c.InspectPod(context.Background(), &v1alpha.InspectPodRequest{Id: p.Id})
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
checkPodDetails(t, ctx, inspectResp.Pod)
}
// ListPods(detail=true).
resp, err = c.ListPods(context.Background(), &v1alpha.ListPodsRequest{Detail: true})
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if len(resp.Pods) != len(podManifests) {
t.Errorf("Unexpected result: %v, should see %v pods", len(resp.Pods), len(podManifests))
}
for _, p := range resp.Pods {
checkPodDetails(t, ctx, p)
}
// ListPods with corrupt pod directory
// Note that we don't checkPodDetails here, the failure this is testing is
// the api server panicing, which results in a list call hanging for ages
// and then failing.
// TODO: do further validation on the partial pods returned
for _, p := range resp.Pods {
numRemoved := 0
podDir := getPodDir(t, ctx, p.Id)
filepath.Walk(filepath.Join(podDir, "appsinfo"), filepath.WalkFunc(func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.Name() == "manifest" {
os.Remove(path)
numRemoved++
}
return nil
}))
if numRemoved == 0 {
t.Fatalf("Expected to remove at least one app manifest for pod %v", p)
}
}
// ListPods(detail=true).
resp, err = c.ListPods(context.Background(), &v1alpha.ListPodsRequest{Detail: true})
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if len(resp.Pods) != len(podManifests) {
t.Fatalf("Expected %v pods, got %v pods", len(podManifests), len(resp.Pods))
}
})
}
func makeTar(targ, workdir string) error {
f, err := os.Create(targ)
if err != nil {
return err
}
zout := gzip.NewWriter(f)
tw := tar.NewWriter(zout)
filepath.Walk(workdir, filepath.WalkFunc(func(path string, fi os.FileInfo, err error) error {
if !strings.HasPrefix(path, workdir) {
log.Panicf("walked filename %q doesn't begin with workdir %q", path, workdir)
}
name := path[len(workdir):]
// Chop of any leading / from filename, leftover from removing workdir.
if strings.HasPrefix(name, "/") {
name = name[1:]
}
// Don't include things outside of the go subdirectory (for instance,
// the zip file that we're currently writing here.)
if !strings.HasPrefix(name, "go/") {
return nil
}
if *verbose {
log.Printf("adding to tar: %s", name)
}
if fi.IsDir() {
return nil
}
hdr, err := tarFileInfoHeader(fi, path)
if err != nil {
return err
}
hdr.Name = name
hdr.Uname = "root"
hdr.Gname = "root"
hdr.Uid = 0
hdr.Gid = 0
// Force permissions to 0755 for executables, 0644 for everything else.
if fi.Mode().Perm()&0111 != 0 {
hdr.Mode = hdr.Mode&^0777 | 0755
} else {
hdr.Mode = hdr.Mode&^0777 | 0644
}
err = tw.WriteHeader(hdr)
if err != nil {
return fmt.Errorf("Error writing file %q: %v", name, err)
}
r, err := os.Open(path)
if err != nil {
return err
}
defer r.Close()
_, err = io.Copy(tw, r)
return err
}))
if err := tw.Close(); err != nil {
return err
}
if err := zout.Close(); err != nil {
return err
}
return f.Close()
}
func parseRepo(repo, out, owner string, bare bool) error {
os.MkdirAll(out, 0775)
site, err := handleSiteDescription(repo, out)
if err != nil {
fmt.Println(err)
return err
}
if site == nil {
println("No website.json found, cannot generate website.")
return nil
}
fmt.Printf("Processing website: %s\n", site.Title)
for i, head := range site.Headings {
progress("(%d/%d) Heading: %s -> %s", i+1, len(site.Headings), head.Title, head.Url)
}
fmt.Println("Processing pages")
err = handlePages(site, out)
if err != nil {
return err
}
fmt.Println("Processing articles")
err = handleArticles(site, out)
if err != nil {
return err
}
// Sort the articles to newest first
sort.Sort(sort.Reverse(ArticleByDate(site.Articles)))
println("Creating homepage")
err = createHomepage(site, out)
if err != nil {
progress(err.Error())
return err
}
progress("Successfully generated homepage")
println("Creating archive index pages")
err = createIndices(site, out)
if err != nil {
progress(err.Error())
return err
}
println("Copying media files into place")
if len(site.Media) == 0 {
progress("No media directories")
} else {
err = handleMedia(site, out)
if err != nil {
return err
}
}
if owner != "" {
usr, err := user.Lookup(owner)
if err != nil {
fmt.Printf("Failed to set owner to '%s'\n", owner)
return err
}
err = filepath.Walk(out, filepath.WalkFunc(func(path string, info os.FileInfo, err error) error {
uid, _ := strconv.ParseInt(usr.Uid, 10, 32)
gid, _ := strconv.ParseInt(usr.Gid, 10, 32)
return os.Chown(path, int(uid), int(gid))
}))
if err != nil {
return err
}
}
return nil
}
