func Test296(t *testing.T) {
	lion.SetLevel(lion.LevelDebug)
	if testing.Short() {
		t.Skip("Skipped because of short mode")
	}

	testFuse(t, func(c client.APIClient, mountpoint string) {
		repo := "test"
		require.NoError(t, c.CreateRepo(repo))
		commit, err := c.StartCommit(repo, "", "")
		require.NoError(t, err)
		path := filepath.Join(mountpoint, repo, commit.ID, "file")
		stdin := strings.NewReader(fmt.Sprintf("echo 1 >%s", path))
		require.NoError(t, pkgexec.RunStdin(stdin, "sh"))
		stdin = strings.NewReader(fmt.Sprintf("echo 2 >%s", path))
		require.NoError(t, pkgexec.RunStdin(stdin, "sh"))
		require.NoError(t, c.FinishCommit(repo, commit.ID))
		commit2, err := c.StartCommit(repo, commit.ID, "")
		require.NoError(t, err)
		path = filepath.Join(mountpoint, repo, commit2.ID, "file")
		stdin = strings.NewReader(fmt.Sprintf("echo 3 >%s", path))
		require.NoError(t, pkgexec.RunStdin(stdin, "sh"))
		require.NoError(t, c.FinishCommit(repo, commit2.ID))
		data, err := ioutil.ReadFile(path)
		require.NoError(t, err)
		require.Equal(t, "3\n", string(data))
	})
}
func TestCreateFileInDir(t *testing.T) {
	lion.SetLevel(lion.LevelDebug)
	if testing.Short() {
		t.Skip("Skipped because of short mode")
	}
	testFuse(t, func(c client.APIClient, mountpoint string) {
		require.NoError(t, c.CreateRepo("repo"))
		commit, err := c.StartCommit("repo", "", "")
		require.NoError(t, err)

		require.NoError(t, os.Mkdir(filepath.Join(mountpoint, "repo", commit.ID, "dir"), 0700))
		require.NoError(t, ioutil.WriteFile(filepath.Join(mountpoint, "repo", commit.ID, "dir", "file"), []byte("foo"), 0644))
		require.NoError(t, c.FinishCommit("repo", commit.ID))
	})
}
Example #3
0
// SetupEnv sets up logging for the given Env.
func SetupEnv(env Env) error {
	var pushers []lion.Pusher
	logAppName := env.LogAppName
	if logAppName == "" {
		logAppName = "app"
	}
	if !env.LogDisableStderr {
		pushers = append(pushers, lion.NewTextWritePusher(os.Stderr))
	}
	if env.CurrentStdout {
		pushers = append(pushers, lion.NewWritePusher(os.Stdout, currentlion.NewMarshaller()))
	}
	if env.LogDirPath != "" {
		pushers = append(pushers, newLogDirPusher(env.LogDirPath, logAppName))
	}
	if env.SyslogNetwork != "" && env.SyslogAddress != "" {
		pusher, err := newSyslogPusher(env.SyslogNetwork, env.SyslogAddress, logAppName)
		if err != nil {
			return err
		}
		pushers = append(pushers, pusher)
	}
	if env.CurrentToken != "" && env.CurrentSyslogAddress != "" {
		pusher, err := currentlion.NewPusher(logAppName, env.CurrentSyslogAddress, env.CurrentToken)
		if err != nil {
			return err
		}
		pushers = append(pushers, pusher)
	}
	switch len(pushers) {
	case 0:
		lion.SetLogger(lion.DiscardLogger)
	case 1:
		lion.SetLogger(lion.NewLogger(pushers[0]))
	default:
		lion.SetLogger(lion.NewLogger(lion.NewMultiPusher(pushers...)))
	}
	lion.RedirectStdLogger()
	if env.LogLevel != "" {
		level, err := lion.NameToLevel(strings.ToUpper(env.LogLevel))
		if err != nil {
			return err
		}
		lion.SetLevel(level)
	}
	return nil
}
Example #4
0
func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	rootCmd := &cobra.Command{
		Use:   os.Args[0] + " job-id",
		Short: `Pachyderm job-shim, coordinates with ppsd to create an output commit and run user work.`,
		Long:  `Pachyderm job-shim, coordinates with ppsd to create an output commit and run user work.`,
		Run: func(_ *cobra.Command, args []string) {
			ppsClient, err := ppsserver.NewInternalJobAPIClientFromAddress(fmt.Sprintf("%v:650", appEnv.PachydermAddress))
			if err != nil {
				errorAndExit(err.Error())
			}
			response, err := ppsClient.StartJob(
				context.Background(),
				&ppsserver.StartJobRequest{
					Job: &ppsclient.Job{
						ID: args[0],
					}})
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s\n", err.Error())
				os.Exit(0)
			}

			if response.Transform.Debug {
				lion.SetLevel(lion.LevelDebug)
			}

			// We want to make sure that we only send FinishJob once.
			// The most bulletproof way would be to check that on server side,
			// but this is easier.
			var finished bool
			// Make sure that we call FinishJob even if something caused a panic
			defer func() {
				if r := recover(); r != nil && !finished {
					fmt.Println("job shim crashed; this is like a bug in pachyderm")
					if _, err := ppsClient.FinishJob(
						context.Background(),
						&ppsserver.FinishJobRequest{
							Job: &ppsclient.Job{
								ID: args[0],
							},
							Success: false,
						},
					); err != nil {
						errorAndExit(err.Error())
					}
				}
			}()

			c, err := client.NewFromAddress(fmt.Sprintf("%v:650", appEnv.PachydermAddress))
			if err != nil {
				errorAndExit(err.Error())
			}

			mounter := fuse.NewMounter(appEnv.PachydermAddress, c.PfsAPIClient)
			ready := make(chan bool)
			go func() {
				if err := mounter.MountAndCreate(
					"/pfs",
					nil,
					response.CommitMounts,
					ready,
					response.Transform.Debug,
				); err != nil {
					errorAndExit(err.Error())
				}
			}()
			<-ready
			defer func() {
				if err := mounter.Unmount("/pfs"); err != nil {
					errorAndExit(err.Error())
				}
			}()
			var readers []io.Reader
			for _, line := range response.Transform.Stdin {
				readers = append(readers, strings.NewReader(line+"\n"))
			}
			if len(response.Transform.Cmd) == 0 {
				fmt.Println("unable to run; a cmd needs to be provided")
				if _, err := ppsClient.FinishJob(
					context.Background(),
					&ppsserver.FinishJobRequest{
						Job:     client.NewJob(args[0]),
						Success: false,
					},
				); err != nil {
					errorAndExit(err.Error())
				}
				finished = true
				return
			}
			cmd := exec.Command(response.Transform.Cmd[0], response.Transform.Cmd[1:]...)
			cmd.Stdin = io.MultiReader(readers...)
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			success := true
			if err := cmd.Run(); err != nil {
				success = false
				if exiterr, ok := err.(*exec.ExitError); ok {
					if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
						for _, returnCode := range response.Transform.AcceptReturnCode {
							if int(returnCode) == status.ExitStatus() {
								success = true
							}
						}
					}
				}
				if !success {
					fmt.Fprintf(os.Stderr, "Error from exec: %s\n", err.Error())
				}
			}
			if _, err := ppsClient.FinishJob(
				context.Background(),
				&ppsserver.FinishJobRequest{
					Job:     client.NewJob(args[0]),
					Success: success,
				},
			); err != nil {
				errorAndExit(err.Error())
			}
			finished = true
		},
	}

	return rootCmd.Execute()
}
Example #5
0
func do(appEnvObj interface{}) error {
	lion.SetLevel(lion.LevelDebug)
	appEnv := appEnvObj.(*appEnv)
	rootCmd := &cobra.Command{
		Use:   os.Args[0] + " job-id",
		Short: `Pachyderm job-shim, coordinates with ppsd to create an output commit and run user work.`,
		Long:  `Pachyderm job-shim, coordinates with ppsd to create an output commit and run user work.`,
		Run: func(cmd *cobra.Command, args []string) {
			pfsAPIClient, err := getPfsAPIClient(getPfsdAddress(appEnv))
			if err != nil {
				errorAndExit(err.Error())
			}

			ppsAPIClient, err := getPpsAPIClient(getPpsdAddress(appEnv))
			if err != nil {
				errorAndExit(err.Error())
			}

			response, err := ppsAPIClient.StartJob(
				context.Background(),
				&pps.StartJobRequest{
					Job: &pps.Job{
						Id: args[0],
					}})
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s\n", err.Error())
				os.Exit(0)
			}

			mounter := fuse.NewMounter(getPfsdAddress(appEnv), pfsAPIClient)
			ready := make(chan bool)
			go func() {
				if err := mounter.Mount(
					"/pfs",
					nil,
					response.CommitMounts,
					ready,
				); err != nil {
					errorAndExit(err.Error())
				}
			}()
			<-ready
			defer func() {
				if err := mounter.Unmount("/pfs"); err != nil {
					errorAndExit(err.Error())
				}
			}()
			io := pkgexec.IO{
				Stdin:  strings.NewReader(response.Transform.Stdin),
				Stdout: os.Stdout,
				Stderr: os.Stderr,
			}
			success := true
			if err := pkgexec.RunIO(io, response.Transform.Cmd...); err != nil {
				fmt.Fprintf(os.Stderr, "%s\n", err.Error())
				success = false
			}
			if _, err := ppsAPIClient.FinishJob(
				context.Background(),
				&pps.FinishJobRequest{
					Job: &pps.Job{
						Id: args[0],
					},
					Index:   response.Index,
					Success: success,
				},
			); err != nil {
				errorAndExit(err.Error())
			}
		},
	}

	return rootCmd.Execute()
}
Example #6
0
// SetupLogging sets up logging.
func SetupLogging(appName string, env Env) error {
	var pushers []lion.Pusher
	if !env.DisableStderrLog {
		pushers = append(
			pushers,
			lion.NewTextWritePusher(
				os.Stderr,
			),
		)
	}
	if env.LogDir != "" {
		pushers = append(
			pushers,
			lion.NewTextWritePusher(
				&lumberjack.Logger{
					Filename:   filepath.Join(env.LogDir, fmt.Sprintf("%s.log", appName)),
					MaxBackups: 3,
				},
			),
		)
	}
	if env.SyslogNetwork != "" && env.SyslogAddress != "" {
		writer, err := syslog.Dial(
			env.SyslogNetwork,
			env.SyslogAddress,
			syslog.LOG_INFO,
			appName,
		)
		if err != nil {
			return err
		}
		pushers = append(
			pushers,
			sysloglion.NewPusher(
				writer,
			),
		)
	}
	if len(pushers) > 0 {
		lion.SetLogger(
			lion.NewLogger(
				lion.NewMultiPusher(
					pushers...,
				),
			),
		)
	} else {
		lion.SetLogger(
			lion.DiscardLogger,
		)
	}
	lion.RedirectStdLogger()
	if env.LogLevel != "" {
		level, err := lion.NameToLevel(strings.ToUpper(env.LogLevel))
		if err != nil {
			return err
		}
		lion.SetLevel(level)
	}
	return nil
}