Beispiel #1
0
func download(c cli.Command) {
	var (
		arglen     = len(c.Args())
		objectName string
		path       = c.Flag("path").String()
	)
	switch arglen {
	case 1:
		objectName = c.Arg(0).String()
	case 2:
		objectName = c.Arg(1).String()
		container = c.Arg(0).String()
	}
	if blank(container) || blank(objectName) {
		log.Fatal(errorNotEnough)
	}
	if blank(path) {
		path = objectName
	}
	reader, err := api.Container(container).Object(objectName).GetReader()
	if err != nil {
		log.Fatal(err)
	}
	defer reader.Close()
	fmt.Printf("downloading %s->%s from %s\n", objectName, path, container)
	f, err := os.Create(path)
	if err != nil {
		log.Fatal(err)
	}
	n, err := io.Copy(f, reader)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("downloaded %s, %d bytes\n", objectName, n)
}
Beispiel #2
0
func roleRm(c cli.Command) {
	rmRoles := c.Args().Strings()
	if len(rmRoles) == 0 {
		log.Fatalln("Must specify some role[s] to remove")
	}

	nodeRoles, err := allNodeRoles()
	if err != nil {
		log.Fatalln("err: ", err)
	}

	myKey, err := selfKey()
	if err != nil {
		log.Fatalln("err: ", err)
	}

	var finalSet []string
	for _, role := range nodeRoles[myKey] {
		if !StrContains(rmRoles, role) {
			finalSet = append(finalSet, role)
		} else {
			fmt.Printf("removing role `%s`\n", role)
		}
	}
	roleActualSet(finalSet, c)
}
Beispiel #3
0
func (v *Run) Run(c cli.Command) {

	jobName := c.Param("jobname").String()
	log.WithFields(
		log.Fields{
			"pkg":  "command",
			"func": "Run",
			"job":  jobName,
		},
	).Info("Starting job")

	j := job.New(metl.GetJobFilePath(jobName))

	j.Lock()

	jf, err := j.Fetch()
	if err != nil {
		log.Fatal("Error fetching job:", err)
	}

	jf.Run()

	j.Unlock()
	j.Done(jf)

	// log stats, notify of errors etc
	// notify of any errors

	// log stats to dashboard, number of processed, rejected rows etc.

	// if parsing option is empty then, we just download the file
}
Beispiel #4
0
func cmRoll(c cli.Command) {
	role := c.Flag("role").String()
	if len(role) == 0 && c.Flag("force").Get() != true {
		log.Fatalln("Must specify -f option to run with no `role` filter specified")
	} else {
		cmRunRoll(role, "")
	}
}
Beispiel #5
0
func nodeRun(c cli.Command) {
	switch c.Param("action").String() {
	case "list":
		nodeList(c)
	default:
		cli.ShowUsage(c)
	}
}
Beispiel #6
0
func (v *Add) Run(c cli.Command) {
	jobName := c.Param("jobname").String()
	var job job.Job
	if _, err := toml.DecodeFile(metl.GetJobFilePath(jobName)+".toml", &job); err != nil {
		log.Fatal(err)
	}

	cronfile := filepath.Join(metl.Etl.GetLocalStoragePath(), crontabFile)

	file, err := os.Open(cronfile)
	if err != nil {
		file, err = os.Create(cronfile)
		if err != nil {
			log.WithFields(
				log.Fields{
					"filename": cronfile,
				},
			).Fatal("Unable to create cronfile: ", err)
		}
	}
	defer file.Close()

	err = syscall.Flock(int(file.Fd()), syscall.LOCK_EX)
	if err != nil {
		log.Fatal("Failed to lock file")
	}

	defer func() {
		err = syscall.Flock(int(file.Fd()), syscall.LOCK_UN)
		if err != nil {
			log.Fatal("Failed to unlock file")
		}
	}()

	buffer := make([]string, 0)
	if err = jobIsInFile(file, jobName, &buffer); err != nil {
		log.Fatal(err)
	}
	buffer = append(buffer, fmt.Sprintf("%s %s >> %s%s.log", cmd, jobName, logDir, jobName))

	tmpf, err := ioutil.TempFile(metl.Etl.GetLocalStoragePath(), "tmp")
	if err != nil {
		log.Fatal("Unable to create temporary file.")
	}

	for _, line := range buffer {
		if _, err := tmpf.WriteString(line + "\n"); err != nil {
			log.Fatal("Unable to write to temporary file")
		}
	}

	if err := os.Rename(tmpf.Name(), cronfile); err != nil {
		log.Fatal("Failed to rename temporary lock file:", err)
	}

	// execute /usr/bin/env crontab cronfile
}
Beispiel #7
0
func (etl *metl) runner(c cli.Command) {
	etl.preRunnerTasks()

	runnable := c.Name()
	if runner, ok := etl.runnables[runnable]; ok {
		runner.Run(c)
	} else {
		panic(fmt.Sprintf("Runnable %s not defined!", runnable))
	}
}
Beispiel #8
0
func create(c cli.Command) {
	if len(c.Args()) == 0 {
		log.Fatal(errorNotEnough)
	}
	var name = c.Arg(0).String()
	if _, err := api.CreateContainer(name, false); err != nil {
		log.Fatal(err)
	}
	fmt.Printf("created container %s\n", name)
}
Beispiel #9
0
func roleRun(c cli.Command) {
	switch c.Param("action").String() {
	case "list":
		roleList(c)
	case "set":
		roleSet(c)
	default:
		cli.ShowUsage(c)
	}
}
Beispiel #10
0
func cmRun(c cli.Command) {
	switch c.Param("action").String() {
	case "local":
		cmLocal(c)
	case "roll":
		cmRoll(c)
	case "single":
		cmSingle(c)
	default:
		cli.ShowUsage(c)
	}
}
Beispiel #11
0
func serviceRun(c cli.Command) {
	switch c.Param("action").String() {
	case "list":
		serviceList(c)
	case "local":
		serviceLocal(c)
	case "find":
		serviceFind(c)
	default:
		cli.ShowUsage(c)
	}
}
Beispiel #12
0
func greetGreetee(c cli.Command) {
	greeting := c.Parent().Param("greeting")
	greetee := c.Param("greetee")
	str := fmt.Sprintf("%s %s", greeting, greetee)
	str = styleByFlags(str, c)
	c.Println(str, strings.Join(c.Args().Strings(), " "))
}
Beispiel #13
0
func upload(c cli.Command) {
	var path string
	switch len(c.Args()) {
	case 1:
		path = c.Arg(0).String()
	case 2:
		container = c.Arg(0).String()
		path = c.Arg(1).String()
	}
	if blank(container) || blank(path) {
		log.Fatal(errorNotEnough)
	}

	f, err := os.Open(path)
	if err != nil {
		log.Fatal(err)
	}

	stat, err := os.Stat(path)
	if err != nil {
		log.Fatal(err)
	}
	ext := filepath.Ext(path)
	mimetype := mime.TypeByExtension(ext)
	bar := pb.New64(stat.Size()).SetUnits(pb.U_BYTES)
	bar.Start()
	reader := io.TeeReader(f, bar)
	if err := api.Container(container).Upload(reader, stat.Name(), mimetype); err != nil {
		log.Fatal(err)
	}
	fmt.Printf("uploaded to %s\n", container)
}
Beispiel #14
0
func roleSet(c cli.Command) {
	client, _ := api.NewClient(api.DefaultConfig())
	agent := client.Agent()

	reg := &api.AgentServiceRegistration{
		Name: "cascade",
		Tags: c.Args().Strings(),
	}

	if err := agent.ServiceRegister(reg); err != nil {
		log.Fatalln("err: ", err)
	}

	roleList(c)
}
Beispiel #15
0
func nodeList(c cli.Command) {
	client, _ := api.NewClient(api.DefaultConfig())
	catalog := client.Catalog()

	nodes, _, err := catalog.Service("cascade", c.Flag("role").String(), nil)

	if err != nil {
		log.Fatalln("Err: ", err)
	}

	for _, node := range nodes {
		fmt.Println(node.Node, node.Address+":")
		for _, role := range node.ServiceTags {
			fmt.Println("  -", role)
		}
	}
}
Beispiel #16
0
func roleRun(c cli.Command) {
	switch c.Param("action").String() {
	case "list":
		roleList(c)
	case "set":
		roleSet(c)
	case "listAll":
		roleListAll(c)
	case "find":
		roleFind(c)
	case "append":
		roleAppend(c)
	case "rm":
		roleRm(c)
	default:
		cli.ShowUsage(c)
	}
}
Beispiel #17
0
func cmSingle(c cli.Command) {
	client, _ := api.NewClient(api.DefaultConfig())
	catalog := client.Catalog()

	node, _, err := catalog.Node(c.Arg(0).String(), nil)

	if err != nil {
		log.Fatalln("err: ", err)
	}

	if node == nil {
		log.Fatalln("node not found")
	}

	if node.Services["cascade"] == nil {
		log.Fatalln("node not managed by cascade")
	}

	cmRunRoll("", c.Arg(0).String())
}
Beispiel #18
0
func list(c cli.Command) {
	var (
		arglen = len(c.Args())
		table  = tablewriter.NewWriter(os.Stdout)
	)
	if arglen == 0 && (blank(container) || c.Flag("type").String() == "storage") {
		containers, err := api.ContainersInfo()
		if err != nil {
			log.Fatal(err)
		}
		table.SetHeader([]string{"Name", "Objects", "Type"})
		for _, cont := range containers {
			v := []string{cont.Name, fmt.Sprint(cont.ObjectCount), cont.Type}
			table.Append(v)
		}
		table.Render()
		return
	}
	if arglen == 1 {
		container = c.Arg(0).String()
	}
	if blank(container) {
		log.Fatal(errorNotEnough)
	}
	objects, err := api.Container(container).ObjectsInfo()
	if err != nil {
		log.Fatal(err)
	}
	table.SetHeader([]string{"Name", "Size", "Downloaded"})
	for _, object := range objects {
		v := []string{object.Name, fmt.Sprint(object.Size), fmt.Sprint(object.Downloaded)}
		table.Append(v)
	}
	table.Render()
}
Beispiel #19
0
func roleFind(c cli.Command) {
	cmdRoles := c.Args().Strings()
	if len(cmdRoles) == 0 {
		log.Fatalln("Must specify a role to find")
	}
	if len(cmdRoles) != 1 {
		// maybe we could support multiple but i don't think it's necessary
		log.Fatalln("One role only is supported for `find` command")
	}
	role := cmdRoles[0]
	allRoles, err := allNodeRoles()
	if err != nil {
		log.Fatalln("err: ", err)
	}

	fmt.Printf("All nodes containing role %s:\n\n", role)
	for node, r := range allRoles {
		if StrContains(r, role) {
			printRole(node, r)
		}
	}
}
Beispiel #20
0
func styleByFlags(str string, c cli.Command) string {
	if c.Flag("loudly").Get() == true {
		str = louden(str)
	}
	if c.Flag("color").String() != "" {
		str = colorize(str, c.Flag("color").String())
	}
	return str
}
Beispiel #21
0
func roleAppend(c cli.Command) {
	nodeRoles, err := allNodeRoles()
	if err != nil {
		log.Fatalln("err: ", err)
	}

	myKey, err := selfKey()
	if err != nil {
		log.Fatalln("err: ", err)
	}

	var finalSet []string
	for _, role := range nodeRoles[myKey] {
		finalSet = append(finalSet, role)
	}

	for _, role := range c.Args().Strings() {
		if !StrContains(finalSet, role) {
			finalSet = append(finalSet, role)
		}
	}

	roleActualSet(finalSet, c)
}
Beispiel #22
0
func serviceFind(c cli.Command) {
	client, _ := api.NewClient(api.DefaultConfig())
	catalog := client.Catalog()

	if len(c.Args().GetAll()) == 0 {
		log.Fatalln("err: missing <servicename> argument")
	}

	nodes, _, err := catalog.Service(c.Arg(0).String(), c.Flag("type").String(), nil)

	if err != nil {
		log.Fatalln("err: ", err)
	}

	fmt.Println(c.Arg(0).String() + ":")
	for _, node := range nodes {
		fmt.Println("  - host:", node.Node)
		fmt.Println("    address:", node.Address)
		fmt.Println("    port:", node.ServicePort)
		fmt.Println("    tags:", strings.Join(node.ServiceTags, ", "))
	}
}
Beispiel #23
0
// the greet command run by the root command
func greet(c cli.Command) {
	greeting := c.Param("greeting")
	str := fmt.Sprintf("%s", greeting)
	str = styleByFlags(str, c)
	c.Println(str)
}
Beispiel #24
0
func readFlag(c cli.Command, name, env string) string {
	if len(os.Getenv(env)) > 0 {
		return os.Getenv(env)
	}
	return c.Flag(name).String()
}
Beispiel #25
0
// info prints information about storage
func info(c cli.Command) {
	var (
		containerName = container
		objectName    string
		data          interface{}
		err           error
		arglen        = len(c.Args())
		command       = c.Flag("type").String()
	)

	defer func() {
		if err != nil {
			log.Fatal(err)
		}
		if blank(containerName) || command == "storage" {
			data = api.Info()
		} else {
			containerApi := api.Container(containerName)
			if blank(objectName) {
				data, err = containerApi.Info()
			} else {
				data, err = containerApi.Object(objectName).Info()
			}
		}
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("%+v\n", data)
	}()

	if arglen > 0 {
		if command == "container" {
			containerName = c.Arg(0).String()
			return
		}
		command = "object"
		if !blank(containerName) && arglen == 1 {
			objectName = c.Arg(0).String()
			return
		}
		if arglen == 2 {
			containerName = c.Arg(0).String()
			objectName = c.Arg(1).String()
			return
		}
	}
	if command == "container" && !blank(containerName) {
		return
	}
	if command == "storage" {
		return
	}
	err = errorNotEnough
}
Beispiel #26
0
func roleSet(c cli.Command) {
	roleActualSet(c.Args().Strings(), c)
}
Beispiel #27
0
func cmRoll(c cli.Command) {
	cmRunRoll(c.Flag("role").String(), "")
}
Beispiel #28
0
func remove(c cli.Command) {
	var (
		arglen  = len(c.Args())
		object  string
		err     error
		message string
		objects []storage.ObjectAPI
	)
	if arglen == 2 {
		container = c.Arg(0).String()
		object = c.Arg(1).String()
	}
	if arglen == 1 {
		if c.Flag("type").String() == "container" {
			container = c.Arg(0).String()
		} else {
			object = c.Arg(0).String()
		}
	}
	if blank(container) {
		log.Fatal(errorNotEnough)
	}
	if blank(object) {
		containerApi := api.Container(container)
		err = containerApi.Remove()

		// forced removal of container
		if err == storage.ErrorConianerNotEmpty && c.Flag("force").Get().(bool) {
			fmt.Println("removing all objects of", container)
			objects, err = containerApi.Objects()
			if err != nil {
				log.Fatal(err)
			}
			for _, object := range objects {
				err = object.Remove()
				// skipping NotFound errors as non-critical
				if err != nil && err != storage.ErrorObjectNotFound {
					log.Fatal(err)
				}
			}
			err = containerApi.Remove()
		}
		message = fmt.Sprintf("container %s removed", container)
	} else {
		err = api.Container(container).Object(object).Remove()
		message = fmt.Sprintf("object %s removed in container %s", object, container)
	}
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(message)
}