func download(c cli.Command) {
var (
arglen = len(c.Args())
objectName string
path = c.Flag("path").String()
)
switch arglen {
case 1:
objectName = c.Arg(0).String()
case 2:
objectName = c.Arg(1).String()
container = c.Arg(0).String()
}
if blank(container) || blank(objectName) {
log.Fatal(errorNotEnough)
}
if blank(path) {
path = objectName
}
reader, err := api.Container(container).Object(objectName).GetReader()
if err != nil {
log.Fatal(err)
}
defer reader.Close()
fmt.Printf("downloading %s->%s from %s\n", objectName, path, container)
f, err := os.Create(path)
if err != nil {
log.Fatal(err)
}
n, err := io.Copy(f, reader)
if err != nil {
log.Fatal(err)
}
fmt.Printf("downloaded %s, %d bytes\n", objectName, n)
}
func roleRm(c cli.Command) {
rmRoles := c.Args().Strings()
if len(rmRoles) == 0 {
log.Fatalln("Must specify some role[s] to remove")
}
nodeRoles, err := allNodeRoles()
if err != nil {
log.Fatalln("err: ", err)
}
myKey, err := selfKey()
if err != nil {
log.Fatalln("err: ", err)
}
var finalSet []string
for _, role := range nodeRoles[myKey] {
if !StrContains(rmRoles, role) {
finalSet = append(finalSet, role)
} else {
fmt.Printf("removing role `%s`\n", role)
}
}
roleActualSet(finalSet, c)
}
func (v *Run) Run(c cli.Command) {
jobName := c.Param("jobname").String()
log.WithFields(
log.Fields{
"pkg": "command",
"func": "Run",
"job": jobName,
},
).Info("Starting job")
j := job.New(metl.GetJobFilePath(jobName))
j.Lock()
jf, err := j.Fetch()
if err != nil {
log.Fatal("Error fetching job:", err)
}
jf.Run()
j.Unlock()
j.Done(jf)
// log stats, notify of errors etc
// notify of any errors
// log stats to dashboard, number of processed, rejected rows etc.
// if parsing option is empty then, we just download the file
}
func cmRoll(c cli.Command) {
role := c.Flag("role").String()
if len(role) == 0 && c.Flag("force").Get() != true {
log.Fatalln("Must specify -f option to run with no `role` filter specified")
} else {
cmRunRoll(role, "")
}
}
func nodeRun(c cli.Command) {
switch c.Param("action").String() {
case "list":
nodeList(c)
default:
cli.ShowUsage(c)
}
}
func (v *Add) Run(c cli.Command) {
jobName := c.Param("jobname").String()
var job job.Job
if _, err := toml.DecodeFile(metl.GetJobFilePath(jobName)+".toml", &job); err != nil {
log.Fatal(err)
}
cronfile := filepath.Join(metl.Etl.GetLocalStoragePath(), crontabFile)
file, err := os.Open(cronfile)
if err != nil {
file, err = os.Create(cronfile)
if err != nil {
log.WithFields(
log.Fields{
"filename": cronfile,
},
).Fatal("Unable to create cronfile: ", err)
}
}
defer file.Close()
err = syscall.Flock(int(file.Fd()), syscall.LOCK_EX)
if err != nil {
log.Fatal("Failed to lock file")
}
defer func() {
err = syscall.Flock(int(file.Fd()), syscall.LOCK_UN)
if err != nil {
log.Fatal("Failed to unlock file")
}
}()
buffer := make([]string, 0)
if err = jobIsInFile(file, jobName, &buffer); err != nil {
log.Fatal(err)
}
buffer = append(buffer, fmt.Sprintf("%s %s >> %s%s.log", cmd, jobName, logDir, jobName))
tmpf, err := ioutil.TempFile(metl.Etl.GetLocalStoragePath(), "tmp")
if err != nil {
log.Fatal("Unable to create temporary file.")
}
for _, line := range buffer {
if _, err := tmpf.WriteString(line + "\n"); err != nil {
log.Fatal("Unable to write to temporary file")
}
}
if err := os.Rename(tmpf.Name(), cronfile); err != nil {
log.Fatal("Failed to rename temporary lock file:", err)
}
// execute /usr/bin/env crontab cronfile
}
func (etl *metl) runner(c cli.Command) {
etl.preRunnerTasks()
runnable := c.Name()
if runner, ok := etl.runnables[runnable]; ok {
runner.Run(c)
} else {
panic(fmt.Sprintf("Runnable %s not defined!", runnable))
}
}
func create(c cli.Command) {
if len(c.Args()) == 0 {
log.Fatal(errorNotEnough)
}
var name = c.Arg(0).String()
if _, err := api.CreateContainer(name, false); err != nil {
log.Fatal(err)
}
fmt.Printf("created container %s\n", name)
}
func roleRun(c cli.Command) {
switch c.Param("action").String() {
case "list":
roleList(c)
case "set":
roleSet(c)
default:
cli.ShowUsage(c)
}
}
func cmRun(c cli.Command) {
switch c.Param("action").String() {
case "local":
cmLocal(c)
case "roll":
cmRoll(c)
case "single":
cmSingle(c)
default:
cli.ShowUsage(c)
}
}
func serviceRun(c cli.Command) {
switch c.Param("action").String() {
case "list":
serviceList(c)
case "local":
serviceLocal(c)
case "find":
serviceFind(c)
default:
cli.ShowUsage(c)
}
}
func greetGreetee(c cli.Command) {
greeting := c.Parent().Param("greeting")
greetee := c.Param("greetee")
str := fmt.Sprintf("%s %s", greeting, greetee)
str = styleByFlags(str, c)
c.Println(str, strings.Join(c.Args().Strings(), " "))
}
func upload(c cli.Command) {
var path string
switch len(c.Args()) {
case 1:
path = c.Arg(0).String()
case 2:
container = c.Arg(0).String()
path = c.Arg(1).String()
}
if blank(container) || blank(path) {
log.Fatal(errorNotEnough)
}
f, err := os.Open(path)
if err != nil {
log.Fatal(err)
}
stat, err := os.Stat(path)
if err != nil {
log.Fatal(err)
}
ext := filepath.Ext(path)
mimetype := mime.TypeByExtension(ext)
bar := pb.New64(stat.Size()).SetUnits(pb.U_BYTES)
bar.Start()
reader := io.TeeReader(f, bar)
if err := api.Container(container).Upload(reader, stat.Name(), mimetype); err != nil {
log.Fatal(err)
}
fmt.Printf("uploaded to %s\n", container)
}
func roleSet(c cli.Command) {
client, _ := api.NewClient(api.DefaultConfig())
agent := client.Agent()
reg := &api.AgentServiceRegistration{
Name: "cascade",
Tags: c.Args().Strings(),
}
if err := agent.ServiceRegister(reg); err != nil {
log.Fatalln("err: ", err)
}
roleList(c)
}
func nodeList(c cli.Command) {
client, _ := api.NewClient(api.DefaultConfig())
catalog := client.Catalog()
nodes, _, err := catalog.Service("cascade", c.Flag("role").String(), nil)
if err != nil {
log.Fatalln("Err: ", err)
}
for _, node := range nodes {
fmt.Println(node.Node, node.Address+":")
for _, role := range node.ServiceTags {
fmt.Println(" -", role)
}
}
}
func roleRun(c cli.Command) {
switch c.Param("action").String() {
case "list":
roleList(c)
case "set":
roleSet(c)
case "listAll":
roleListAll(c)
case "find":
roleFind(c)
case "append":
roleAppend(c)
case "rm":
roleRm(c)
default:
cli.ShowUsage(c)
}
}
func cmSingle(c cli.Command) {
client, _ := api.NewClient(api.DefaultConfig())
catalog := client.Catalog()
node, _, err := catalog.Node(c.Arg(0).String(), nil)
if err != nil {
log.Fatalln("err: ", err)
}
if node == nil {
log.Fatalln("node not found")
}
if node.Services["cascade"] == nil {
log.Fatalln("node not managed by cascade")
}
cmRunRoll("", c.Arg(0).String())
}
func list(c cli.Command) {
var (
arglen = len(c.Args())
table = tablewriter.NewWriter(os.Stdout)
)
if arglen == 0 && (blank(container) || c.Flag("type").String() == "storage") {
containers, err := api.ContainersInfo()
if err != nil {
log.Fatal(err)
}
table.SetHeader([]string{"Name", "Objects", "Type"})
for _, cont := range containers {
v := []string{cont.Name, fmt.Sprint(cont.ObjectCount), cont.Type}
table.Append(v)
}
table.Render()
return
}
if arglen == 1 {
container = c.Arg(0).String()
}
if blank(container) {
log.Fatal(errorNotEnough)
}
objects, err := api.Container(container).ObjectsInfo()
if err != nil {
log.Fatal(err)
}
table.SetHeader([]string{"Name", "Size", "Downloaded"})
for _, object := range objects {
v := []string{object.Name, fmt.Sprint(object.Size), fmt.Sprint(object.Downloaded)}
table.Append(v)
}
table.Render()
}
func roleFind(c cli.Command) {
cmdRoles := c.Args().Strings()
if len(cmdRoles) == 0 {
log.Fatalln("Must specify a role to find")
}
if len(cmdRoles) != 1 {
// maybe we could support multiple but i don't think it's necessary
log.Fatalln("One role only is supported for `find` command")
}
role := cmdRoles[0]
allRoles, err := allNodeRoles()
if err != nil {
log.Fatalln("err: ", err)
}
fmt.Printf("All nodes containing role %s:\n\n", role)
for node, r := range allRoles {
if StrContains(r, role) {
printRole(node, r)
}
}
}
func styleByFlags(str string, c cli.Command) string {
if c.Flag("loudly").Get() == true {
str = louden(str)
}
if c.Flag("color").String() != "" {
str = colorize(str, c.Flag("color").String())
}
return str
}
func roleAppend(c cli.Command) {
nodeRoles, err := allNodeRoles()
if err != nil {
log.Fatalln("err: ", err)
}
myKey, err := selfKey()
if err != nil {
log.Fatalln("err: ", err)
}
var finalSet []string
for _, role := range nodeRoles[myKey] {
finalSet = append(finalSet, role)
}
for _, role := range c.Args().Strings() {
if !StrContains(finalSet, role) {
finalSet = append(finalSet, role)
}
}
roleActualSet(finalSet, c)
}
func serviceFind(c cli.Command) {
client, _ := api.NewClient(api.DefaultConfig())
catalog := client.Catalog()
if len(c.Args().GetAll()) == 0 {
log.Fatalln("err: missing <servicename> argument")
}
nodes, _, err := catalog.Service(c.Arg(0).String(), c.Flag("type").String(), nil)
if err != nil {
log.Fatalln("err: ", err)
}
fmt.Println(c.Arg(0).String() + ":")
for _, node := range nodes {
fmt.Println(" - host:", node.Node)
fmt.Println(" address:", node.Address)
fmt.Println(" port:", node.ServicePort)
fmt.Println(" tags:", strings.Join(node.ServiceTags, ", "))
}
}
// the greet command run by the root command
func greet(c cli.Command) {
greeting := c.Param("greeting")
str := fmt.Sprintf("%s", greeting)
str = styleByFlags(str, c)
c.Println(str)
}
func readFlag(c cli.Command, name, env string) string {
if len(os.Getenv(env)) > 0 {
return os.Getenv(env)
}
return c.Flag(name).String()
}
// info prints information about storage
func info(c cli.Command) {
var (
containerName = container
objectName string
data interface{}
err error
arglen = len(c.Args())
command = c.Flag("type").String()
)
defer func() {
if err != nil {
log.Fatal(err)
}
if blank(containerName) || command == "storage" {
data = api.Info()
} else {
containerApi := api.Container(containerName)
if blank(objectName) {
data, err = containerApi.Info()
} else {
data, err = containerApi.Object(objectName).Info()
}
}
if err != nil {
log.Fatal(err)
}
fmt.Printf("%+v\n", data)
}()
if arglen > 0 {
if command == "container" {
containerName = c.Arg(0).String()
return
}
command = "object"
if !blank(containerName) && arglen == 1 {
objectName = c.Arg(0).String()
return
}
if arglen == 2 {
containerName = c.Arg(0).String()
objectName = c.Arg(1).String()
return
}
}
if command == "container" && !blank(containerName) {
return
}
if command == "storage" {
return
}
err = errorNotEnough
}
func roleSet(c cli.Command) {
roleActualSet(c.Args().Strings(), c)
}
func cmRoll(c cli.Command) {
cmRunRoll(c.Flag("role").String(), "")
}
func remove(c cli.Command) {
var (
arglen = len(c.Args())
object string
err error
message string
objects []storage.ObjectAPI
)
if arglen == 2 {
container = c.Arg(0).String()
object = c.Arg(1).String()
}
if arglen == 1 {
if c.Flag("type").String() == "container" {
container = c.Arg(0).String()
} else {
object = c.Arg(0).String()
}
}
if blank(container) {
log.Fatal(errorNotEnough)
}
if blank(object) {
containerApi := api.Container(container)
err = containerApi.Remove()
// forced removal of container
if err == storage.ErrorConianerNotEmpty && c.Flag("force").Get().(bool) {
fmt.Println("removing all objects of", container)
objects, err = containerApi.Objects()
if err != nil {
log.Fatal(err)
}
for _, object := range objects {
err = object.Remove()
// skipping NotFound errors as non-critical
if err != nil && err != storage.ErrorObjectNotFound {
log.Fatal(err)
}
}
err = containerApi.Remove()
}
message = fmt.Sprintf("container %s removed", container)
} else {
err = api.Container(container).Object(object).Remove()
message = fmt.Sprintf("object %s removed in container %s", object, container)
}
if err != nil {
log.Fatal(err)
}
fmt.Println(message)
}