func log_tailing(conn *scrapinghub.Connection, job_id string) {
var jobs scrapinghub.Jobs
jobinfo, err := jobs.JobInfo(conn, job_id)
if err != nil {
log.Fatalf("%s\n", err)
}
// Number of log lines in the job
offset := jobinfo.Logs
if offset > 0 {
offset -= 1 // start one line before
}
count := 10 // ask for this lines in every call
ls := scrapinghub.LinesStream{Conn: conn, Count: count, Offset: offset}
for {
retrieved := 0
ch_lines, ch_err := ls.LogLines(job_id)
for line := range ch_lines {
retrieved++
fmt.Fprintf(os.Stdout, "%s\n", line)
}
for err := range ch_err {
log.Fatalf("%s\n", err)
}
ls.Offset += retrieved
time.Sleep(time.Second)
}
}
func cmd_jobs(conn *scrapinghub.Connection, args []string, flags *PFlags) {
if len(args) < 1 {
log.Fatalf("Missing argument: <project_id>\n")
}
project_id := args[0]
filters := equality_list_to_map(args[1:])
count := flags.Count
offset := flags.Offset
if flags.AsJsonLines {
ls := scrapinghub.LinesStream{Conn: conn, Count: count, Offset: offset}
ch_jobs, errch := ls.JobsAsJsonLines(project_id, filters)
for line := range ch_jobs {
fmt.Println(line)
}
for err := range errch {
log.Fatalf("jobs error: %s\n", err)
}
} else {
var jobs scrapinghub.Jobs
jobs_list, err := jobs.List(conn, project_id, count, filters)
if err != nil {
log.Fatalf("jobs error: %s", err)
}
outfmt := "| %10s | %25s | %12s | %10s | %10s | %10s | %20s |\n"
print_out(flags, outfmt, "id", "spider", "state", "items", "errors", "log lines", "started_time")
print_out(flags, dashes(106))
for _, j := range jobs_list.Jobs {
print_out(flags, "| %10s | %25s | %12s | %10d | %10d | %10d | %20s |\n", j.Id, j.Spider, j.State,
j.ItemsScraped, j.ErrorsCount, j.Logs, j.StartedTime)
}
}
}
func cmd_jobs_stop(conn *scrapinghub.Connection, args []string, flags *PFlags) {
if len(args) < 1 {
log.Fatalf("Missing argument: <job_id>\n")
}
var jobs scrapinghub.Jobs
job_id := args[0]
err := jobs.Stop(conn, job_id)
if err != nil {
log.Fatalf("stop error: %s\n", err)
} else {
fmt.Printf("Stopped job: %s\n", job_id)
}
}
func cmd_reschedule(conn *scrapinghub.Connection, args []string, flags *PFlags) {
if len(args) < 1 {
log.Fatalf("Missing argument: <job_id>\n")
}
job_id := args[0]
var jobs scrapinghub.Jobs
new_job_id, err := jobs.Reschedule(conn, job_id)
if err != nil {
log.Fatalf("reschedule error: %s\n", err)
} else {
fmt.Printf("Re-scheduled job new id: %s\n", new_job_id)
}
}
func cmd_jobs_update(conn *scrapinghub.Connection, args []string, flags *PFlags) {
if len(args) < 1 {
log.Fatalf("Missing argument: <job_id>\n")
}
var jobs scrapinghub.Jobs
job_id := args[0]
update_data := equality_list_to_map(args[1:])
err := jobs.Update(conn, job_id, update_data)
if err != nil {
log.Fatalf("update error: %s\n", err)
} else {
fmt.Printf("Updated job: %s\n", job_id)
}
}
func cmd_schedule(conn *scrapinghub.Connection, args []string, flags *PFlags) {
if len(args) < 2 {
log.Fatalf("Missing arguments: <project_id> and <spider_name>\n")
}
var jobs scrapinghub.Jobs
project_id := args[0]
spider_name := args[1]
spider_args := equality_list_to_map(args[2:])
job_id, err := jobs.Schedule(conn, project_id, spider_name, spider_args)
if err != nil {
log.Fatalf("schedule error: %s\n", err)
} else {
fmt.Printf("Scheduled job: %s\n", job_id)
}
}
func cmd_jobinfo(conn *scrapinghub.Connection, args []string, flags *PFlags) {
if len(args) < 1 {
log.Fatalf("Missing argument: <job_id>\n")
}
job_id := args[0]
var jobs scrapinghub.Jobs
jobinfo, err := jobs.JobInfo(conn, job_id)
if err != nil {
log.Fatalf("jobinfo error: %s\n", err)
} else {
outfmt := "| %-30s | %60s |\n"
print_out(flags, outfmt, "key", "value")
print_out(flags, dashes(97))
print_out(flags, outfmt, "id", jobinfo.Id)
print_out(flags, outfmt, "spider", jobinfo.Spider)
print_out(flags, outfmt, "spider_args", "")
for k, v := range jobinfo.SpiderArgs {
print_out(flags, outfmt, " ", fmt.Sprintf("%s = %s", k, v))
}
print_out(flags, outfmt, "spider_type", jobinfo.SpiderType)
print_out(flags, outfmt, "state", jobinfo.State)
print_out(flags, outfmt, "close_reason", jobinfo.CloseReason)
print_out(flags, dashes(97))
print_out(flags, outfmt, "responses_received", fmt.Sprintf("%d", jobinfo.ResponsesReceived))
print_out(flags, outfmt, "items_scraped", fmt.Sprintf("%d", jobinfo.ItemsScraped))
print_out(flags, outfmt, "errors_count", fmt.Sprintf("%d", jobinfo.ErrorsCount))
print_out(flags, outfmt, "logs", fmt.Sprintf("%d", jobinfo.Logs))
print_out(flags, dashes(97))
print_out(flags, outfmt, "started_time", jobinfo.StartedTime)
print_out(flags, outfmt, "updated_time", jobinfo.UpdatedTime)
print_out(flags, outfmt, "elapsed", fmt.Sprintf("%d", jobinfo.Elapsed))
print_out(flags, outfmt, "tags", " ")
for _, e := range jobinfo.Tags {
print_out(flags, outfmt, " ", e)
}
print_out(flags, outfmt, "priority", fmt.Sprintf("%d", jobinfo.Priority))
print_out(flags, outfmt, "version", jobinfo.Version)
print_out(flags, dashes(97))
}
}