Exemplo n.º 1
0
func log_tailing(conn *scrapinghub.Connection, job_id string) {
	var jobs scrapinghub.Jobs
	jobinfo, err := jobs.JobInfo(conn, job_id)
	if err != nil {
		log.Fatalf("%s\n", err)
	}
	// Number of log lines in the job
	offset := jobinfo.Logs
	if offset > 0 {
		offset -= 1 // start one line before
	}
	count := 10 // ask for this lines in every call
	ls := scrapinghub.LinesStream{Conn: conn, Count: count, Offset: offset}
	for {
		retrieved := 0
		ch_lines, ch_err := ls.LogLines(job_id)
		for line := range ch_lines {
			retrieved++
			fmt.Fprintf(os.Stdout, "%s\n", line)
		}
		for err := range ch_err {
			log.Fatalf("%s\n", err)
		}
		ls.Offset += retrieved
		time.Sleep(time.Second)
	}
}
Exemplo n.º 2
0
func cmd_jobinfo(conn *scrapinghub.Connection, args []string, flags *PFlags) {
	if len(args) < 1 {
		log.Fatalf("Missing argument: <job_id>\n")
	}
	job_id := args[0]

	var jobs scrapinghub.Jobs
	jobinfo, err := jobs.JobInfo(conn, job_id)

	if err != nil {
		log.Fatalf("jobinfo error: %s\n", err)
	} else {
		outfmt := "| %-30s | %60s |\n"
		print_out(flags, outfmt, "key", "value")
		print_out(flags, dashes(97))
		print_out(flags, outfmt, "id", jobinfo.Id)
		print_out(flags, outfmt, "spider", jobinfo.Spider)
		print_out(flags, outfmt, "spider_args", "")
		for k, v := range jobinfo.SpiderArgs {
			print_out(flags, outfmt, " ", fmt.Sprintf("%s = %s", k, v))
		}
		print_out(flags, outfmt, "spider_type", jobinfo.SpiderType)
		print_out(flags, outfmt, "state", jobinfo.State)
		print_out(flags, outfmt, "close_reason", jobinfo.CloseReason)
		print_out(flags, dashes(97))
		print_out(flags, outfmt, "responses_received", fmt.Sprintf("%d", jobinfo.ResponsesReceived))
		print_out(flags, outfmt, "items_scraped", fmt.Sprintf("%d", jobinfo.ItemsScraped))
		print_out(flags, outfmt, "errors_count", fmt.Sprintf("%d", jobinfo.ErrorsCount))
		print_out(flags, outfmt, "logs", fmt.Sprintf("%d", jobinfo.Logs))
		print_out(flags, dashes(97))
		print_out(flags, outfmt, "started_time", jobinfo.StartedTime)
		print_out(flags, outfmt, "updated_time", jobinfo.UpdatedTime)
		print_out(flags, outfmt, "elapsed", fmt.Sprintf("%d", jobinfo.Elapsed))
		print_out(flags, outfmt, "tags", " ")
		for _, e := range jobinfo.Tags {
			print_out(flags, outfmt, " ", e)
		}
		print_out(flags, outfmt, "priority", fmt.Sprintf("%d", jobinfo.Priority))
		print_out(flags, outfmt, "version", jobinfo.Version)
		print_out(flags, dashes(97))
	}
}