func parseGauges(info string) map[string]int64 { gauges_with_values := map[string]int64{ "blocked_clients": 0, "connected_clients": 0, "instantaneous_ops_per_sec": 0, "latest_fork_usec": 0, "mem_fragmentation_ratio": 0, "migrate_cached_sockets": 0, "pubsub_channels": 0, "pubsub_patterns": 0, "uptime_in_seconds": 0, "used_memory": 0, "used_memory_lua": 0, "used_memory_peak": 0, "used_memory_rss": 0, } color.White("-------------------") color.White("GAUGES:") for gauge, _ := range gauges_with_values { r, _ := regexp.Compile(fmt.Sprint(gauge, ":([0-9]*)")) matches := r.FindStringSubmatch(info) if matches == nil { color.Yellow(fmt.Sprint("WARN: ", gauge, "is not displayed in redis info")) } else { value := matches[len(matches)-1] color.Cyan(fmt.Sprint(gauge, ": ", value)) v, _ := strconv.ParseInt(value, 10, 64) gauges_with_values[gauge] = v } } return gauges_with_values }
func parseCounters(info string) map[string]int64 { counters := map[string]int64{ "evicted_keys": 0, "expired_keys": 0, "keyspace_hits": 0, "keyspace_misses": 0, "rejected_connections": 0, "sync_full": 0, "sync_partial_err": 0, "sync_partial_ok": 0, "total_commands_processed": 0, "total_connections_received": 0, } color.White("-------------------") color.White("COUNTERS:") for counter, _ := range counters { r, _ := regexp.Compile(fmt.Sprint(counter, ":([0-9]*)")) matches := r.FindStringSubmatch(info) if matches == nil { color.Yellow(fmt.Sprint("ERROR: ", counter, "is not displayed in redis info")) } else { value := matches[len(matches)-1] color.Cyan(fmt.Sprint(counter, ": ", value)) v, _ := strconv.ParseInt(value, 10, 64) counters[counter] = v } } return counters }
func endNotice() { fmt.Print("\n\n") color.Green("******") color.Green("Download complete!\n") color.White("Your files have been saved to " + setDownloadFolder() + "\n\n") color.White("For your convenience, your files have been sorted by extension.") color.Green("******") }
func startNotice() { color.Green("******") color.Green("~ Notice: ~\n") color.White("When your download is complete,\nyou will find your files under:\n") color.Magenta(setDownloadFolder()) color.Green("******" + "\n\n") }
// Info is a convenience method appending a info style message to the logger func Info(obj interface{}) { // Get the line number and calling func sig _, fn, line, _ := runtime.Caller(1) msg := fmt.Sprintf("%+v\n%s:%d\n\n", obj, fn, line) formattedMessage := formattedLogMessage("INFO", msg) color.White(formattedMessage) }
// // required for serach options // func getSearchOpt() (SearchType, FilterType) { var ( search SearchType = SearchBySubject filter FilterType = SearchFilterAll ) for { color.White("Please select search range:\n") for k := SearchFilterAll; k <= SearchConference; k++ { fmt.Fprintf(color.Output, "\t %s: %s\n", color.CyanString("%d", k), searchFilterHints[k]) } fmt.Fprintf(color.Output, "$ %s", color.CyanString("select: ")) s := getInputString() if len(s) > 0 { selected, err := strconv.ParseInt(s, 16, 32) if err != nil || selected < int64(SearchFilterAll) || selected > int64(SearchConference) { color.Red("Invalid selection\n") continue } filter = FilterType(selected) } break } return search, filter }
// Infof is a convenience method appending a info style message to the logger func Infof(msg string, a ...interface{}) { _, fn, line, _ := runtime.Caller(1) msg = fmt.Sprintf(msg, a...) msg = fmt.Sprintf("%+v%s:%d\n\n", msg, fn, line) formattedMessage := formattedLogMessage("INFO", msg) color.White(formattedMessage) }
func writeInfo(format string, a ...interface{}) { if silent { return } color.White(format, a...) }
func monitor(hostname string, interval int, changes chan<- string) { var knownAddresses []string for tick := range time.NewTicker(time.Duration(interval) * time.Second).C { addresses := lookup(hostname) color.White("[LOOKUP] %s %s %s", hostname, addresses, tick) if !equivalent(knownAddresses, addresses) { changes <- hostname knownAddresses = addresses } } }
// conect to server operation func connect(matched []Server, printOnly bool) { if len(matched) == 0 { color.Cyan("No server match patterns") } else if len(matched) == 1 { color.Green("%s", matched[0].getConnectionString()) if !printOnly { matched[0].connect() } } else { color.Cyan("Multiple servers match patterns:") for _, s := range matched { color.White(s.getConnectionString()) } } }
// download file from server operation func download(src string, dest string, matched []Server, printOnly bool) { if len(matched) == 0 { color.Cyan("No server match patterns") } else if len(matched) == 1 { color.Green("%s", matched[0].getDownloadString(src, dest)) if !printOnly { matched[0].download(src, dest) } } else { color.Cyan("Multiple servers match patterns:") for _, s := range matched { color.White(s.getDownloadString(src, dest)) } } }
func workspaces_list(c *cli.Context) error { max := c.Int("max") url := c.GlobalString("url") if url == "" { return cli.NewExitError("A URL must be provided with the SCALE_URL environment variable or the --url argument", 1) } workspaces, err := scalecli.GetWorkspaceList(url, max) if err != nil { return cli.NewExitError(err.Error(), 1) } for _, workspace := range workspaces { if workspace.Is_active { color.Green(workspace.String()) } else { color.White(workspace.String()) } } return nil }
func main() { // init app := cli.NewApp() app.Name = "stadis" app.Version = "0.0.2" app.Usage = "get redis info and submit to statsd" app.Flags = []cli.Flag{ cli.StringFlag{ Name: "redis-host, r", Value: "localhost:6379", Usage: "host:port of redis servier", }, cli.StringFlag{ Name: "statsd-host, s", Value: "localhost:8125", Usage: "host:port of statsd servier", }, cli.StringFlag{ Name: "prefix,p", Usage: "host:port of redis servier", EnvVar: "HOSTNAME", }, cli.StringFlag{ Name: "interval,i", Usage: "time in milliseconds to periodically check redis", Value: "5000", }, } app.Action = func(c *cli.Context) { for { info := getStats(c.String("redis-host")) gauges := parseGauges(info) counters := parseCounters(info) sendStats(c.String("statsd-host"), c.String("prefix"), gauges, counters) color.White("-------------------") interval, _ := strconv.ParseInt(c.String("interval"), 10, 64) time.Sleep(time.Duration(interval) * time.Millisecond) } } app.Run(os.Args) }
func jobs_validate(c *cli.Context) error { var job_type scalecli.JobType err := Parse_json_or_yaml("job_type", &job_type) if err != nil { return cli.NewExitError(err.Error(), 1) } url := c.GlobalString("url") if url == "" { return cli.NewExitError("A URL must be provided with the SCALE_URL environment variable or the --url argument", 1) } warnings, err := scalecli.ValidateJobType(url, job_type) if err != nil { return cli.NewExitError(err.Error(), 1) } if warnings == "" { color.White("Job type specification is valid.") } else { color.Yellow(warnings) } return nil }
func main() { flag.Parse() if *sitemapURL == "" { fmt.Println("-sitemap に sitemap.xml/.xml.gz のURLを指定してください") return } smap, err := sitemap.Get(*sitemapURL) if err != nil { fmt.Println(err) } for _, URL := range smap.URL { time.Sleep(time.Second) resp, err := http.Get(URL.Loc) if err != nil { fmt.Println(err) continue } defer resp.Body.Close() switch statusType(resp.StatusCode) { case 100: color.Cyan(resp.Status + " " + URL.Loc) case 200: color.Green(resp.Status + " " + URL.Loc) case 300: color.Magenta(resp.Status + " " + URL.Loc) case 400: color.Red(resp.Status + " " + URL.Loc) case 500: color.Yellow(resp.Status + " " + URL.Loc) default: color.White(resp.Status + " " + URL.Loc) } } }
RootCmd.AddCommand(versionCmd) } // Get svfs version information. var versionCmd = &cobra.Command{ Use: "version", Short: "Prints the svfs version and any available update", Long: "Display information about current SVFS version and\n" + "check if a new version is available on GitHub.", RunE: func(cmd *cobra.Command, args []string) error { cmd.SilenceUsage = true cmd.SilenceErrors = false client := github.NewClient(nil) // Print current version information color.White("Version information:\n\n") current, _, err := client.Repositories.GetReleaseByTag(gitHubOwner, gitHubRepo, currentVersion) printRelease(current, err) if err != nil { return err } // Get the latest release releases, _, err := client.Repositories.ListReleases( gitHubOwner, gitHubRepo, &github.ListOptions{PerPage: 1}) latest := releases[len(releases)-1] // Print if an update is available if *current.TagName == *latest.TagName {
func CommandGrade(cmd *cobra.Command, args []string) { mustLoadConfig(cmd) now := time.Now() // find the directory dir := "" switch len(args) { case 0: dir = "." case 1: dir = args[0] default: cmd.Help() return } problem, _, commit, dotfile := gather(now, dir) commit.Action = "grade" commit.Note = "grading from grind tool" unsigned := &CommitBundle{Commit: commit} // send the commit bundle to the server signed := new(CommitBundle) mustPostObject("/commit_bundles/unsigned", nil, unsigned, signed) // TODO: get a daycare referral // get the user ID user := new(User) mustGetObject("/users/me", nil, user) // send it to the daycare for grading log.Printf("submitting %s step %d for grading", problem.Unique, commit.Step) graded := mustConfirmCommitBundle(user.ID, signed, nil) // save the commit with report card toSave := &CommitBundle{ Commit: graded.Commit, CommitSignature: graded.CommitSignature, } saved := new(CommitBundle) mustPostObject("/commit_bundles/signed", nil, toSave, saved) commit = saved.Commit if commit.ReportCard != nil && commit.ReportCard.Passed && commit.Score == 1.0 { if nextStep(dir, dotfile.Problems[problem.Unique], problem, commit) { // save the updated dotfile with whitelist updates and new step number contents, err := json.MarshalIndent(dotfile, "", " ") if err != nil { log.Fatalf("JSON error encoding %s: %v", dotfile.Path, err) } contents = append(contents, '\n') if err := ioutil.WriteFile(dotfile.Path, contents, 0644); err != nil { log.Fatalf("error saving file %s: %v", dotfile.Path, err) } } } else { // solution failed log.Printf(" solution for step %d failed", commit.Step) if commit.ReportCard != nil { log.Printf(" ReportCard: %s", commit.ReportCard.Note) } // play the transcript for _, event := range commit.Transcript { switch event.Event { case "exec": color.Cyan("$ %s\n", strings.Join(event.ExecCommand, " ")) case "stdin": color.Yellow("%s", event.StreamData) case "stdout": color.White("%s", event.StreamData) case "stderr": color.Red("%s", event.StreamData) case "exit": color.Cyan("%s\n", event.ExitStatus) case "error": color.Red("Error: %s\n", event.Error) } } } }
func main() { app := cli.NewApp() app.Name = "stxy" app.Version = "0.0.4" app.Usage = "haproxy stats to statsd" app.Flags = []cli.Flag{ cli.StringFlag{ Name: "haproxy-url", Value: "localhost:22002/;csv", Usage: "host:port of haproxy server", }, cli.StringFlag{ Name: "statsd-url, s", Value: "localhost:8125", Usage: "host:port of statsd server", }, cli.StringFlag{ Name: "prefix,p", Usage: "statsd namespace", Value: "haproxy", }, cli.StringFlag{ Name: "interval,i", Usage: "time in milliseconds", Value: "10000", }, } app.Action = func(c *cli.Context) { interval, _ := strconv.ParseInt(c.String("interval"), 10, 64) for { client, err := statsd.NewClient(c.String("s"), c.String("p")) // handle any errors if err != nil { log.Fatal(err) } // make sure to clean up defer client.Close() initial_stats := get_stats(c.String("haproxy-url")) previous := map[string]int64{} for _, v := range initial_stats { if v[1] == "BACKEND" { previous[fmt.Sprint("1xx_", v[0])] = get_value(v, "hrsp_1xx", 39) previous[fmt.Sprint("2xx_", v[0])] = get_value(v, "hrsp_2xx", 40) previous[fmt.Sprint("3xx_", v[0])] = get_value(v, "hrsp_3xx", 41) previous[fmt.Sprint("4xx_", v[0])] = get_value(v, "hrsp_4xx", 42) previous[fmt.Sprint("5xx_", v[0])] = get_value(v, "hrsp_5xx", 43) } } time.Sleep(time.Duration(interval) * time.Millisecond) records := get_stats(c.String("haproxy-url")) for _, record := range records { if record[1] == "BACKEND" { go send_gauge(client, record, "scur", 4) go send_gauge(client, record, "smax", 5) go send_gauge(client, record, "ereq", 12) go send_gauge(client, record, "econ", 13) go send_gauge(client, record, "rate", 33) go send_gauge(client, record, "bin", 8) go send_gauge(client, record, "bout", 9) go send_counter(previous[fmt.Sprint("1xx_", record[0])], client, record, "hrsp_1xx", 39) go send_counter(previous[fmt.Sprint("2xx_", record[0])], client, record, "hrsp_2xx", 40) go send_counter(previous[fmt.Sprint("3xx_", record[0])], client, record, "hrsp_3xx", 41) go send_counter(previous[fmt.Sprint("4xx_", record[0])], client, record, "hrsp_4xx", 42) go send_counter(previous[fmt.Sprint("5xx_", record[0])], client, record, "hrsp_5xx", 43) go send_gauge(client, record, "qtime", 58) go send_gauge(client, record, "ctime", 59) go send_gauge(client, record, "rtime", 60) go send_gauge(client, record, "ttime", 61) } } color.White("-------------------") } } app.Run(os.Args) }
func main() { app := cli.NewApp() app.Name = "flint" app.Usage = "Check a project for common sources of contributor friction" app.Version = "0.0.3" app.Flags = []cli.Flag{ cli.BoolFlag{"skip-readme", "skip check for README", ""}, cli.BoolFlag{"skip-contributing", "skip check for contributing guide", ""}, cli.BoolFlag{"skip-license", "skip check for license", ""}, cli.BoolFlag{"skip-bootstrap", "skip check for bootstrap script", ""}, cli.BoolFlag{"skip-test", "skip check for test script", ""}, cli.BoolFlag{"skip-scripts", "skip check for all scripts", ""}, cli.BoolFlag{"no-color", "skip coloring the terminal output", ""}, } app.Action = func(c *cli.Context) { path, _ := os.Getwd() if len(c.Args()) > 0 { path = c.Args()[0] } linter := &flint.Lint{Path: path} if !c.Bool("skip-readme") { linter.CheckReadme() } if !c.Bool("skip-contributing") { linter.CheckContributing() } if !c.Bool("skip-license") { linter.CheckLicense() } if !c.Bool("skip-scripts") { if !c.Bool("skip-bootstrap") { linter.CheckBootstrap() } if !c.Bool("skip-test") { linter.CheckTest() } } if len(linter.Errors) > 0 { for _, element := range linter.Errors { if !c.Bool("no-color") { // if not skipping output color if element.Level == 0 { // if [FIXME] color.White(element.Message) } else { // if [ERROR] color.Yellow(element.Message) } } else { fmt.Println(element.Message) } } level := linter.Severity() if level > 0 { if !c.Bool("no-color") { color.Red("[CRITICAL] Some critical problems found. Please fix right away!") } else { fmt.Println("[CRITICAL] Some critical problems found. Please fix right away!") } } os.Exit(level) } else { if !c.Bool("no-color") { color.Green("[OK] All is well!") } else { fmt.Println("[OK] All is well!") } } } if err := app.Run(os.Args); err != nil { fmt.Println(err) os.Exit(1) } }
func main() { username := "" password := "" language := "English" quality := "1080p" trim := "" daisukiIntroTrim := false aniplexIntroTrim := false sunriseIntroTrim := false app := cli.NewApp() app.Name = "anirip" app.Author = "Steven Wolfe" app.Email = "*****@*****.**" app.Version = "v1.4.0(7/7/2016)" app.Usage = "Crunchyroll/Daisuki show ripper CLI" color.Cyan(app.Name + " " + app.Version + " - by " + app.Author + " <" + app.Email + ">\n") app.Flags = []cli.Flag{ cli.StringFlag{ Name: "lang, l", Value: "english", Usage: "desired subtitle language", Destination: &language, }, cli.StringFlag{ Name: "quality, q", Value: "1080p", Usage: "desired video quality", Destination: &quality, }, cli.StringFlag{ Name: "trim, t", Value: "", Usage: "desired intros to be trimmed off of final video", Destination: &trim, }, } app.Commands = []cli.Command{ { Name: "login", Aliases: []string{"l"}, Usage: "creates and stores cookies for a stream provider", Flags: []cli.Flag{ cli.StringFlag{ Name: "user, u", Value: "myusername", Usage: "premium username used to access video stream", Destination: &username, }, cli.StringFlag{ Name: "pass, p", Value: "mypassword", Usage: "premium password used to access video stream", Destination: &password, }, }, Action: func(c *cli.Context) error { // Gets the provider name from the cli argument provider := "" if c.NArg() > 0 { provider = c.Args()[0] } else { color.Red("[anirip] No provider given...") return anirip.Error{Message: "No provider given"} } // Creates session with cookies to store in file var session anirip.Session if strings.Contains(provider, "crunchyroll") { color.Cyan("[anirip] Logging to CrunchyRoll as " + username + "...") session = new(crunchyroll.CrunchyrollSession) } else if strings.Contains(provider, "daisuki") { color.Cyan("[anirip] Logging to Daisuki as " + username + "...") session = new(daisuki.DaisukiSession) } else { color.Red("[anirip] The given provider is not supported.") return anirip.Error{Message: "The given provider is not supported"} } // Performs the login procedure, storing the login information to file if err := session.Login(username, password, tempDir); err != nil { color.Red("[anirip] " + err.Error()) return anirip.Error{Message: "Unable to login to provider", Err: err} } color.Green("[anirip] Successfully logged in... Cookies saved to " + tempDir) return nil }, }, { Name: "clear", Aliases: []string{"c"}, Usage: "erases the temporary directory used for cookies and temp files", Action: func(c *cli.Context) error { // Attempts to erase the temporary directory if err := os.RemoveAll(tempDir); err != nil { color.Red("[anirip] There was an error erasing the temporary directory : " + err.Error()) return anirip.Error{Message: "There was an error erasing the temporary directory", Err: err} } color.Green("[anirip] Successfully erased the temporary directory " + tempDir) return nil }, }, } app.Action = func(c *cli.Context) error { if c.NArg() == 0 { color.Red("[anirip] No show URLs provided.") return anirip.Error{Message: "No show URLs provided"} } for _, showURL := range c.Args() { // Parses the URL so we can accurately judge the provider based on the host url, err := url.Parse(showURL) if err != nil { color.Red("[anirip] There was an error parsing the URL you entered.\n") return anirip.Error{Message: "There was an error parsing the URL you entered"} } // Creates the authentication & show objects for the provider we're ripping from var session anirip.Session var show anirip.Show if strings.Contains(strings.ToLower(url.Host), "crunchyroll") { show = new(crunchyroll.CrunchyrollShow) session = new(crunchyroll.CrunchyrollSession) } else if strings.Contains(strings.ToLower(url.Host), "daisuki") { show = new(daisuki.DaisukiShow) session = new(daisuki.DaisukiSession) } else { color.Red("[anirip] The URL provided is not supported.") return anirip.Error{Message: "The URL provided is not supported"} } // Performs the generic login procedure if err = session.Login(username, password, tempDir); err != nil { color.Red("[anirip] " + err.Error()) return anirip.Error{Message: "Unable to login to provider", Err: err} } // Attempts to scrape the shows metadata/information color.White("[anirip] Getting a list of episodes for the show...") if err = show.ScrapeEpisodes(showURL, session.GetCookies()); err != nil { color.Red("[anirip] " + err.Error()) return anirip.Error{Message: "Unable to get episodes", Err: err} } // Sets the boolean values for what intros we would like to trim if strings.Contains(strings.ToLower(trim), "daisuki") { daisukiIntroTrim = true } if strings.Contains(strings.ToLower(trim), "aniplex") { daisukiIntroTrim = true } if strings.Contains(strings.ToLower(trim), "sunrise") { daisukiIntroTrim = true } seasonMap := map[int]string{ 0: "Specials", 1: "Season One", 2: "Season Two", 3: "Season Three", 4: "Season Four", 5: "Season Five", 6: "Season Six", 7: "Season Seven", 8: "Season Eight", 9: "Season Nine", 10: "Season Ten", } os.Mkdir(show.GetTitle(), 0777) for _, season := range show.GetSeasons() { os.Mkdir(show.GetTitle()+string(os.PathSeparator)+seasonMap[season.GetNumber()], 0777) for _, episode := range season.GetEpisodes() { color.White("[anirip] Getting Episode Info...\n") if err = episode.GetEpisodeInfo(quality, session.GetCookies()); err != nil { color.Red("[anirip] " + err.Error()) continue } // Checks to see if the episode already exists, in which case we continue to the next _, err = os.Stat(show.GetTitle() + string(os.PathSeparator) + seasonMap[season.GetNumber()] + string(os.PathSeparator) + episode.GetFileName() + ".mkv") if err == nil { color.Green("[anirip] " + episode.GetFileName() + ".mkv has already been downloaded successfully..." + "\n") continue } subOffset := 0 color.Cyan("[anirip] Downloading " + episode.GetFileName() + "\n") // Downloads full MKV video from stream provider color.White("[anirip] Downloading video...\n") if err := episode.DownloadEpisode(quality, tempDir, session.GetCookies()); err != nil { color.Red("[anirip] " + err.Error() + "\n") continue } // Trims down the downloaded MKV if the user wants to trim a Daisuki intro if daisukiIntroTrim { subOffset = subOffset + daisukiIntroLength color.White("[anirip] Trimming off Daisuki Intro - " + strconv.Itoa(daisukiIntroLength) + "ms\n") if err := trimMKV(daisukiIntroLength, tempDir); err != nil { color.Red("[anirip] " + err.Error() + "\n") continue } } // Trims down the downloaded MKV if the user wants to trim an Aniplex intro if aniplexIntroTrim { subOffset = subOffset + aniplexIntroLength color.White("[anirip] Trimming off Aniplex Intro - " + strconv.Itoa(aniplexIntroLength) + "ms\n") if err := trimMKV(aniplexIntroLength, tempDir); err != nil { color.Red("[anirip] " + err.Error() + "\n") continue } } // Trims down the downloaded MKV if the user wants to trim a Sunrise intro if sunriseIntroTrim { subOffset = subOffset + sunriseIntroLength color.White("[anirip] Trimming off Sunrise Intro - " + strconv.Itoa(sunriseIntroLength) + "ms\n") if err := trimMKV(sunriseIntroLength, tempDir); err != nil { color.Red("[anirip] " + err.Error() + "\n") continue } } // Downloads the subtitles to .ass format and // offsets their times by the passed provided interval color.White("[anirip] Downloading subtitles with a total offset of " + strconv.Itoa(subOffset) + "ms...\n") subtitleLang, err := episode.DownloadSubtitles(language, subOffset, tempDir, session.GetCookies()) if err != nil { color.Red("[anirip] " + err.Error() + "\n") continue } // Attempts to merge the downloaded subtitles into the video strea color.White("[anirip] Merging subtitles into mkv container...\n") if err := mergeSubtitles("jpn", subtitleLang, tempDir); err != nil { color.Red("[anirip] " + err.Error() + "\n") continue } // Cleans the MKVs metadata for better reading by clients color.White("[anirip] Cleaning MKV...\n") if err := cleanMKV(tempDir); err != nil { color.Red("[anirip] " + err.Error() + "\n") continue } // Moves the episode to the appropriate season sub-directory if err := anirip.Rename(tempDir+string(os.PathSeparator)+"episode.mkv", show.GetTitle()+string(os.PathSeparator)+seasonMap[season.GetNumber()]+string(os.PathSeparator)+episode.GetFileName()+".mkv", 10); err != nil { color.Red(err.Error() + "\n\n") } color.Green("[anirip] Downloading and merging completed successfully.\n") } } color.Cyan("[anirip] Completed processing episodes for " + show.GetTitle() + "\n") } return nil } app.Run(os.Args) }
func mustConfirmCommitBundle(userID int64, bundle *CommitBundle, args []string) *CommitBundle { verbose := false // create a websocket connection to the server headers := make(http.Header) url := "wss://" + Config.Host + "/v2/sockets/" + bundle.Problem.ProblemType + "/" + bundle.Commit.Action socket, resp, err := websocket.DefaultDialer.Dial(url, headers) if err != nil { log.Printf("error dialing %s: %v", url, err) if resp != nil && resp.Body != nil { io.Copy(os.Stderr, resp.Body) resp.Body.Close() } log.Fatalf("giving up") } defer socket.Close() // form the initial request req := &DaycareRequest{UserID: userID, CommitBundle: bundle} if err := socket.WriteJSON(req); err != nil { log.Fatalf("error writing request message: %v", err) } // start listening for events for { reply := new(DaycareResponse) if err := socket.ReadJSON(reply); err != nil { log.Fatalf("socket error reading event: %v", err) break } switch { case reply.Error != "": log.Printf("server returned an error:") log.Fatalf(" %s", reply.Error) case reply.CommitBundle != nil: return reply.CommitBundle case reply.Event != nil: if verbose { switch reply.Event.Event { case "exec": color.Cyan("$ %s\n", strings.Join(reply.Event.ExecCommand, " ")) case "stdin": color.Yellow("%s", reply.Event.StreamData) case "stdout": color.White("%s", reply.Event.StreamData) case "stderr": color.Red("%s", reply.Event.StreamData) case "exit": color.Cyan("exit: %s\n", reply.Event.ExitStatus) case "error": color.Red("Error: %s\n", reply.Event.Error) } } default: log.Fatalf("unexpected reply from server") } } log.Fatalf("no commit returned from server") return nil }
func CommandCreate(cmd *cobra.Command, args []string) { mustLoadConfig(cmd) now := time.Now() // find the directory d := "" switch len(args) { case 0: d = "." case 1: d = args[0] default: cmd.Help() return } dir, err := filepath.Abs(d) if err != nil { log.Fatalf("error finding directory %q: %v", d, err) } // find the problem.cfg file for { path := filepath.Join(dir, ProblemConfigName) if _, err := os.Stat(path); err != nil { if os.IsNotExist(err) { // try moving up a directory old := dir dir = filepath.Dir(dir) if dir == old { log.Fatalf("unable to find %s in %s or an ancestor directory", ProblemConfigName, d) } log.Printf("could not find %s in %s, trying %s", ProblemConfigName, old, dir) continue } log.Fatalf("error searching for %s in %s: %v", ProblemConfigName, dir, err) } break } // parse problem.cfg cfg := struct { Problem struct { Unique string Note string Type string Tag []string Option []string } Step map[string]*struct { Note string Weight float64 } }{} configPath := filepath.Join(dir, ProblemConfigName) fmt.Printf("reading %s\n", configPath) err = gcfg.ReadFileInto(&cfg, configPath) if err != nil { log.Fatalf("failed to parse %s: %v", configPath, err) } // create problem object problem := &Problem{ Unique: cfg.Problem.Unique, Note: cfg.Problem.Note, ProblemType: cfg.Problem.Type, Tags: cfg.Problem.Tag, Options: cfg.Problem.Option, CreatedAt: now, UpdatedAt: now, } // start forming the problem bundle unsigned := &ProblemBundle{ Problem: problem, } // check if this is an existing problem existing := []*Problem{} mustGetObject("/problems", map[string]string{"unique": problem.Unique}, &existing) switch len(existing) { case 0: // new problem if cmd.Flag("update").Value.String() == "true" { log.Fatalf("you specified --update, but no existing problem with unique ID %q was found", problem.Unique) } // make sure the problem set with this unique name is free as well existingSets := []*ProblemSet{} mustGetObject("/problem_sets", map[string]string{"unique": problem.Unique}, &existingSets) if len(existingSets) > 1 { log.Fatalf("error: server found multiple problem sets with matching unique ID %q", problem.Unique) } if len(existingSets) != 0 { log.Printf("problem set %d already exists with unique ID %q", existingSets[0].ID, existingSets[0].Unique) log.Fatalf(" this would prevent creating a problem set containing just this problem with matching id") } log.Printf("this problem is new--no existing problem has the same unique ID") case 1: // update to existing problem if cmd.Flag("update").Value.String() == "false" { log.Fatalf("you did not specify --update, but a problem already exists with unique ID %q", problem.Unique) } log.Printf("unique ID is %s", problem.Unique) log.Printf(" this is an update of problem %d (%q)", existing[0].ID, existing[0].Note) problem.ID = existing[0].ID problem.CreatedAt = existing[0].CreatedAt default: // server does not know what "unique" means log.Fatalf("error: server found multiple problems with matching unique ID %q", problem.Unique) } // generate steps whitelist := make(map[string]bool) for i := int64(1); cfg.Step[strconv.FormatInt(i, 10)] != nil; i++ { log.Printf("gathering step %d", i) s := cfg.Step[strconv.FormatInt(i, 10)] step := &ProblemStep{ Step: i, Note: s.Note, Weight: s.Weight, Files: make(map[string]string), } commit := &Commit{ Step: i, Action: "confirm", Note: "author solution submitted via grind", Files: make(map[string]string), CreatedAt: now, UpdatedAt: now, } // read files starter, solution, root := make(map[string]string), make(map[string]string), make(map[string]string) stepdir := filepath.Join(dir, strconv.FormatInt(i, 10)) err := filepath.Walk(stepdir, func(path string, info os.FileInfo, err error) error { if err != nil { log.Fatalf("walk error for %s: %v", path, err) } if info.IsDir() { return nil } relpath, err := filepath.Rel(stepdir, path) if err != nil { log.Fatalf("error finding relative path of %s: %v", path, err) } // load the file and add it to the appropriate place contents, err := ioutil.ReadFile(path) if err != nil { log.Fatalf("error reading %s: %v", relpath, err) } // pick out solution/starter files reldir, relfile := filepath.Split(relpath) if reldir == "_solution/" && relfile != "" { solution[relfile] = string(contents) } else if reldir == "_starter/" && relfile != "" { starter[relfile] = string(contents) } else if reldir == "" && relfile != "" { root[relfile] = string(contents) } else { step.Files[relpath] = string(contents) } return nil }) if err != nil { log.Fatalf("walk error for %s: %v", stepdir, err) } // find starter files and solution files if len(solution) > 0 && len(starter) > 0 && len(root) > 0 { log.Fatalf("found files in _starter, _solution, and root directory; unsure how to proceed") } if len(solution) > 0 { // explicit solution } else if len(root) > 0 { // files in root directory must be the solution solution = root root = nil } else { log.Fatalf("no solution files found in _solution or root directory; problem must have a solution") } if len(starter) == 0 && root != nil { starter = root } // copy the starter files into the step for name, contents := range starter { step.Files[name] = contents // if the file exists as a starter in this or earlier steps, it can be part of the solution whitelist[name] = true } // copy the solution files into the commit for name, contents := range solution { if whitelist[name] { commit.Files[name] = contents } else { log.Printf("Warning: skipping solution file %q", name) log.Printf(" because it is not in the starter file set of this or any previous step") } } unsigned.ProblemSteps = append(unsigned.ProblemSteps, step) unsigned.Commits = append(unsigned.Commits, commit) log.Printf(" found %d problem definition file%s and %d solution file%s", len(step.Files), plural(len(step.Files)), len(commit.Files), plural(len(commit.Files))) } if len(unsigned.ProblemSteps) != len(cfg.Step) { log.Fatalf("expected to find %d step%s, but only found %d", len(cfg.Step), plural(len(cfg.Step)), len(unsigned.ProblemSteps)) } // get user ID user := new(User) mustGetObject("/users/me", nil, user) // get the request validated and signed signed := new(ProblemBundle) mustPostObject("/problem_bundles/unconfirmed", nil, unsigned, signed) // validate the commits one at a time for n := 0; n < len(signed.ProblemSteps); n++ { log.Printf("validating solution for step %d", n+1) unvalidated := &CommitBundle{ Problem: signed.Problem, ProblemSteps: signed.ProblemSteps, ProblemSignature: signed.ProblemSignature, Commit: signed.Commits[n], CommitSignature: signed.CommitSignatures[n], } validated := mustConfirmCommitBundle(user.ID, unvalidated, nil) log.Printf(" finished validating solution") if validated.Commit.ReportCard == nil || validated.Commit.Score != 1.0 || !validated.Commit.ReportCard.Passed { log.Printf(" solution for step %d failed: %s", n+1, validated.Commit.ReportCard.Note) // play the transcript for _, event := range validated.Commit.Transcript { switch event.Event { case "exec": color.Cyan("$ %s\n", strings.Join(event.ExecCommand, " ")) case "stdin": color.Yellow("%s", event.StreamData) case "stdout": color.White("%s", event.StreamData) case "stderr": color.Red("%s", event.StreamData) case "exit": color.Cyan("%s\n", event.ExitStatus) case "error": color.Red("Error: %s\n", event.Error) } } log.Fatalf("please fix solution and try again") } signed.Problem = validated.Problem signed.ProblemSteps = validated.ProblemSteps signed.ProblemSignature = validated.ProblemSignature signed.Commits[n] = validated.Commit signed.CommitSignatures[n] = validated.CommitSignature } log.Printf("problem and solution confirmed successfully") // save the problem final := new(ProblemBundle) if signed.Problem.ID == 0 { mustPostObject("/problem_bundles/confirmed", nil, signed, final) } else { mustPutObject(fmt.Sprintf("/problem_bundles/%d", signed.Problem.ID), nil, signed, final) } log.Printf("problem %q saved and ready to use", final.Problem.Unique) if signed.Problem.ID == 0 { // create a matching problem set // pause for a bit since the database seems to need to catch up time.Sleep(time.Second) // create a problem set with just this problem and the same unique name psBundle := &ProblemSetBundle{ ProblemSet: &ProblemSet{ Unique: final.Problem.Unique, Note: "set for single problem " + final.Problem.Unique + "\n" + final.Problem.Note, Tags: final.Problem.Tags, CreatedAt: now, UpdatedAt: now, }, ProblemIDs: []int64{final.Problem.ID}, Weights: []float64{1.0}, } finalPSBundle := new(ProblemSetBundle) mustPostObject("/problem_set_bundles", nil, psBundle, finalPSBundle) log.Printf("problem set %q created and ready to use for this problem", finalPSBundle.ProblemSet.Unique) } }
// given a file path as arg, render the markdown func Render(arg string, stats bool) { // get the current Directory workingDir, err := os.Getwd() if err != nil { fmt.Println(err) os.Exit(1) } // construct the file path filePath := path.Join(workingDir, arg) file, err := os.Open(filePath) if err != nil { log.Fatal(err) } defer file.Close() scanner := bufio.NewScanner(file) input := "" meta_info := make(map[string]string) // isolated meta info from the markdown real body i := 0 for scanner.Scan() { line := scanner.Text() if strings.HasPrefix(line, "---") { i += 1 continue } // until we reach the end meta block, just continue if i > 0 && i < 2 { key_value := strings.SplitN(line, ":", 2) if len(key_value) == 2 { key := strings.Trim(key_value[0], " ") value := strings.Trim(key_value[1], " ") value = strings.Trim(value, "\"") value = strings.Trim(value, " ") meta_info[key] = value } continue } if strings.HasPrefix(line, string('#')) { line = goarabic.RemoveTashkeel(line) } input += line + "\n" } if stats { current_input := input defer func() { color.Magenta("---- stats ----\n") fmt.Println("Length:", goarabic.SmartLength(¤t_input), "chars.") fmt.Println("Words: ~", len(strings.Split(current_input, " ")), "words.") links := xurls.Relaxed.FindAllString(current_input, -1) for i := 0; i < len(links); i++ { color.White("Found external Link: " + links[i]) } }() } if len(meta_info) > 0 { orig_link := "" orig_title := "" orig_author := "" title := "" if val, ok := meta_info["عنوان المقال"]; ok { title = val title_no_tashkeel := goarabic.RemoveTashkeel(title) if title != goarabic.RemoveTashkeel(title) { input = "# " + title_no_tashkeel + "\n" + input } } if val, ok := meta_info["الرابط الأصلي للمقال (إن كان مترجما)"]; ok { orig_link = val } else if val, ok := meta_info["الرابط الأصلي للمقال"]; ok { orig_link = val } if val, ok := meta_info["العنوان الأصلي للمقال (إن كان مترجما)"]; ok { orig_title = val } else if val, ok := meta_info["العنوان الأصلي للمقال"]; ok { orig_title = val } if len(orig_link) > 0 && len(orig_title) > 0 { input += "\nترجمة -وبتصرف- للمقال: [" + orig_title + "](" + orig_link + ")" } if val, ok := meta_info["الكاتب الأصلي (إن كان مترجما)"]; ok { orig_author = val } else if val, ok := meta_info["الكاتب الأصلي"]; ok { orig_author = val } if len(orig_author) > 0 { input += " لصاحبه: " + orig_author + ".\n" } else { input += ".\n" } } if err := scanner.Err(); err != nil { log.Fatal(err) } // set up Markdown options extensions := 0 extensions |= blackfriday.EXTENSION_TABLES // extensions |= blackfriday.EXTENSION_FENCED_CODE extensions |= blackfriday.EXTENSION_AUTOLINK extensions |= blackfriday.EXTENSION_STRIKETHROUGH // extensions |= blackfriday.EXTENSION_SPACE_HEADERS extensions |= blackfriday.EXTENSION_HARD_LINE_BREAK extensions |= blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK extensions |= blackfriday.EXTENSION_LAX_HTML_BLOCKS // Make a custom Renderer var renderer blackfriday.Renderer htmlFlags := 0 htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS renderer = blackfriday.HtmlRenderer(htmlFlags, "", "") output := blackfriday.Markdown([]byte(input), renderer, extensions) strOutput := string(output) err = clipboard.WriteAll(strOutput) // fmt.Println(strOutput) if err != nil { log.Println(err) color.Yellow("Content here instead.\n\n") fmt.Println(strOutput) } else { color.Green("Markdown content copied to clipboard.") } }
// // lord commander // func main() { color.Cyan("***************************************************************************\n") color.Cyan("**** Welcome use CNKI-Downloader, Let's f**k these knowledge mongers ****\n") color.Cyan("**** Good luck. ****\n") color.Cyan("***************************************************************************\n") defer func() { color.Yellow("** Bye.\n") }() // // note // fmt.Println() fmt.Println("** NOTE: if you cannot download any document, maybe the service of") fmt.Println("** CNKI is unavailable again, in this situation, nothing ") fmt.Println("** we can do but wait, please do not open a issue on GitHub, thanks") fmt.Println("**") // // update // v := update() if !v { return } // // login // downloader := &CNKIDownloader{ username: "******", password: "******", http_client: &http.Client{}, } fmt.Printf("** Login...") err := downloader.Auth() if err != nil { fmt.Fprintf(color.Output, "%s : %s \n", color.RedString("Failure"), err.Error()) return } else { fmt.Fprintf(color.Output, "%s\n\n", color.GreenString("Success")) } for { fmt.Fprintf(color.Output, "$ %s", color.CyanString("input anything you wanna search: ")) s := getInputString() if len(s) == 0 { continue } // // search first page // opt := getSearchOpt() result, err := downloader.SearchFirst(s, opt) if err != nil { fmt.Fprintf(color.Output, "Search %s %s (%s)\n", "zzr", color.RedString("Failure"), err.Error()) continue } printArticles(1, result.GetPageData()) // // tips // fmt.Fprintf(color.Output, "We got (%s) entries in total. (if u don't know how to do next, just type '%s') \n", color.GreenString("%d", result.GetRecordInfo()), color.RedString("help")) for { out := false ctx, err := downloader.CurrentPage() if err != nil { break } psize, pindex, pcount := ctx.GetPageInfo() fmt.Fprintf(color.Output, "$ [%d/%d] %s", pindex, pcount, color.CyanString("command: ")) s = getInputString() cmd_parts := strings.Split(s, " ") switch strings.ToLower(cmd_parts[0]) { case "help": { fmt.Fprintf(color.Output, "Support follow commands:\n") fmt.Fprintf(color.Output, "\t %s: show page's information\n", color.YellowString("INFO")) fmt.Fprintf(color.Output, "\t %s: turn to next page\n", color.YellowString("NEXT")) fmt.Fprintf(color.Output, "\t %s: turn to previous page\n", color.YellowString("PREV")) fmt.Fprintf(color.Output, "\t %s: (GET ID), download the specified item in this page, eg: GET 1, GET 14...etc\n", color.YellowString("GET")) fmt.Fprintf(color.Output, "\t %s: (SHOW ID), show the information about specified item, eg: SHOW 2, SHOW 9...etc\n", color.YellowString("SHOW")) fmt.Fprintf(color.Output, "\t%s: break out, and search the other papers\n", color.YellowString("BREAK")) } case "info": { color.White(" page size: %d\n page index: %d\ntotal pages: %d\n", psize, pindex, pcount) } case "next": { next_page, err := downloader.SearchNext(pindex + 1) if err != nil { fmt.Fprintf(color.Output, "Next page is invalid (%s)\n", color.RedString(err.Error())) } else { _, index, _ := next_page.GetPageInfo() printArticles(index, next_page.GetPageData()) } } case "prev": { prev_page, err := downloader.SearchPrev() if err != nil { color.Red("Previous page is invalid") } else { _, index, _ := prev_page.GetPageInfo() printArticles(index, prev_page.GetPageData()) } } case "show": { if len(cmd_parts) < 2 { color.Red("Invalid input") break } id, err := strconv.ParseInt(cmd_parts[1], 10, 32) if err != nil { fmt.Fprintf(color.Output, "Invalid input %s\n", color.RedString(err.Error())) break } id-- entries := ctx.GetPageData() entry := entries[id] fmt.Println() fmt.Fprintf(color.Output, "* PAGE: %s\n", color.WhiteString("%d", pindex)) fmt.Fprintf(color.Output, "* ID: %s\n", color.WhiteString("%d", id+1)) fmt.Fprintf(color.Output, "* Title: %s\n", color.WhiteString(entry.Information.Title)) fmt.Fprintf(color.Output, "* Created: %s\n", color.WhiteString(entry.Information.CreateTime)) fmt.Fprintf(color.Output, "* Authors: %s\n", color.GreenString(strings.Join(entry.Information.Creator, " "))) fmt.Fprintf(color.Output, "* Source: %s\n", color.GreenString("%s(%s)", entry.Information.SourceName, entry.Information.SourceAlias)) fmt.Fprintf(color.Output, "* Code: %s\n", color.WhiteString("%s.%s", entry.Information.ClassifyName, entry.Information.ClassifyCode)) fmt.Fprintf(color.Output, "* Reference: %s\n", color.RedString("%d", entry.Information.RefCount)) fmt.Fprintf(color.Output, "* Downloaded: %s\n", color.WhiteString("%d", entry.Information.DownloadCount)) fmt.Fprintf(color.Output, "*Description: \n") //text := mahonia.NewDecoder("gbk").ConvertString(entry.Information.Description) textSeq := []rune(entry.Information.Description) for j := 0; j < len(textSeq); { end := j + 40 if len(textSeq)-j < 40 { end = len(textSeq) - 1 } fmt.Printf("* %s\n", string(textSeq[j:end])) j = end + 1 } fmt.Println() } case "get": { if len(cmd_parts) < 2 { color.Red("Invalid input") break } id, err := strconv.ParseInt(cmd_parts[1], 10, 32) if err != nil { fmt.Fprintf(color.Output, "Invalid input %s\n", color.RedString(err.Error())) break } id-- entries := ctx.GetPageData() color.White("Downloading... %s\n", entries[id].Information.Title) path, err := downloader.Download(&entries[id]) if err != nil { fmt.Fprintf(color.Output, "Download failed %s\n", color.RedString(err.Error())) break } fmt.Fprintf(color.Output, "Download success (%s) \n", color.GreenString(path)) } case "break": { downloader.SearchStop() color.YellowString("Break out.\n") out = true } } if out { break } } } return }
func jobs_deploy(c *cli.Context) error { // pull the image err := error(nil) // some weird scoping issues if we don't declare here docker_image := c.String("image") if docker_image == "" { docker_image, err = get_docker_image_name(c) if err != nil { return cli.NewExitError(err.Error(), 1) } } else { if c.GlobalString("registry") != "" { docker_image = c.GlobalString("registry") + "/" + docker_image } if c.GlobalString("tag") != "" { docker_image = docker_image + ":" + c.GlobalString("tag") } } log.Info("Using docker image:", docker_image) if c.Bool("pull") { log.Info("Pulling", docker_image) cmd := exec.Command("docker", "pull", docker_image) _, err = cmd.CombinedOutput() if err != nil { log.Warning("Unable to pull the image. Checking if it is locally available.", err) } } // extract the JSON cmd := exec.Command("docker", "inspect", "-f", "{{(index .Config.Labels \"com.ngageoint.scale.job-type\")}}", docker_image) output, err := cmd.CombinedOutput() if err != nil { log.Debug(string(output)) return cli.NewExitError(err.Error(), 1) } json_value := strings.Replace(string(output), "$ {", "${", -1) if strings.TrimSpace(json_value) == "" { return cli.NewExitError(fmt.Sprint("Scale job type information not found in ", docker_image), 1) } var job_type scalecli.JobType err = json.Unmarshal([]byte(json_value), &job_type) if err != nil { return cli.NewExitError(err.Error(), 1) } url := c.GlobalString("url") if url == "" { return cli.NewExitError("A URL must be provided with the SCALE_URL environment variable or the --url argument", 1) } if c.Bool("n") { // validate only warnings, err := scalecli.ValidateJobType(url, job_type) if err != nil { return cli.NewExitError(err.Error(), 1) } if warnings == "" { color.White("Job type specification is valid.") } else { color.Yellow(warnings) } return nil } // check for existing job type job_types, err := scalecli.GetJobTypes(url, job_type.Name) if err != nil { return cli.NewExitError(err.Error(), 1) } if len(job_types) == 0 { // create a new job type log.Info("Creating new job type entry") err = scalecli.CreateJobType(url, job_type) if err != nil { return cli.NewExitError(err.Error(), 1) } } else { for _, jt := range job_types { if jt.Version == job_type.Version { // found an exising entry, either update or create a new one log.Info("Updating job type metadata", jt.Id) err = scalecli.UpdateJobType(url, jt.Id, job_type) if err != nil { return cli.NewExitError(err.Error(), 1) } return nil } } // no entry found, create a new one log.Info("Creating new job type entry for", job_type.Name, "version", job_type.Version) err = scalecli.CreateJobType(url, job_type) if err != nil { return cli.NewExitError(err.Error(), 1) } } return nil }