func parseGauges(info string) map[string]int64 {
gauges_with_values := map[string]int64{
"blocked_clients": 0,
"connected_clients": 0,
"instantaneous_ops_per_sec": 0,
"latest_fork_usec": 0,
"mem_fragmentation_ratio": 0,
"migrate_cached_sockets": 0,
"pubsub_channels": 0,
"pubsub_patterns": 0,
"uptime_in_seconds": 0,
"used_memory": 0,
"used_memory_lua": 0,
"used_memory_peak": 0,
"used_memory_rss": 0,
}
color.White("-------------------")
color.White("GAUGES:")
for gauge, _ := range gauges_with_values {
r, _ := regexp.Compile(fmt.Sprint(gauge, ":([0-9]*)"))
matches := r.FindStringSubmatch(info)
if matches == nil {
color.Yellow(fmt.Sprint("WARN: ", gauge, "is not displayed in redis info"))
} else {
value := matches[len(matches)-1]
color.Cyan(fmt.Sprint(gauge, ": ", value))
v, _ := strconv.ParseInt(value, 10, 64)
gauges_with_values[gauge] = v
}
}
return gauges_with_values
}
func Start() {
if os.Getenv("IMGSTORAGE_LOCATION") != "" {
ImgStoragePath = os.Getenv("IMGSTORAGE_LOCATION")
} else {
reader := bufio.NewReader(os.Stdin)
color.Cyan("Enter location of image storage: ")
ImgStoragePath, _ = reader.ReadString('\n')
}
cio.PrintMessage(1, "Image storage is being created...")
if _, err := os.Stat(ImgStoragePath); os.IsNotExist(err) {
// doesn't exist
os.Mkdir(ImgStoragePath, 0776)
cio.PrintMessage(2, (ImgStoragePath + " created."))
} else {
cio.PrintMessage(2, (ImgStoragePath + " already existed."))
}
if os.Getenv("IMG_NAME_LENGTH") != "" {
ImgNameLength, _ = strconv.Atoi(os.Getenv("IMG_NAME_LENGTH"))
} else {
reader := bufio.NewReader(os.Stdin)
color.Cyan("Enter length of image names: ")
temp, _ := reader.ReadString('\n')
ImgNameLength, _ = strconv.Atoi(temp)
}
}
// TEST="asdf,asdf,asdf" ./main template.tpl
func main() {
if len(os.Args) != 2 {
color.Red("tenv: v.01")
color.Blue("------------------------")
color.Cyan("Prepopulates to stdin given template with information stored in env variables")
color.Cyan("variables should follow go template syntax {{.VAR_NAME}}")
color.Cyan("and must be declared on the environment")
color.Cyan("Usage : tenv filename")
os.Exit(1)
}
var funcMap template.FuncMap
funcMap = template.FuncMap{
"split": strings.Split,
}
file := filepath.Base(os.Args[1])
t, err := template.New(file).Funcs(funcMap).ParseFiles(os.Args[1])
if err != nil {
log.Fatalf("Error: %s", err)
}
context := make(map[string]string)
for _, v := range os.Environ() {
p := strings.Split(v, "=")
context[p[0]] = p[1]
}
err = t.ExecuteTemplate(os.Stdout, file, context)
if err != nil {
log.Fatal(err)
}
}
// Start is the database package launch method
// it enters or fetches the data required for the database
func Start() {
/*
* allow user to enter db data
* used instead of environment variables
* if there are none
* since the service is open source
*/
var (
uname string
pw string
name string
)
if os.Getenv("DB_UNAME") == "" && os.Getenv("DB_NAME") == "" {
reader := bufio.NewReader(os.Stdin)
color.Cyan("Enter db user name: ")
uname, _ = reader.ReadString('\n')
color.Cyan("Enter db pw: ")
pw, _ = reader.ReadString('\n')
color.Cyan("Enter db name: ")
name, _ = reader.ReadString('\n')
} else {
uname = os.Getenv("DB_UNAME")
pw = os.Getenv("DB_PW")
name = os.Getenv("DB_NAME")
}
var err error
db, err = sql.Open("postgres",
"user="******" password="******" dbname="+name+
" sslmode=disable")
if err != nil {
cio.PrintMessage(1, err.Error())
return
}
// test connection
err = db.Ping()
if err != nil { // connection not successful
cio.PrintMessage(1, err.Error())
var rundb string
reader := bufio.NewReader(os.Stdin)
color.Cyan("Do you want to run the server without a working database module? (y/n) ")
rundb, _ = reader.ReadString('\n')
if rundb != "y\n" && rundb != "Y\n" {
os.Exit(-1)
}
}
}
// conect to server operation
func connect(matched []Server, printOnly bool) {
if len(matched) == 0 {
color.Cyan("No server match patterns")
} else if len(matched) == 1 {
color.Green("%s", matched[0].getConnectionString())
if !printOnly {
matched[0].connect()
}
} else {
color.Cyan("Multiple servers match patterns:")
for _, s := range matched {
color.White(s.getConnectionString())
}
}
}
// download file from server operation
func download(src string, dest string, matched []Server, printOnly bool) {
if len(matched) == 0 {
color.Cyan("No server match patterns")
} else if len(matched) == 1 {
color.Green("%s", matched[0].getDownloadString(src, dest))
if !printOnly {
matched[0].download(src, dest)
}
} else {
color.Cyan("Multiple servers match patterns:")
for _, s := range matched {
color.White(s.getDownloadString(src, dest))
}
}
}
func TesTFetchingJob(t *testing.T) {
out, _ := jobworker.FetchJob("parsebin")
color.Cyan(out)
}
// Build listens watch events from Watcher and sends messages to Runner
// when new changes are built.
func (b *Builder) Build(p *Params) {
go b.registerSignalHandler()
go func() {
b.watcher.update <- true
}()
for <-b.watcher.Wait() {
fileName := p.createBinaryName()
pkg := p.GetPackage()
color.Cyan("Building %s...\n", pkg)
// build package
cmd, err := runCommand("go", "build", "-o", fileName, pkg)
if err != nil {
log.Fatalf("Could not run 'go build' command: %s", err)
continue
}
if err := cmd.Wait(); err != nil {
if err := interpretError(err); err != nil {
color.Red("An error occurred while building: %s", err)
} else {
color.Red("A build error occurred. Please update your code...")
}
continue
}
// and start the new process
b.runner.restart(fileName)
}
}
func parseCounters(info string) map[string]int64 {
counters := map[string]int64{
"evicted_keys": 0,
"expired_keys": 0,
"keyspace_hits": 0,
"keyspace_misses": 0,
"rejected_connections": 0,
"sync_full": 0,
"sync_partial_err": 0,
"sync_partial_ok": 0,
"total_commands_processed": 0,
"total_connections_received": 0,
}
color.White("-------------------")
color.White("COUNTERS:")
for counter, _ := range counters {
r, _ := regexp.Compile(fmt.Sprint(counter, ":([0-9]*)"))
matches := r.FindStringSubmatch(info)
if matches == nil {
color.Yellow(fmt.Sprint("ERROR: ", counter, "is not displayed in redis info"))
} else {
value := matches[len(matches)-1]
color.Cyan(fmt.Sprint(counter, ": ", value))
v, _ := strconv.ParseInt(value, 10, 64)
counters[counter] = v
}
}
return counters
}
func writeDebug(format string, a ...interface{}) {
if !debug {
return
}
color.Cyan(format, a...)
}
func TesTGettingJobInfo(t *testing.T) {
jid := jobworker.GetPendingJids("parsebin")
out := jobworker.GetJobInfo("parsebin", jid)
color.Cyan(out)
}
func PrintMessage(cl int, msg string) {
if cl == 0 {
color.Green("[pixmate] %s", msg)
} else if cl == 1 {
color.Red("[pixmate] %s", msg)
} else if cl == 2 {
color.Cyan("[pixmate] %s", msg)
}
}
func (this *Options) Usage() {
var banner string = ` ____ _
| _ \ ___ _ __ | | __ _ ___ ___
| |_) / _ \ '_ \| |/ _' |/ __/ _ \
| _ < __/ |_) | | (_| | (_| __/
|_| \_\___| .__/|_|\__'_|\___\___|
|_|
`
color.Cyan(banner)
flag.Usage()
}
// Do accepts a function argument that returns an error. It will keep executing this
// function NumRetries times until no error is returned.
// Optionally pass another function as an argument that it will execute before retrying
// in case an error is returned from the first argument fuction.
func Do(args ...interface{}) error {
if len(args) == 0 {
panic("Wrong number of arguments")
}
task, ok := args[0].(func() error)
beforeRetry := func() {}
if len(args) > 1 {
beforeRetry, ok = args[1].(func())
}
if ok == false {
panic("Wrong Type of Arguments given to retry.Do")
}
retries := NumRetries
var atleastOneError error
atleastOneError = nil
err := errors.New("Non-Nil error")
for retries > 0 && err != nil {
err = task()
if err != nil {
atleastOneError = err
color.Magenta("\nError: %v\nRetrying #%d after %v", err, (NumRetries - retries + 1), Delay)
time.Sleep(Delay)
beforeRetry()
}
retries = retries - 1
}
if err != nil {
color.Red("\nError even after %d retries:\n%v", NumRetries, err)
if PanicEnabled == true {
panic(err)
}
return err
}
if atleastOneError != nil {
color.Cyan("\nRecovered from error: %v in %d tries\n", atleastOneError, (NumRetries - retries - 1))
}
return err
}
func show(dict Dict) {
for _, ps := range dict.Ps {
color.Green(ps)
}
for index, pos := range dict.Pos {
color.Red(strings.TrimSpace(pos))
color.Yellow(strings.TrimSpace(dict.Acceptation[index]))
}
for _, sent := range dict.SentList {
color.Blue("ex. %s", strings.TrimSpace(sent.Orig))
color.Cyan(" %s", strings.TrimSpace(sent.Trans))
}
}
func printSSHKeys(v *vaulted.Vault) {
color.Cyan("\nSSH Keys:")
if len(v.SSHKeys) > 0 {
keys := []string{}
for key := range v.SSHKeys {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
green.Printf(" %s\n", key)
}
} else {
print(" [Empty]")
}
}
func main() {
url := flag.String("url", "", "webservice url")
count := flag.Int("count", 1, "requests count")
stats := flag.Bool("stats", false, "print stats only")
method := flag.String("method", "GET", "http method (POST or GET)")
flag.Parse()
if len(*url) == 0 {
fmt.Println("Missing param url")
return
}
httpFunc := get
if *method == "POST" {
httpFunc = post
}
responses := make(chan int, *count)
for i := 0; i < *count; i++ {
go httpFunc(responses, *url)
}
respMap := make(map[int]int)
for i := 0; i < *count; i++ {
statusCode := <-responses
respMap[statusCode]++
if *stats {
continue
}
statusMsg := fmt.Sprintf("%d :: %d", i, statusCode)
if statusCode != 200 {
color.Red(statusMsg)
} else {
color.Cyan(statusMsg)
}
}
if *stats {
for k, v := range respMap {
fmt.Printf("%d :: %-5d\n", k, v)
}
}
}
func printVariables(v *vaulted.Vault) {
color.Cyan("\nVariables:")
if len(v.Vars) > 0 {
var keys []string
for key := range v.Vars {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
green.Printf(" %s: ", key)
fmt.Printf("%s\n", v.Vars[key])
}
} else {
print(" [Empty]")
}
}
func promptUserToSelectModule(modules []Module, optional bool) (Module, error) {
// Prompt the user to select a module.
kind := modules[0].Kind()
for {
// Write the dialog into the console.
ctx := &dialogTemplateContext{
Kind: string(kind),
Modules: modules,
Optional: optional,
}
if err := dialogTemplate.Execute(os.Stdout, ctx); err != nil {
return nil, err
}
// Prompt the user for the answer.
// An empty answer is aborting the dialog.
answer, err := prompt.Prompt("You choice: ")
if err != nil {
if err == prompt.ErrCanceled {
prompt.PanicCancel()
}
return nil, err
}
if optional && answer == "s" {
fmt.Println()
color.Cyan("Skipping module kind '%v'", kind)
return nil, nil
}
// Parse the index and return the associated module.
i, err := strconv.Atoi(answer)
if err == nil {
if 0 < i && i <= len(modules) {
return modules[i-1], nil
}
}
// In case we failed to parse the index or something, run the dialog again.
color.Yellow("Not a valid choice, please try again!")
}
}
func RunRecursively(cmd CliCommand) {
t := time.Now()
wg := new(sync.WaitGroup)
pathChan := make(chan vcsPath)
count := 0
go findRepositories(".", pathChan)
for p := range pathChan {
vcsCmd := cmd.GetVcsCommand(p.Sign)
if vcsCmd != nil {
wg.Add(1)
go execVcsCmd(vcsCmd, p.Path, wg)
count++
}
}
wg.Wait()
color.Cyan("Done \"%s\" command for %d repos in %s\n\n", cmd.Name, count, time.Since(t))
}
func main() {
flag.Parse()
if *sitemapURL == "" {
fmt.Println("-sitemap に sitemap.xml/.xml.gz のURLを指定してください")
return
}
smap, err := sitemap.Get(*sitemapURL)
if err != nil {
fmt.Println(err)
}
for _, URL := range smap.URL {
time.Sleep(time.Second)
resp, err := http.Get(URL.Loc)
if err != nil {
fmt.Println(err)
continue
}
defer resp.Body.Close()
switch statusType(resp.StatusCode) {
case 100:
color.Cyan(resp.Status + " " + URL.Loc)
case 200:
color.Green(resp.Status + " " + URL.Loc)
case 300:
color.Magenta(resp.Status + " " + URL.Loc)
case 400:
color.Red(resp.Status + " " + URL.Loc)
case 500:
color.Yellow(resp.Status + " " + URL.Loc)
default:
color.White(resp.Status + " " + URL.Loc)
}
}
}
func colorLineDiff(a, b string) {
aLines := strings.Split(a, "\n")
bLines := strings.Split(b, "\n")
fmt.Print("Overlapping portion of reassembled TCP Stream:\n")
for i := 0; i < len(aLines); i++ {
if aLines[i] == bLines[i] {
color.Blue(aLines[i])
} else {
color.Green(aLines[i])
}
}
fmt.Print("Injection packet whose contents did not coalesce into the TCP Stream:\n")
for i := 0; i < len(aLines); i++ {
if aLines[i] == bLines[i] {
color.Cyan(bLines[i])
} else {
color.Red(bLines[i])
}
}
}
func printAWS(v *vaulted.Vault, show bool) {
color.Cyan("\nAWS Key:")
if v.AWSKey != nil {
green.Printf(" Key ID: ")
fmt.Printf("%s\n", v.AWSKey.ID)
green.Printf(" Secret: ")
if !show {
fmt.Printf("%s\n", "<hidden>")
} else {
fmt.Printf("%s\n", v.AWSKey.Secret)
}
if v.AWSKey.MFA != "" {
green.Printf(" MFA: ")
fmt.Printf("%s\n", v.AWSKey.MFA)
}
if v.AWSKey.Role != "" {
green.Printf(" Role: ")
fmt.Printf("%s\n", v.AWSKey.Role)
}
} else {
print(" [Empty]")
}
}
// Build listens watch events from Watcher and sends messages to Runner
// when new changes are built.
func (b *Builder) Build(p *Params) {
go b.registerSignalHandler()
go func() {
// used for triggering the first build
b.watcher.update <- struct{}{}
}()
for range b.watcher.Wait() {
fileName := p.generateBinaryName()
pkg := p.packagePath()
log.Println("build started")
color.Cyan("Building %s...\n", pkg)
// build package
cmd, err := runCommand("go", "build", "-o", fileName, pkg)
if err != nil {
log.Fatalf("Could not run 'go build' command: %s", err)
continue
}
if err := cmd.Wait(); err != nil {
if err := interpretError(err); err != nil {
color.Red("An error occurred while building: %s", err)
} else {
color.Red("A build error occurred. Please update your code...")
}
continue
}
log.Println("build completed")
// and start the new process
b.runner.restart(fileName)
}
}
func main() {
username := ""
password := ""
language := "English"
quality := "1080p"
trim := ""
daisukiIntroTrim := false
aniplexIntroTrim := false
sunriseIntroTrim := false
app := cli.NewApp()
app.Name = "anirip"
app.Author = "Steven Wolfe"
app.Email = "*****@*****.**"
app.Version = "v1.4.0(7/7/2016)"
app.Usage = "Crunchyroll/Daisuki show ripper CLI"
color.Cyan(app.Name + " " + app.Version + " - by " + app.Author + " <" + app.Email + ">\n")
app.Flags = []cli.Flag{
cli.StringFlag{
Name: "lang, l",
Value: "english",
Usage: "desired subtitle language",
Destination: &language,
},
cli.StringFlag{
Name: "quality, q",
Value: "1080p",
Usage: "desired video quality",
Destination: &quality,
},
cli.StringFlag{
Name: "trim, t",
Value: "",
Usage: "desired intros to be trimmed off of final video",
Destination: &trim,
},
}
app.Commands = []cli.Command{
{
Name: "login",
Aliases: []string{"l"},
Usage: "creates and stores cookies for a stream provider",
Flags: []cli.Flag{
cli.StringFlag{
Name: "user, u",
Value: "myusername",
Usage: "premium username used to access video stream",
Destination: &username,
},
cli.StringFlag{
Name: "pass, p",
Value: "mypassword",
Usage: "premium password used to access video stream",
Destination: &password,
},
},
Action: func(c *cli.Context) error {
// Gets the provider name from the cli argument
provider := ""
if c.NArg() > 0 {
provider = c.Args()[0]
} else {
color.Red("[anirip] No provider given...")
return anirip.Error{Message: "No provider given"}
}
// Creates session with cookies to store in file
var session anirip.Session
if strings.Contains(provider, "crunchyroll") {
color.Cyan("[anirip] Logging to CrunchyRoll as " + username + "...")
session = new(crunchyroll.CrunchyrollSession)
} else if strings.Contains(provider, "daisuki") {
color.Cyan("[anirip] Logging to Daisuki as " + username + "...")
session = new(daisuki.DaisukiSession)
} else {
color.Red("[anirip] The given provider is not supported.")
return anirip.Error{Message: "The given provider is not supported"}
}
// Performs the login procedure, storing the login information to file
if err := session.Login(username, password, tempDir); err != nil {
color.Red("[anirip] " + err.Error())
return anirip.Error{Message: "Unable to login to provider", Err: err}
}
color.Green("[anirip] Successfully logged in... Cookies saved to " + tempDir)
return nil
},
},
{
Name: "clear",
Aliases: []string{"c"},
Usage: "erases the temporary directory used for cookies and temp files",
Action: func(c *cli.Context) error {
// Attempts to erase the temporary directory
if err := os.RemoveAll(tempDir); err != nil {
color.Red("[anirip] There was an error erasing the temporary directory : " + err.Error())
return anirip.Error{Message: "There was an error erasing the temporary directory", Err: err}
}
color.Green("[anirip] Successfully erased the temporary directory " + tempDir)
return nil
},
},
}
app.Action = func(c *cli.Context) error {
if c.NArg() == 0 {
color.Red("[anirip] No show URLs provided.")
return anirip.Error{Message: "No show URLs provided"}
}
for _, showURL := range c.Args() {
// Parses the URL so we can accurately judge the provider based on the host
url, err := url.Parse(showURL)
if err != nil {
color.Red("[anirip] There was an error parsing the URL you entered.\n")
return anirip.Error{Message: "There was an error parsing the URL you entered"}
}
// Creates the authentication & show objects for the provider we're ripping from
var session anirip.Session
var show anirip.Show
if strings.Contains(strings.ToLower(url.Host), "crunchyroll") {
show = new(crunchyroll.CrunchyrollShow)
session = new(crunchyroll.CrunchyrollSession)
} else if strings.Contains(strings.ToLower(url.Host), "daisuki") {
show = new(daisuki.DaisukiShow)
session = new(daisuki.DaisukiSession)
} else {
color.Red("[anirip] The URL provided is not supported.")
return anirip.Error{Message: "The URL provided is not supported"}
}
// Performs the generic login procedure
if err = session.Login(username, password, tempDir); err != nil {
color.Red("[anirip] " + err.Error())
return anirip.Error{Message: "Unable to login to provider", Err: err}
}
// Attempts to scrape the shows metadata/information
color.White("[anirip] Getting a list of episodes for the show...")
if err = show.ScrapeEpisodes(showURL, session.GetCookies()); err != nil {
color.Red("[anirip] " + err.Error())
return anirip.Error{Message: "Unable to get episodes", Err: err}
}
// Sets the boolean values for what intros we would like to trim
if strings.Contains(strings.ToLower(trim), "daisuki") {
daisukiIntroTrim = true
}
if strings.Contains(strings.ToLower(trim), "aniplex") {
daisukiIntroTrim = true
}
if strings.Contains(strings.ToLower(trim), "sunrise") {
daisukiIntroTrim = true
}
seasonMap := map[int]string{
0: "Specials",
1: "Season One",
2: "Season Two",
3: "Season Three",
4: "Season Four",
5: "Season Five",
6: "Season Six",
7: "Season Seven",
8: "Season Eight",
9: "Season Nine",
10: "Season Ten",
}
os.Mkdir(show.GetTitle(), 0777)
for _, season := range show.GetSeasons() {
os.Mkdir(show.GetTitle()+string(os.PathSeparator)+seasonMap[season.GetNumber()], 0777)
for _, episode := range season.GetEpisodes() {
color.White("[anirip] Getting Episode Info...\n")
if err = episode.GetEpisodeInfo(quality, session.GetCookies()); err != nil {
color.Red("[anirip] " + err.Error())
continue
}
// Checks to see if the episode already exists, in which case we continue to the next
_, err = os.Stat(show.GetTitle() + string(os.PathSeparator) + seasonMap[season.GetNumber()] + string(os.PathSeparator) + episode.GetFileName() + ".mkv")
if err == nil {
color.Green("[anirip] " + episode.GetFileName() + ".mkv has already been downloaded successfully..." + "\n")
continue
}
subOffset := 0
color.Cyan("[anirip] Downloading " + episode.GetFileName() + "\n")
// Downloads full MKV video from stream provider
color.White("[anirip] Downloading video...\n")
if err := episode.DownloadEpisode(quality, tempDir, session.GetCookies()); err != nil {
color.Red("[anirip] " + err.Error() + "\n")
continue
}
// Trims down the downloaded MKV if the user wants to trim a Daisuki intro
if daisukiIntroTrim {
subOffset = subOffset + daisukiIntroLength
color.White("[anirip] Trimming off Daisuki Intro - " + strconv.Itoa(daisukiIntroLength) + "ms\n")
if err := trimMKV(daisukiIntroLength, tempDir); err != nil {
color.Red("[anirip] " + err.Error() + "\n")
continue
}
}
// Trims down the downloaded MKV if the user wants to trim an Aniplex intro
if aniplexIntroTrim {
subOffset = subOffset + aniplexIntroLength
color.White("[anirip] Trimming off Aniplex Intro - " + strconv.Itoa(aniplexIntroLength) + "ms\n")
if err := trimMKV(aniplexIntroLength, tempDir); err != nil {
color.Red("[anirip] " + err.Error() + "\n")
continue
}
}
// Trims down the downloaded MKV if the user wants to trim a Sunrise intro
if sunriseIntroTrim {
subOffset = subOffset + sunriseIntroLength
color.White("[anirip] Trimming off Sunrise Intro - " + strconv.Itoa(sunriseIntroLength) + "ms\n")
if err := trimMKV(sunriseIntroLength, tempDir); err != nil {
color.Red("[anirip] " + err.Error() + "\n")
continue
}
}
// Downloads the subtitles to .ass format and
// offsets their times by the passed provided interval
color.White("[anirip] Downloading subtitles with a total offset of " + strconv.Itoa(subOffset) + "ms...\n")
subtitleLang, err := episode.DownloadSubtitles(language, subOffset, tempDir, session.GetCookies())
if err != nil {
color.Red("[anirip] " + err.Error() + "\n")
continue
}
// Attempts to merge the downloaded subtitles into the video strea
color.White("[anirip] Merging subtitles into mkv container...\n")
if err := mergeSubtitles("jpn", subtitleLang, tempDir); err != nil {
color.Red("[anirip] " + err.Error() + "\n")
continue
}
// Cleans the MKVs metadata for better reading by clients
color.White("[anirip] Cleaning MKV...\n")
if err := cleanMKV(tempDir); err != nil {
color.Red("[anirip] " + err.Error() + "\n")
continue
}
// Moves the episode to the appropriate season sub-directory
if err := anirip.Rename(tempDir+string(os.PathSeparator)+"episode.mkv",
show.GetTitle()+string(os.PathSeparator)+seasonMap[season.GetNumber()]+string(os.PathSeparator)+episode.GetFileName()+".mkv", 10); err != nil {
color.Red(err.Error() + "\n\n")
}
color.Green("[anirip] Downloading and merging completed successfully.\n")
}
}
color.Cyan("[anirip] Completed processing episodes for " + show.GetTitle() + "\n")
}
return nil
}
app.Run(os.Args)
}
func (j SubjectsImport) Run() {
start := time.Now()
fmt.Println("------------------------------")
fmt.Printf("| Starting import at %s\n", start.Format("2006-01-02 15:04:05.000"))
db := controllers.Gdb
db.LogMode(false)
file := OpenExcelFile("/home/kolan/Documents/ONR_FITIS.xlsx")
fmt.Println("------------------------------")
rowCount := 0
successCount := 0
alreadyExist := 0
for _, row := range file.Sheets[1].Rows[10:] {
var subject models.Subjects
re := regexp.MustCompile("[^\\S]+")
re2 := regexp.MustCompile("([^\x00-\x7F])i")
re3 := regexp.MustCompile("i([^\x00-\x7F])")
subjectName := strings.TrimSpace(row.Cells[1].Value)
subjectName = re.ReplaceAllString(subjectName, " ")
subjectName = re2.ReplaceAllString(subjectName, "${1}і")
subjectName = re3.ReplaceAllString(subjectName, "i${1}")
subject.Subject = subjectName
result := db.Where(&subject).First(&subject)
if result.Error != nil {
if result.RecordNotFound() {
db.NewRecord(subject)
err := db.Save(&subject).Error
if err != nil {
if regexp.MustCompile(`^Error 1062:`).MatchString(err.Error()) == true {
alreadyExist += 1
color.Cyan("| \"%s\" - is already exist", subjectName)
} else {
color.Red("| Saving error for - \"%s\"", subjectName)
}
} else {
successCount += 1
color.Green("| \"%s\" - added", subjectName)
}
}
} else {
alreadyExist += 1
color.Cyan("| \"%s\" - is already exist", subjectName)
}
rowCount += 1
}
elapsed := time.Since(start)
fmt.Println("------------------------------")
fmt.Printf("| Import finished at %s\n", time.Now().Format("2006-01-02 15:04:05.000"))
fmt.Printf("| Importing time %s\n", elapsed)
color.Blue("------------------------------")
color.Blue("| Subjects in file : %d", rowCount)
color.Blue("| Imported subjects : %d", successCount)
color.Blue("| Already exist subjects : %d", alreadyExist)
color.Blue("------------------------------")
}
func mustConfirmCommitBundle(userID int64, bundle *CommitBundle, args []string) *CommitBundle {
verbose := false
// create a websocket connection to the server
headers := make(http.Header)
url := "wss://" + Config.Host + "/v2/sockets/" + bundle.Problem.ProblemType + "/" + bundle.Commit.Action
socket, resp, err := websocket.DefaultDialer.Dial(url, headers)
if err != nil {
log.Printf("error dialing %s: %v", url, err)
if resp != nil && resp.Body != nil {
io.Copy(os.Stderr, resp.Body)
resp.Body.Close()
}
log.Fatalf("giving up")
}
defer socket.Close()
// form the initial request
req := &DaycareRequest{UserID: userID, CommitBundle: bundle}
if err := socket.WriteJSON(req); err != nil {
log.Fatalf("error writing request message: %v", err)
}
// start listening for events
for {
reply := new(DaycareResponse)
if err := socket.ReadJSON(reply); err != nil {
log.Fatalf("socket error reading event: %v", err)
break
}
switch {
case reply.Error != "":
log.Printf("server returned an error:")
log.Fatalf(" %s", reply.Error)
case reply.CommitBundle != nil:
return reply.CommitBundle
case reply.Event != nil:
if verbose {
switch reply.Event.Event {
case "exec":
color.Cyan("$ %s\n", strings.Join(reply.Event.ExecCommand, " "))
case "stdin":
color.Yellow("%s", reply.Event.StreamData)
case "stdout":
color.White("%s", reply.Event.StreamData)
case "stderr":
color.Red("%s", reply.Event.StreamData)
case "exit":
color.Cyan("exit: %s\n", reply.Event.ExitStatus)
case "error":
color.Red("Error: %s\n", reply.Event.Error)
}
}
default:
log.Fatalf("unexpected reply from server")
}
}
log.Fatalf("no commit returned from server")
return nil
}
func CommandCreate(cmd *cobra.Command, args []string) {
mustLoadConfig(cmd)
now := time.Now()
// find the directory
d := ""
switch len(args) {
case 0:
d = "."
case 1:
d = args[0]
default:
cmd.Help()
return
}
dir, err := filepath.Abs(d)
if err != nil {
log.Fatalf("error finding directory %q: %v", d, err)
}
// find the problem.cfg file
for {
path := filepath.Join(dir, ProblemConfigName)
if _, err := os.Stat(path); err != nil {
if os.IsNotExist(err) {
// try moving up a directory
old := dir
dir = filepath.Dir(dir)
if dir == old {
log.Fatalf("unable to find %s in %s or an ancestor directory", ProblemConfigName, d)
}
log.Printf("could not find %s in %s, trying %s", ProblemConfigName, old, dir)
continue
}
log.Fatalf("error searching for %s in %s: %v", ProblemConfigName, dir, err)
}
break
}
// parse problem.cfg
cfg := struct {
Problem struct {
Unique string
Note string
Type string
Tag []string
Option []string
}
Step map[string]*struct {
Note string
Weight float64
}
}{}
configPath := filepath.Join(dir, ProblemConfigName)
fmt.Printf("reading %s\n", configPath)
err = gcfg.ReadFileInto(&cfg, configPath)
if err != nil {
log.Fatalf("failed to parse %s: %v", configPath, err)
}
// create problem object
problem := &Problem{
Unique: cfg.Problem.Unique,
Note: cfg.Problem.Note,
ProblemType: cfg.Problem.Type,
Tags: cfg.Problem.Tag,
Options: cfg.Problem.Option,
CreatedAt: now,
UpdatedAt: now,
}
// start forming the problem bundle
unsigned := &ProblemBundle{
Problem: problem,
}
// check if this is an existing problem
existing := []*Problem{}
mustGetObject("/problems", map[string]string{"unique": problem.Unique}, &existing)
switch len(existing) {
case 0:
// new problem
if cmd.Flag("update").Value.String() == "true" {
log.Fatalf("you specified --update, but no existing problem with unique ID %q was found", problem.Unique)
}
// make sure the problem set with this unique name is free as well
existingSets := []*ProblemSet{}
mustGetObject("/problem_sets", map[string]string{"unique": problem.Unique}, &existingSets)
if len(existingSets) > 1 {
log.Fatalf("error: server found multiple problem sets with matching unique ID %q", problem.Unique)
}
if len(existingSets) != 0 {
log.Printf("problem set %d already exists with unique ID %q", existingSets[0].ID, existingSets[0].Unique)
log.Fatalf(" this would prevent creating a problem set containing just this problem with matching id")
}
log.Printf("this problem is new--no existing problem has the same unique ID")
case 1:
// update to existing problem
if cmd.Flag("update").Value.String() == "false" {
log.Fatalf("you did not specify --update, but a problem already exists with unique ID %q", problem.Unique)
}
log.Printf("unique ID is %s", problem.Unique)
log.Printf(" this is an update of problem %d (%q)", existing[0].ID, existing[0].Note)
problem.ID = existing[0].ID
problem.CreatedAt = existing[0].CreatedAt
default:
// server does not know what "unique" means
log.Fatalf("error: server found multiple problems with matching unique ID %q", problem.Unique)
}
// generate steps
whitelist := make(map[string]bool)
for i := int64(1); cfg.Step[strconv.FormatInt(i, 10)] != nil; i++ {
log.Printf("gathering step %d", i)
s := cfg.Step[strconv.FormatInt(i, 10)]
step := &ProblemStep{
Step: i,
Note: s.Note,
Weight: s.Weight,
Files: make(map[string]string),
}
commit := &Commit{
Step: i,
Action: "confirm",
Note: "author solution submitted via grind",
Files: make(map[string]string),
CreatedAt: now,
UpdatedAt: now,
}
// read files
starter, solution, root := make(map[string]string), make(map[string]string), make(map[string]string)
stepdir := filepath.Join(dir, strconv.FormatInt(i, 10))
err := filepath.Walk(stepdir, func(path string, info os.FileInfo, err error) error {
if err != nil {
log.Fatalf("walk error for %s: %v", path, err)
}
if info.IsDir() {
return nil
}
relpath, err := filepath.Rel(stepdir, path)
if err != nil {
log.Fatalf("error finding relative path of %s: %v", path, err)
}
// load the file and add it to the appropriate place
contents, err := ioutil.ReadFile(path)
if err != nil {
log.Fatalf("error reading %s: %v", relpath, err)
}
// pick out solution/starter files
reldir, relfile := filepath.Split(relpath)
if reldir == "_solution/" && relfile != "" {
solution[relfile] = string(contents)
} else if reldir == "_starter/" && relfile != "" {
starter[relfile] = string(contents)
} else if reldir == "" && relfile != "" {
root[relfile] = string(contents)
} else {
step.Files[relpath] = string(contents)
}
return nil
})
if err != nil {
log.Fatalf("walk error for %s: %v", stepdir, err)
}
// find starter files and solution files
if len(solution) > 0 && len(starter) > 0 && len(root) > 0 {
log.Fatalf("found files in _starter, _solution, and root directory; unsure how to proceed")
}
if len(solution) > 0 {
// explicit solution
} else if len(root) > 0 {
// files in root directory must be the solution
solution = root
root = nil
} else {
log.Fatalf("no solution files found in _solution or root directory; problem must have a solution")
}
if len(starter) == 0 && root != nil {
starter = root
}
// copy the starter files into the step
for name, contents := range starter {
step.Files[name] = contents
// if the file exists as a starter in this or earlier steps, it can be part of the solution
whitelist[name] = true
}
// copy the solution files into the commit
for name, contents := range solution {
if whitelist[name] {
commit.Files[name] = contents
} else {
log.Printf("Warning: skipping solution file %q", name)
log.Printf(" because it is not in the starter file set of this or any previous step")
}
}
unsigned.ProblemSteps = append(unsigned.ProblemSteps, step)
unsigned.Commits = append(unsigned.Commits, commit)
log.Printf(" found %d problem definition file%s and %d solution file%s", len(step.Files), plural(len(step.Files)), len(commit.Files), plural(len(commit.Files)))
}
if len(unsigned.ProblemSteps) != len(cfg.Step) {
log.Fatalf("expected to find %d step%s, but only found %d", len(cfg.Step), plural(len(cfg.Step)), len(unsigned.ProblemSteps))
}
// get user ID
user := new(User)
mustGetObject("/users/me", nil, user)
// get the request validated and signed
signed := new(ProblemBundle)
mustPostObject("/problem_bundles/unconfirmed", nil, unsigned, signed)
// validate the commits one at a time
for n := 0; n < len(signed.ProblemSteps); n++ {
log.Printf("validating solution for step %d", n+1)
unvalidated := &CommitBundle{
Problem: signed.Problem,
ProblemSteps: signed.ProblemSteps,
ProblemSignature: signed.ProblemSignature,
Commit: signed.Commits[n],
CommitSignature: signed.CommitSignatures[n],
}
validated := mustConfirmCommitBundle(user.ID, unvalidated, nil)
log.Printf(" finished validating solution")
if validated.Commit.ReportCard == nil || validated.Commit.Score != 1.0 || !validated.Commit.ReportCard.Passed {
log.Printf(" solution for step %d failed: %s", n+1, validated.Commit.ReportCard.Note)
// play the transcript
for _, event := range validated.Commit.Transcript {
switch event.Event {
case "exec":
color.Cyan("$ %s\n", strings.Join(event.ExecCommand, " "))
case "stdin":
color.Yellow("%s", event.StreamData)
case "stdout":
color.White("%s", event.StreamData)
case "stderr":
color.Red("%s", event.StreamData)
case "exit":
color.Cyan("%s\n", event.ExitStatus)
case "error":
color.Red("Error: %s\n", event.Error)
}
}
log.Fatalf("please fix solution and try again")
}
signed.Problem = validated.Problem
signed.ProblemSteps = validated.ProblemSteps
signed.ProblemSignature = validated.ProblemSignature
signed.Commits[n] = validated.Commit
signed.CommitSignatures[n] = validated.CommitSignature
}
log.Printf("problem and solution confirmed successfully")
// save the problem
final := new(ProblemBundle)
if signed.Problem.ID == 0 {
mustPostObject("/problem_bundles/confirmed", nil, signed, final)
} else {
mustPutObject(fmt.Sprintf("/problem_bundles/%d", signed.Problem.ID), nil, signed, final)
}
log.Printf("problem %q saved and ready to use", final.Problem.Unique)
if signed.Problem.ID == 0 {
// create a matching problem set
// pause for a bit since the database seems to need to catch up
time.Sleep(time.Second)
// create a problem set with just this problem and the same unique name
psBundle := &ProblemSetBundle{
ProblemSet: &ProblemSet{
Unique: final.Problem.Unique,
Note: "set for single problem " + final.Problem.Unique + "\n" + final.Problem.Note,
Tags: final.Problem.Tags,
CreatedAt: now,
UpdatedAt: now,
},
ProblemIDs: []int64{final.Problem.ID},
Weights: []float64{1.0},
}
finalPSBundle := new(ProblemSetBundle)
mustPostObject("/problem_set_bundles", nil, psBundle, finalPSBundle)
log.Printf("problem set %q created and ready to use for this problem", finalPSBundle.ProblemSet.Unique)
}
}
func (j GroupsImport) Run() {
start := time.Now()
fmt.Println("------------------------------")
fmt.Printf("| Starting import at %s\n", start.Format("2006-01-02 15:04:05.000"))
db := controllers.Gdb
file := OpenExcelFile("/home/kolan/Documents/ONR_FITIS.xlsx")
fmt.Println("------------------------------")
rowCount := 0
successCount := 0
alreadyExist := 0
wrongYear := 0
wrongNumberOfStudents := 0
for _, row := range file.Sheets[1].Rows[10:] {
group := models.Groups{}
re := regexp.MustCompile("[^\\S]+")
re2 := regexp.MustCompile("([^\x00-\x7F])i")
re3 := regexp.MustCompile("i([^\x00-\x7F])")
groupYear, err := strconv.Atoi(strings.TrimSpace(row.Cells[7].Value))
if err != nil {
group.Year = 0
wrongYear += 1
} else {
group.Year = groupYear
}
groupName := strings.TrimSpace(row.Cells[27].Value)
groupName = re.ReplaceAllString(groupName, " ")
groupName = re2.ReplaceAllString(groupName, "${1}і")
groupName = re3.ReplaceAllString(groupName, "i${1}")
group.Name = groupName
groupStudents, err := strconv.Atoi(strings.TrimSpace(row.Cells[11].Value))
if err != nil {
group.NumOfStudents = 0
wrongNumberOfStudents += 1
} else {
group.NumOfStudents = groupStudents
}
group.FacultyId = 1
result := db.Where(&group).First(&group)
if result.Error != nil {
if result.RecordNotFound() {
db.NewRecord(group)
err = db.Save(&group).Error
if err != nil {
if regexp.MustCompile(`^Error 1062:`).MatchString(err.Error()) == true {
alreadyExist += 1
color.Cyan("| \"%s\" - is already exist", groupName)
} else {
color.Red("| Saving error for - \"%s\"", groupName)
}
} else {
successCount += 1
color.Green("| \"%s\" - added", groupName)
}
}
} else {
alreadyExist += 1
color.Cyan("| \"%s\" - is already exist", groupName)
}
rowCount += 1
}
elapsed := time.Since(start)
fmt.Println("------------------------------")
fmt.Printf("| Import finished at %s\n", time.Now().Format("2006-01-02 15:04:05.000"))
fmt.Printf("| Importing time %s\n", elapsed)
color.Blue("------------------------------------------")
color.Blue("| Groups in file : %d", rowCount)
color.Blue("| Imported groups : %d", successCount)
color.Blue("| Already exist groups : %d", alreadyExist)
color.Blue("| Groups without year : %d", wrongYear)
color.Blue("| Groups without students count : %d", wrongNumberOfStudents)
color.Blue("------------------------------------------")
}
func main() {
// Server Options
opts := Options{}
var showVersion bool
var debugAndTrace bool
var configFile string
// Parse flags
flag.IntVar(&opts.Port, "port", 0, "Port to listen on.")
flag.IntVar(&opts.Port, "p", 0, "Port to listen on.")
flag.StringVar(&opts.Host, "host", "", "Network host to listen on.")
flag.StringVar(&opts.Host, "h", "", "Network host to listen on.")
flag.StringVar(&opts.Host, "net", "", "Network host to listen on.")
flag.BoolVar(&opts.Debug, "D", false, "Enable Debug logging.")
flag.BoolVar(&opts.Debug, "debug", false, "Enable Debug logging.")
flag.BoolVar(&opts.Trace, "V", false, "Enable Trace logging.")
flag.BoolVar(&opts.Trace, "trace", false, "Enable Trace logging.")
flag.BoolVar(&debugAndTrace, "DV", false, "Enable Debug and Trace logging.")
flag.BoolVar(&opts.Logtime, "T", true, "Timestamp log entries.")
flag.BoolVar(&opts.Logtime, "logtime", true, "Timestamp log entries.")
flag.StringVar(&opts.Username, "user", "", "Username required for connection.")
flag.StringVar(&opts.Password, "pass", "", "Password required for connection.")
flag.StringVar(&opts.Authorization, "auth", "", "Authorization token required for connection.")
flag.StringVar(&configFile, "c", "", "Configuration file.")
flag.StringVar(&configFile, "config", "", "Configuration file.")
flag.StringVar(&opts.PidFile, "P", "", "File to store process pid.")
flag.StringVar(&opts.PidFile, "pid", "", "File to store process pid.")
flag.StringVar(&opts.LogFile, "l", "", "File to store logging output.")
flag.StringVar(&opts.LogFile, "log", "", "File to store logging output.")
flag.BoolVar(&showVersion, "version", false, "Print version information.")
flag.BoolVar(&showVersion, "v", false, "Print version information.")
flag.Usage = usage
flag.Parse()
// Show version and exit
if showVersion {
PrintServerAndExit()
}
// One flag can set multiple options.
if debugAndTrace {
opts.Trace, opts.Debug = true, true
}
// Process args looking for non-flag options,
// 'version' and 'help' only for now
for _, arg := range flag.Args() {
switch strings.ToLower(arg) {
case "version":
PrintServerAndExit()
case "help":
usage()
}
}
// Parse config
var cfg config.Config
cfg.Parse()
// MongoDb
db.InitMongo(cfg.MongoHost, cfg.MongoPort, cfg.MongoDatabase)
// MQTT
mqc := new(clients.MqttConn)
//Sub to everything comming on all channels of all devices
mqc.MqttSub()
// Serve HTTP
go servers.HttpServer(cfg)
// Print banner
color.Cyan(banner)
color.Cyan("Magic happens on port " + strconv.Itoa(cfg.HttpPort))
/** Keep main() runnig */
runtime.Goexit()
}
