func DumpFiles() { var ret = dna.StringArray{} for _, table := range GetAlbumTables().Concat(GetSongTables()).Concat(GetVideoTables()) { ret.Push(`"` + table + `"`) } dna.Log(ret.Join(",")) }
// RecoverSQLLogError re-executes failed sql queries in sql error log file from specified path. // It returns the number of failed -reexec queries, and new failed // queries will be written to the file // // The format of error file is: // Error description - $$$error$$$SQL_QUERY$$$error$$$ // Therefore only get statements enclosed by special `$$$error$$$` func RecoverSQLLogError(sqlErrFilePath dna.String, db *sqlpg.DB) dna.Int { var errCount = 0 var errStrings = dna.StringArray{} b, err := ioutil.ReadFile(sqlErrFilePath.String()) if err != nil { panic(err) } data := dna.String(string(b)) // dna.Log("\n", data.Length()) sqlArr := data.FindAllString(`(?mis)\$\$\$error\$\$\$.+?\$\$\$error\$\$\$`, -1) // dna.Log("\nTOTAL SQL STATEMENTS FOUND:", sqlArr.Length()) for _, val := range sqlArr { sqlStmtArr := val.FindAllStringSubmatch(`(?mis)\$\$\$error\$\$\$(.+?)\$\$\$error\$\$\$`, -1) if len(sqlStmtArr) > 0 { _, err := db.Exec(sqlStmtArr[0][1].String()) if err != nil { if dna.String(err.Error()).Contains(`duplicate key value violates unique constraint`) == false { errCount += 1 errStrings.Push("$$$error$$$" + sqlStmtArr[0][1] + "$$$error$$$") } } } } if errCount == 0 { err = ioutil.WriteFile(sqlErrFilePath.String(), []byte{}, 0644) } else { err = ioutil.WriteFile(sqlErrFilePath.String(), []byte(errStrings.Join("\n").String()), 0644) } if err != nil { panic(err) } return dna.Int(errCount) }
// GetFormats returns proper formatStr for a song or a video. // If it is a song, IsSong will be set to true. Otherwise, it will set to false false. func GetFormats(urls dna.StringArray) (formatStr dna.String, IsSong dna.Bool) { switch getType(urls.Join("")) { case "mp3", "m4a", "flac": formatStr, IsSong = getStringifiedSongUrls(urls), IS_SONG return formatStr, IsSong case "mp4", "flv": formatStr, IsSong = getStringifiedVideoUrls(urls), IS_VIDEO return formatStr, IsSong default: panic("Wrong type. Cannot indentify song or video") } return "", false }
// GetUpdateStatement returns an update statement from specified snake-case columns. // If columns's names are not found, it will return an error. // It updates some fields from a struct. // // * tbName : A name of update table. // * structValue : A struct-typed value being scanned. Its fields have to be dna basic type or time.Time. // * conditionColumn : A snake-case column name in the condition, usually it's an id // * columns : A list of args of column names in the table being updated. // * Returns an update statement. func GetUpdateStatement(tbName dna.String, structValue interface{}, conditionColumn dna.String, columns ...dna.String) (dna.String, error) { if reflect.TypeOf(structValue).Kind() != reflect.Ptr { panic("StructValue has to be pointer") if reflect.TypeOf(structValue).Elem().Kind() != reflect.Struct { panic("StructValue has to be struct type") } } query := "UPDATE " + tbName + " SET\n" result := dna.StringArray{} for _, column := range columns { result.Push(getPairValue(structValue, column)) } conditionRet := "\nWHERE " + getPairValue(structValue, conditionColumn) + ";" return query + result.Join(",\n") + conditionRet, nil }
func getStringifiedSongUrls(urls dna.StringArray) dna.String { var baseLink = dna.String("") songUrls := []SongUrl{} urls.ForEach(func(val dna.String, idx dna.Int) { // dna.Log(val) // Finding bitrate switch { case val.Match(`128kbps`) == true: songUrl := getSongUrl(val, "128kbps") baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "") songUrls = append(songUrls, *songUrl) case val.Match(`320kbps`) == true: songUrl := getSongUrl(val, "320kbps") baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "") songUrls = append(songUrls, *songUrl) case val.Match(`32kbps`) == true: songUrl := getSongUrl(val, "32kbps") baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "") songUrls = append(songUrls, *songUrl) case val.Match(`500kbps`) == true: songUrl := getSongUrl(val, "500kbps") songUrl.Link = baseLink + "m4a/file-name.m4a" songUrls = append(songUrls, *songUrl) case val.Match(`Lossless`) == true: songUrl := getSongUrl(val, "Lossless") songUrl.Link = baseLink + "flac/file-name.flac" songUrls = append(songUrls, *songUrl) } }) // http://data.chiasenhac.com/downloads/1184/2/1183017-cfc5f7df/flac/file-name.flac // replace the link 500kps and lossless with available link, apply for registered user only // and reduce the link length var ret = dna.StringArray{} for _, songUrl := range songUrls { var br dna.String if songUrl.Bitrate == "Lossless" { br = "1411" } else { br = songUrl.Bitrate.Replace("kbps", "") } t := `"(` + songUrl.Link + "," + songUrl.Type + "," + songUrl.Size.ToString() + "," + br + `)"` ret.Push(t) } // dna.Log(`{` + ret.Join(",") + `}`) return `{` + ret.Join(",") + `}` }
// DumpHashTables uses psql command. Set its path to ENV func DumpHashTables() { var commands = dna.StringArray{} for index, table := range GetAlbumTables().Concat(GetSongTables()).Concat(GetVideoTables()) { shortForm := table.Replace("songs", "").Replace("albums", "").Replace("videos", "") stmt := dna.Sprintf(`SELECT dna_hash(title,artists), ROW(%v,id) from %v`, ToSiteid(shortForm), table) command := dna.Sprintf(`psql -c 'COPY (%v) TO STDOUT'`, stmt) switch { case table.Match("albums") == true: command += " >> data/hash_albums.log" case table.Match("songs") == true: command += " >> data/hash_songs.log" case table.Match("videos") == true: command += " >> data/hash_videos.log" } commands.Push(`INTERNAL_TIME=$(date +%s)`) commands.Push(dna.Sprintf(`printf "%-3v:Extracting hashids from %-12v "`, index+1, table+"...")) commands.Push(command) commands.Push(`echo "Completed in $(($(date +%s) - $INTERNAL_TIME)) seconds!"`) commands.Push("#--------------------------") } dna.Log(commands.Join("\n")) }
func getStringifiedVideoUrls(urls dna.StringArray) dna.String { var baseLink = dna.String("") videoUrls := []VideoUrl{} urls.ForEach(func(val dna.String, idx dna.Int) { // dna.Log(val) // Finding bitrate switch { case val.Match(`MV 360p`) == true: songUrl := getVideoUrl(val, "360p") baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "") videoUrls = append(videoUrls, *songUrl) case val.Match(`MV 480p`) == true: songUrl := getVideoUrl(val, "480p") baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "") videoUrls = append(videoUrls, *songUrl) case val.Match(`MV 180p`) == true: songUrl := getVideoUrl(val, "180p") baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "") videoUrls = append(videoUrls, *songUrl) case val.Match(`HD 720p`) == true: songUrl := getVideoUrl(val, "720p") songUrl.Link = baseLink + "m4a/file-name.mp4" videoUrls = append(videoUrls, *songUrl) case val.Match(`HD 1080p`) == true: songUrl := getVideoUrl(val, "1080p") songUrl.Link = baseLink + "flac/file-name.mp4" videoUrls = append(videoUrls, *songUrl) } }) var ret = dna.StringArray{} for _, videoUrl := range videoUrls { t := `"(` + videoUrl.Link + "," + videoUrl.Type + "," + videoUrl.Size.ToString() + "," + videoUrl.Resolution.Replace("p", "") + `)"` ret.Push(t) } // dna.Log(`{` + ret.Join(",") + `}`) return `{` + ret.Join(",") + `}` }
// GetInsertIgnoreStatement returns insert ignore statement from a struct. If input value is not struct, it will panic. // * tbName : A name of table in database you want to insert // * structValue : A struct-typed value. The struct's fields has to be dna basic types (dna.Int, dna.String..) or time.Time // * primaryColumn : A name of primary column. If a row is duplicate, it will be discarded. // * primaryValue : A value of row needed to be compared. // * isPrintable: A param determines where to print the pretty result statement // * Return an insert statement // Notice: Insert statement uses Dollar-quoted String Constants with special tag "binhdna". // So string or array is contained between $binhdna$ symbols. // Therefore no need to escape values. func GetInsertIgnoreStatement(tbName dna.String, structValue interface{}, primaryColumn dna.String, primaryValue interface{}, isPrintable dna.Bool) dna.String { var realKind string var columnNames, columnValues dna.StringArray tempintslice := []int{0} var ielements int var kind string = reflect.TypeOf(structValue).Kind().String() if kind == "ptr" { realKind = reflect.TypeOf(structValue).Elem().Kind().String() } else { realKind = reflect.TypeOf(structValue).Kind().String() } if realKind != "struct" { panic("Param has to be struct") } if kind == "ptr" { ielements = reflect.TypeOf(structValue).Elem().NumField() } else { ielements = reflect.TypeOf(structValue).NumField() } for i := 0; i < ielements; i++ { tempintslice[0] = i if kind == "ptr" { f := reflect.TypeOf(structValue).Elem().FieldByIndex(tempintslice) v := reflect.ValueOf(structValue).Elem().FieldByIndex(tempintslice) clName, clValue := getColumn(f, v.Interface()) columnNames.Push(clName) columnValues.Push(clValue) } else { f := reflect.TypeOf(structValue).FieldByIndex(tempintslice) v := reflect.ValueOf(structValue).FieldByIndex(tempintslice) clName, clValue := getColumn(f, v.Interface()) columnNames.Push(clName) columnValues.Push(clValue) } } condStr := dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v)", tbName, primaryValue) if isPrintable == true { return "INSERT INTO " + tbName + "\n(" + columnNames.Join(",") + ")\n" + " SELECT " + columnValues.Join(",\n") + " \n" + condStr + ";" + " \n" } else { return "INSERT INTO " + tbName + "(" + columnNames.Join(",") + ")" + " SELECT " + columnValues.Join(",") + " " + condStr + ";" } }
// GetInsertStatement returns insert statement from a struct. If input value is not struct, it will panic. // * tbName : A name of table in database you want to insert // * structValue : A struct-typed value. The struct's fields has to be dna basic types (dna.Int, dna.String..) or time.Time // * isPrintable: A param determines where to print the pretty result statement // * Return an insert statement // Notice: Insert statement uses Dollar-quoted String Constants with special tag "binhdna". // So string or array is contained between $binhdna$ symbols. // Therefore no need to escape values. func GetInsertStatement(tbName dna.String, structValue interface{}, isPrintable dna.Bool) dna.String { var realKind string var columnNames, columnValues dna.StringArray tempintslice := []int{0} var ielements int var kind string = reflect.TypeOf(structValue).Kind().String() if kind == "ptr" { realKind = reflect.TypeOf(structValue).Elem().Kind().String() } else { realKind = reflect.TypeOf(structValue).Kind().String() } if realKind != "struct" { panic("Param has to be struct") } if kind == "ptr" { ielements = reflect.TypeOf(structValue).Elem().NumField() } else { ielements = reflect.TypeOf(structValue).NumField() } for i := 0; i < ielements; i++ { tempintslice[0] = i if kind == "ptr" { f := reflect.TypeOf(structValue).Elem().FieldByIndex(tempintslice) v := reflect.ValueOf(structValue).Elem().FieldByIndex(tempintslice) clName, clValue := getColumn(f, v.Interface()) columnNames.Push(clName) columnValues.Push(clValue) } else { f := reflect.TypeOf(structValue).FieldByIndex(tempintslice) v := reflect.ValueOf(structValue).FieldByIndex(tempintslice) clName, clValue := getColumn(f, v.Interface()) columnNames.Push(clName) columnValues.Push(clValue) } } if isPrintable == true { return "INSERT INTO " + tbName + "\n(" + columnNames.Join(",") + ")\n" + "VALUES (\n" + columnValues.Join(",\n") + "\n);" } else { return "INSERT INTO " + tbName + "(" + columnNames.Join(",") + ")" + " VALUES (" + columnValues.Join(",") + ");" } }
func (q Question) ToRecord() []string { if q.Cat.Subject == "non-verbal-reasoning/questions-and-answers" { imagesArr := q.QuestionContent.FindAllString(`<img.+?src=.+?>`, -1) images := dna.StringArray{} for _, image := range imagesArr { images.Push("http://indiabix.com" + image.GetTagAttributes("src")) } return []string{"Non Verbal Reasoning", q.Cat.Name.String(), q.Description.String(), images.Join(",").String(), q.QuestionContent.String(), q.OptionA.String(), q.OptionB.String(), q.OptionC.String(), q.OptionD.String(), q.OptionE.String(), q.Answer.String(), q.Explaination.String(), q.ExerciseNo.ToString().String()} } else { return []string{q.Cat.Subject.String(), q.Cat.Name.String(), q.Description.String(), q.QuestionContent.String(), q.OptionA.String(), q.OptionB.String(), q.OptionC.String(), q.OptionD.String(), q.OptionE.String(), q.Answer.String(), q.Explaination.String(), q.ExerciseNo.ToString().String()} } }