func DumpFiles() {
var ret = dna.StringArray{}
for _, table := range GetAlbumTables().Concat(GetSongTables()).Concat(GetVideoTables()) {
ret.Push(`"` + table + `"`)
}
dna.Log(ret.Join(","))
}
// RecoverSQLLogError re-executes failed sql queries in sql error log file from specified path.
// It returns the number of failed -reexec queries, and new failed
// queries will be written to the file
//
// The format of error file is:
// Error description - $$$error$$$SQL_QUERY$$$error$$$
// Therefore only get statements enclosed by special `$$$error$$$`
func RecoverSQLLogError(sqlErrFilePath dna.String, db *sqlpg.DB) dna.Int {
var errCount = 0
var errStrings = dna.StringArray{}
b, err := ioutil.ReadFile(sqlErrFilePath.String())
if err != nil {
panic(err)
}
data := dna.String(string(b))
// dna.Log("\n", data.Length())
sqlArr := data.FindAllString(`(?mis)\$\$\$error\$\$\$.+?\$\$\$error\$\$\$`, -1)
// dna.Log("\nTOTAL SQL STATEMENTS FOUND:", sqlArr.Length())
for _, val := range sqlArr {
sqlStmtArr := val.FindAllStringSubmatch(`(?mis)\$\$\$error\$\$\$(.+?)\$\$\$error\$\$\$`, -1)
if len(sqlStmtArr) > 0 {
_, err := db.Exec(sqlStmtArr[0][1].String())
if err != nil {
if dna.String(err.Error()).Contains(`duplicate key value violates unique constraint`) == false {
errCount += 1
errStrings.Push("$$$error$$$" + sqlStmtArr[0][1] + "$$$error$$$")
}
}
}
}
if errCount == 0 {
err = ioutil.WriteFile(sqlErrFilePath.String(), []byte{}, 0644)
} else {
err = ioutil.WriteFile(sqlErrFilePath.String(), []byte(errStrings.Join("\n").String()), 0644)
}
if err != nil {
panic(err)
}
return dna.Int(errCount)
}
// GetFormats returns proper formatStr for a song or a video.
// If it is a song, IsSong will be set to true. Otherwise, it will set to false false.
func GetFormats(urls dna.StringArray) (formatStr dna.String, IsSong dna.Bool) {
switch getType(urls.Join("")) {
case "mp3", "m4a", "flac":
formatStr, IsSong = getStringifiedSongUrls(urls), IS_SONG
return formatStr, IsSong
case "mp4", "flv":
formatStr, IsSong = getStringifiedVideoUrls(urls), IS_VIDEO
return formatStr, IsSong
default:
panic("Wrong type. Cannot indentify song or video")
}
return "", false
}
// GetUpdateStatement returns an update statement from specified snake-case columns.
// If columns's names are not found, it will return an error.
// It updates some fields from a struct.
//
// * tbName : A name of update table.
// * structValue : A struct-typed value being scanned. Its fields have to be dna basic type or time.Time.
// * conditionColumn : A snake-case column name in the condition, usually it's an id
// * columns : A list of args of column names in the table being updated.
// * Returns an update statement.
func GetUpdateStatement(tbName dna.String, structValue interface{}, conditionColumn dna.String, columns ...dna.String) (dna.String, error) {
if reflect.TypeOf(structValue).Kind() != reflect.Ptr {
panic("StructValue has to be pointer")
if reflect.TypeOf(structValue).Elem().Kind() != reflect.Struct {
panic("StructValue has to be struct type")
}
}
query := "UPDATE " + tbName + " SET\n"
result := dna.StringArray{}
for _, column := range columns {
result.Push(getPairValue(structValue, column))
}
conditionRet := "\nWHERE " + getPairValue(structValue, conditionColumn) + ";"
return query + result.Join(",\n") + conditionRet, nil
}
func getStringifiedSongUrls(urls dna.StringArray) dna.String {
var baseLink = dna.String("")
songUrls := []SongUrl{}
urls.ForEach(func(val dna.String, idx dna.Int) {
// dna.Log(val)
// Finding bitrate
switch {
case val.Match(`128kbps`) == true:
songUrl := getSongUrl(val, "128kbps")
baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
songUrls = append(songUrls, *songUrl)
case val.Match(`320kbps`) == true:
songUrl := getSongUrl(val, "320kbps")
baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
songUrls = append(songUrls, *songUrl)
case val.Match(`32kbps`) == true:
songUrl := getSongUrl(val, "32kbps")
baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
songUrls = append(songUrls, *songUrl)
case val.Match(`500kbps`) == true:
songUrl := getSongUrl(val, "500kbps")
songUrl.Link = baseLink + "m4a/file-name.m4a"
songUrls = append(songUrls, *songUrl)
case val.Match(`Lossless`) == true:
songUrl := getSongUrl(val, "Lossless")
songUrl.Link = baseLink + "flac/file-name.flac"
songUrls = append(songUrls, *songUrl)
}
})
// http://data.chiasenhac.com/downloads/1184/2/1183017-cfc5f7df/flac/file-name.flac
// replace the link 500kps and lossless with available link, apply for registered user only
// and reduce the link length
var ret = dna.StringArray{}
for _, songUrl := range songUrls {
var br dna.String
if songUrl.Bitrate == "Lossless" {
br = "1411"
} else {
br = songUrl.Bitrate.Replace("kbps", "")
}
t := `"(` + songUrl.Link + "," + songUrl.Type + "," + songUrl.Size.ToString() + "," + br + `)"`
ret.Push(t)
}
// dna.Log(`{` + ret.Join(",") + `}`)
return `{` + ret.Join(",") + `}`
}
// DumpHashTables uses psql command. Set its path to ENV
func DumpHashTables() {
var commands = dna.StringArray{}
for index, table := range GetAlbumTables().Concat(GetSongTables()).Concat(GetVideoTables()) {
shortForm := table.Replace("songs", "").Replace("albums", "").Replace("videos", "")
stmt := dna.Sprintf(`SELECT dna_hash(title,artists), ROW(%v,id) from %v`, ToSiteid(shortForm), table)
command := dna.Sprintf(`psql -c 'COPY (%v) TO STDOUT'`, stmt)
switch {
case table.Match("albums") == true:
command += " >> data/hash_albums.log"
case table.Match("songs") == true:
command += " >> data/hash_songs.log"
case table.Match("videos") == true:
command += " >> data/hash_videos.log"
}
commands.Push(`INTERNAL_TIME=$(date +%s)`)
commands.Push(dna.Sprintf(`printf "%-3v:Extracting hashids from %-12v "`, index+1, table+"..."))
commands.Push(command)
commands.Push(`echo "Completed in $(($(date +%s) - $INTERNAL_TIME)) seconds!"`)
commands.Push("#--------------------------")
}
dna.Log(commands.Join("\n"))
}
func getStringifiedVideoUrls(urls dna.StringArray) dna.String {
var baseLink = dna.String("")
videoUrls := []VideoUrl{}
urls.ForEach(func(val dna.String, idx dna.Int) {
// dna.Log(val)
// Finding bitrate
switch {
case val.Match(`MV 360p`) == true:
songUrl := getVideoUrl(val, "360p")
baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
videoUrls = append(videoUrls, *songUrl)
case val.Match(`MV 480p`) == true:
songUrl := getVideoUrl(val, "480p")
baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
videoUrls = append(videoUrls, *songUrl)
case val.Match(`MV 180p`) == true:
songUrl := getVideoUrl(val, "180p")
baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
videoUrls = append(videoUrls, *songUrl)
case val.Match(`HD 720p`) == true:
songUrl := getVideoUrl(val, "720p")
songUrl.Link = baseLink + "m4a/file-name.mp4"
videoUrls = append(videoUrls, *songUrl)
case val.Match(`HD 1080p`) == true:
songUrl := getVideoUrl(val, "1080p")
songUrl.Link = baseLink + "flac/file-name.mp4"
videoUrls = append(videoUrls, *songUrl)
}
})
var ret = dna.StringArray{}
for _, videoUrl := range videoUrls {
t := `"(` + videoUrl.Link + "," + videoUrl.Type + "," + videoUrl.Size.ToString() + "," + videoUrl.Resolution.Replace("p", "") + `)"`
ret.Push(t)
}
// dna.Log(`{` + ret.Join(",") + `}`)
return `{` + ret.Join(",") + `}`
}
// GetInsertIgnoreStatement returns insert ignore statement from a struct. If input value is not struct, it will panic.
// * tbName : A name of table in database you want to insert
// * structValue : A struct-typed value. The struct's fields has to be dna basic types (dna.Int, dna.String..) or time.Time
// * primaryColumn : A name of primary column. If a row is duplicate, it will be discarded.
// * primaryValue : A value of row needed to be compared.
// * isPrintable: A param determines where to print the pretty result statement
// * Return an insert statement
// Notice: Insert statement uses Dollar-quoted String Constants with special tag "binhdna".
// So string or array is contained between $binhdna$ symbols.
// Therefore no need to escape values.
func GetInsertIgnoreStatement(tbName dna.String, structValue interface{}, primaryColumn dna.String, primaryValue interface{}, isPrintable dna.Bool) dna.String {
var realKind string
var columnNames, columnValues dna.StringArray
tempintslice := []int{0}
var ielements int
var kind string = reflect.TypeOf(structValue).Kind().String()
if kind == "ptr" {
realKind = reflect.TypeOf(structValue).Elem().Kind().String()
} else {
realKind = reflect.TypeOf(structValue).Kind().String()
}
if realKind != "struct" {
panic("Param has to be struct")
}
if kind == "ptr" {
ielements = reflect.TypeOf(structValue).Elem().NumField()
} else {
ielements = reflect.TypeOf(structValue).NumField()
}
for i := 0; i < ielements; i++ {
tempintslice[0] = i
if kind == "ptr" {
f := reflect.TypeOf(structValue).Elem().FieldByIndex(tempintslice)
v := reflect.ValueOf(structValue).Elem().FieldByIndex(tempintslice)
clName, clValue := getColumn(f, v.Interface())
columnNames.Push(clName)
columnValues.Push(clValue)
} else {
f := reflect.TypeOf(structValue).FieldByIndex(tempintslice)
v := reflect.ValueOf(structValue).FieldByIndex(tempintslice)
clName, clValue := getColumn(f, v.Interface())
columnNames.Push(clName)
columnValues.Push(clValue)
}
}
condStr := dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v)", tbName, primaryValue)
if isPrintable == true {
return "INSERT INTO " + tbName + "\n(" + columnNames.Join(",") + ")\n" + " SELECT " + columnValues.Join(",\n") + " \n" + condStr + ";" + " \n"
} else {
return "INSERT INTO " + tbName + "(" + columnNames.Join(",") + ")" + " SELECT " + columnValues.Join(",") + " " + condStr + ";"
}
}
// GetInsertStatement returns insert statement from a struct. If input value is not struct, it will panic.
// * tbName : A name of table in database you want to insert
// * structValue : A struct-typed value. The struct's fields has to be dna basic types (dna.Int, dna.String..) or time.Time
// * isPrintable: A param determines where to print the pretty result statement
// * Return an insert statement
// Notice: Insert statement uses Dollar-quoted String Constants with special tag "binhdna".
// So string or array is contained between $binhdna$ symbols.
// Therefore no need to escape values.
func GetInsertStatement(tbName dna.String, structValue interface{}, isPrintable dna.Bool) dna.String {
var realKind string
var columnNames, columnValues dna.StringArray
tempintslice := []int{0}
var ielements int
var kind string = reflect.TypeOf(structValue).Kind().String()
if kind == "ptr" {
realKind = reflect.TypeOf(structValue).Elem().Kind().String()
} else {
realKind = reflect.TypeOf(structValue).Kind().String()
}
if realKind != "struct" {
panic("Param has to be struct")
}
if kind == "ptr" {
ielements = reflect.TypeOf(structValue).Elem().NumField()
} else {
ielements = reflect.TypeOf(structValue).NumField()
}
for i := 0; i < ielements; i++ {
tempintslice[0] = i
if kind == "ptr" {
f := reflect.TypeOf(structValue).Elem().FieldByIndex(tempintslice)
v := reflect.ValueOf(structValue).Elem().FieldByIndex(tempintslice)
clName, clValue := getColumn(f, v.Interface())
columnNames.Push(clName)
columnValues.Push(clValue)
} else {
f := reflect.TypeOf(structValue).FieldByIndex(tempintslice)
v := reflect.ValueOf(structValue).FieldByIndex(tempintslice)
clName, clValue := getColumn(f, v.Interface())
columnNames.Push(clName)
columnValues.Push(clValue)
}
}
if isPrintable == true {
return "INSERT INTO " + tbName + "\n(" + columnNames.Join(",") + ")\n" + "VALUES (\n" + columnValues.Join(",\n") + "\n);"
} else {
return "INSERT INTO " + tbName + "(" + columnNames.Join(",") + ")" + " VALUES (" + columnValues.Join(",") + ");"
}
}
func (q Question) ToRecord() []string {
if q.Cat.Subject == "non-verbal-reasoning/questions-and-answers" {
imagesArr := q.QuestionContent.FindAllString(`<img.+?src=.+?>`, -1)
images := dna.StringArray{}
for _, image := range imagesArr {
images.Push("http://indiabix.com" + image.GetTagAttributes("src"))
}
return []string{"Non Verbal Reasoning", q.Cat.Name.String(), q.Description.String(), images.Join(",").String(), q.QuestionContent.String(), q.OptionA.String(), q.OptionB.String(), q.OptionC.String(), q.OptionD.String(), q.OptionE.String(), q.Answer.String(), q.Explaination.String(), q.ExerciseNo.ToString().String()}
} else {
return []string{q.Cat.Subject.String(), q.Cat.Name.String(), q.Description.String(), q.QuestionContent.String(), q.OptionA.String(), q.OptionB.String(), q.OptionC.String(), q.OptionD.String(), q.OptionE.String(), q.Answer.String(), q.Explaination.String(), q.ExerciseNo.ToString().String()}
}
}