// RecoverSQLLogError re-executes failed sql queries in sql error log file from specified path.
// It returns the number of failed -reexec queries, and new failed
// queries will be written to the file
//
// The format of error file is:
// Error description - $$$error$$$SQL_QUERY$$$error$$$
// Therefore only get statements enclosed by special `$$$error$$$`
func RecoverSQLLogError(sqlErrFilePath dna.String, db *sqlpg.DB) dna.Int {
var errCount = 0
var errStrings = dna.StringArray{}
b, err := ioutil.ReadFile(sqlErrFilePath.String())
if err != nil {
panic(err)
}
data := dna.String(string(b))
// dna.Log("\n", data.Length())
sqlArr := data.FindAllString(`(?mis)\$\$\$error\$\$\$.+?\$\$\$error\$\$\$`, -1)
// dna.Log("\nTOTAL SQL STATEMENTS FOUND:", sqlArr.Length())
for _, val := range sqlArr {
sqlStmtArr := val.FindAllStringSubmatch(`(?mis)\$\$\$error\$\$\$(.+?)\$\$\$error\$\$\$`, -1)
if len(sqlStmtArr) > 0 {
_, err := db.Exec(sqlStmtArr[0][1].String())
if err != nil {
if dna.String(err.Error()).Contains(`duplicate key value violates unique constraint`) == false {
errCount += 1
errStrings.Push("$$$error$$$" + sqlStmtArr[0][1] + "$$$error$$$")
}
}
}
}
if errCount == 0 {
err = ioutil.WriteFile(sqlErrFilePath.String(), []byte{}, 0644)
} else {
err = ioutil.WriteFile(sqlErrFilePath.String(), []byte(errStrings.Join("\n").String()), 0644)
}
if err != nil {
panic(err)
}
return dna.Int(errCount)
}
func DumpFiles() {
var ret = dna.StringArray{}
for _, table := range GetAlbumTables().Concat(GetSongTables()).Concat(GetVideoTables()) {
ret.Push(`"` + table + `"`)
}
dna.Log(ret.Join(","))
}
// This function will decode a cipher string into id.
func Decrypt(cipher dna.String) dna.Int {
arr := dna.StringArray{cipher[0:2], cipher[2:3], cipher[3:4], cipher[4:6], cipher[6:7], cipher[7:8], cipher[8:10]}.Filter(
func(value dna.String, index dna.Int) dna.Bool {
return value != ""
})
return dna.IntArray(arr.Map(func(v dna.String, i dna.Int) dna.Int {
return ns[6-i].IndexOf(v)
}).([]dna.Int)).Join("").ToInt()
}
func GetArtistTables() dna.StringArray {
var tblArr = dna.StringArray{}
for key, _ := range RefTables {
if RefTables.HasArtist(key) {
tblArr.Push(key + "artists")
}
}
return tblArr
}
func GetSongTables() dna.StringArray {
var tblArr = dna.StringArray{}
for key, _ := range RefTables {
if RefTables.HasSong(key) {
tblArr.Push(key + "songs")
}
}
return tblArr
}
func GetVideoTables() dna.StringArray {
var tblArr = dna.StringArray{}
for key, _ := range RefTables {
if RefTables.HasVideo(key) {
tblArr.Push(key + "videos")
}
}
return tblArr
}
func GetAlbumTables() dna.StringArray {
var tblArr = dna.StringArray{}
for key, _ := range RefTables {
if RefTables.HasAlbum(key) {
tblArr.Push(key + "albums")
}
}
return tblArr
}
func (apiSong *APIFullSong) Save(db *sqlpg.DB) error {
var queries = dna.StringArray{}
var err error
// Getting artist queries
artists := apiSong.ToArtists()
for _, artist := range artists {
queries.Push(sqlpg.GetInsertIgnoreStatement(sqlpg.GetTableName(artist), artist, "id", artist.Id, false))
}
// Getting album query
album := apiSong.ToAlbum()
queries.Push(sqlpg.GetInsertIgnoreStatement(sqlpg.GetTableName(album), album, "id", album.Id, false))
// Getting song query
song := apiSong.ToSong()
queries.Push(sqlpg.GetInsertStatement(sqlpg.GetTableName(song), song, false))
for _, query := range queries {
_, err = db.Exec(query.String())
}
if err != nil {
errQueries := dna.StringArray(queries.Map(func(val dna.String, idx dna.Int) dna.String {
return "$$$error$$$" + val + "$$$error$$$"
}).([]dna.String))
return errors.New(err.Error() + errQueries.Join("\n").String())
} else {
return nil
}
}
// splitAndTruncateArtists splits stringarray by the key "feat:"
// and filter only string elements not equal to empty string.
func splitAndTruncateArtists(artists dna.StringArray) dna.StringArray {
return dna.StringArray(artists.SplitWithRegexp("feat:").Map(func(val dna.String, idx dna.Int) dna.String {
return val.Trim()
}).([]dna.String)).Filter(func(val dna.String, idx dna.Int) dna.Bool {
if val != "" {
return true
} else {
return false
}
})
}
// GetFormats returns proper formatStr for a song or a video.
// If it is a song, IsSong will be set to true. Otherwise, it will set to false false.
func GetFormats(urls dna.StringArray) (formatStr dna.String, IsSong dna.Bool) {
switch getType(urls.Join("")) {
case "mp3", "m4a", "flac":
formatStr, IsSong = getStringifiedSongUrls(urls), IS_SONG
return formatStr, IsSong
case "mp4", "flv":
formatStr, IsSong = getStringifiedVideoUrls(urls), IS_VIDEO
return formatStr, IsSong
default:
panic("Wrong type. Cannot indentify song or video")
}
return "", false
}
func (q Question) ToRecord() []string {
if q.Cat.Subject == "non-verbal-reasoning/questions-and-answers" {
imagesArr := q.QuestionContent.FindAllString(`<img.+?src=.+?>`, -1)
images := dna.StringArray{}
for _, image := range imagesArr {
images.Push("http://indiabix.com" + image.GetTagAttributes("src"))
}
return []string{"Non Verbal Reasoning", q.Cat.Name.String(), q.Description.String(), images.Join(",").String(), q.QuestionContent.String(), q.OptionA.String(), q.OptionB.String(), q.OptionC.String(), q.OptionD.String(), q.OptionE.String(), q.Answer.String(), q.Explaination.String(), q.ExerciseNo.ToString().String()}
} else {
return []string{q.Cat.Subject.String(), q.Cat.Name.String(), q.Description.String(), q.QuestionContent.String(), q.OptionA.String(), q.OptionB.String(), q.OptionC.String(), q.OptionD.String(), q.OptionE.String(), q.Answer.String(), q.Explaination.String(), q.ExerciseNo.ToString().String()}
}
}
// GetUpdateStatement returns an update statement from specified snake-case columns.
// If columns's names are not found, it will return an error.
// It updates some fields from a struct.
//
// * tbName : A name of update table.
// * structValue : A struct-typed value being scanned. Its fields have to be dna basic type or time.Time.
// * conditionColumn : A snake-case column name in the condition, usually it's an id
// * columns : A list of args of column names in the table being updated.
// * Returns an update statement.
func GetUpdateStatement(tbName dna.String, structValue interface{}, conditionColumn dna.String, columns ...dna.String) (dna.String, error) {
if reflect.TypeOf(structValue).Kind() != reflect.Ptr {
panic("StructValue has to be pointer")
if reflect.TypeOf(structValue).Elem().Kind() != reflect.Struct {
panic("StructValue has to be struct type")
}
}
query := "UPDATE " + tbName + " SET\n"
result := dna.StringArray{}
for _, column := range columns {
result.Push(getPairValue(structValue, column))
}
conditionRet := "\nWHERE " + getPairValue(structValue, conditionColumn) + ";"
return query + result.Join(",\n") + conditionRet, nil
}
func convertCategoryToStringArray(catStr dna.String) dna.StringArray {
var cats []Category
var ret = dna.StringArray{}
err := json.Unmarshal([]byte(string(catStr)), &cats)
if err != nil {
dna.Log(catStr)
dna.Log(err.Error())
panic("Invalid category string input")
} else {
for _, cat := range cats {
ret.Push(cat.Name)
}
}
return ret
}
func (sf *APISongFreaksTrack) ToSong() (*Song, error) {
if sferr := sf.HasError(); sferr != nil {
return nil, sferr
}
song := NewSong()
song.Id = sf.Track.Id
song.TrackGroupId = sf.Track.TrackGroupId
song.AMG = sf.Track.AMG
song.UrlSlug = sf.Track.UrlSlug
song.IsInstrumental = sf.Track.IsInstrumental
song.Viewable = sf.Track.Viewable
song.Duration = utils.ToSeconds(sf.Track.Duration)
song.Lyricid = sf.Track.LyricId
song.HasLrc = sf.Track.HasLrc
song.TrackNumber = sf.Track.TrackNumber
song.DiscNumber = sf.Track.DiscNumber
song.Title = sf.Track.Title
song.Rating = dna.IntArray{sf.Track.Rating.AverageRating.ToInt(), sf.Track.Rating.UserRating, sf.Track.Rating.TotalRatings}
song.Albumid = sf.Track.Album.Id
artistIds := dna.IntArray{}
artists := dna.StringArray{}
for _, artist := range sf.Track.Artists {
artistIds.Push(artist.Id)
artists.Push(artist.Name)
}
song.Artistids = artistIds
song.Artists = artists
if sf.Track.Lrc.Lines != nil && len(sf.Track.Lrc.Lines) > 0 {
lines, err := json.Marshal(sf.Track.Lrc.Lines)
if err == nil {
song.Lrc = dna.String(string(lines))
}
}
song.Link = sf.Track.Link
song.Lyric = sf.Track.Lyrics
if song.Lyric != "" {
song.HasLyric = true
}
song.Copyright = sf.Track.Copyright
song.Writer = sf.Track.Writer
song.SubmittedLyric = sf.Track.SubmittedLyric
song.Checktime = time.Now()
return song, nil
}
func getTSGM(data *dna.String, kind dna.String) dna.String {
var itemArr dna.StringArray
switch kind {
case "genres":
itemArr = data.FindAllString(`(?mis)<h4>Genre</h4>(.+?)</div>`, 1)
case "styles":
itemArr = data.FindAllString(`(?mis)<h4>Styles</h4>(.+?)</div>`, 1)
case "moods":
itemArr = data.FindAllString(`(?mis)<h4>Album Moods</h4>(.+?)</div>`, 1)
case "themes":
itemArr = data.FindAllString(`(?mis)<h4>Themes</h4>(.+?)</div>`, 1)
default:
panic("Wrong kind!!!")
}
if itemArr.Length() > 0 {
catArr := itemArr[0].FindAllString(`<a href=.+?</a>`, -1)
categories := catArr.Map(func(val dna.String, idx dna.Int) Category {
var idArr []dna.StringArray
var id dna.Int = 0
name := val.RemoveHtmlTags("")
if kind == "moods" {
idArr = val.FindAllStringSubmatch(`xa([0-9]+)`, 1)
} else {
idArr = val.FindAllStringSubmatch(`ma([0-9]+)`, 1)
}
if len(idArr) > 0 {
id = idArr[0][1].ToInt()
}
return Category{id, name}
}).([]Category)
if len(categories) > 0 {
bCat, merr := json.Marshal(categories)
if merr != nil {
return "[]"
} else {
return dna.String(string(bCat))
}
} else {
return "[]"
}
} else {
return "[]"
}
}
// ExecQueriesInTransaction executes queries in a transaction.
// If one statement fails, the whole queries cannot commit.
//
// The returned error is nil if there is no error.
// If an error occurs, each statement will be enclosed in
// format $$$error$$$.
// $$$error$$$ Your Custom Query $$$error$$$
//
// This function is seen in songfreaks and allmusic sites.
func ExecQueriesInTransaction(db *DB, queries *dna.StringArray) error {
var err error
globalSqlTransactoNo += 1
// tx, err := db.Begin()
// if err != nil {
// dna.Log("Transaction No:" + dna.Sprintf("%v", globalSqlTransactoNo).String() + err.Error() + " Could not create transaction\n")
// }
for idx, query := range *queries {
_, err = db.Exec(query.String())
// _, err = tx.Exec(query.String())
if err != nil {
dna.Log(dna.Sprintf("DNAError: Query series No: %v - %v - %v - %v\n", dna.Sprintf("%v", globalSqlTransactoNo), idx, err.Error(), "Could not execute the statement"))
}
// stmt, err := tx.Prepare(query.String())
// if err != nil {
// dna.Log(dna.Sprintf("DNAError Transaction No: %v - %v - %v - %v \n", dna.Sprintf("%v", globalSqlTransactoNo), idx, err.Error(), "Could not prepare"))
// } else {
// _, err = stmt.Exec()
// if err != nil {
// dna.Log(dna.Sprintf("DNAError: Transaction No: %v - %v - %v - %v\n", dna.Sprintf("%v", globalSqlTransactoNo), idx, err.Error(), "Could not execute the prepared statement"))
// }
// err = stmt.Close()
// if err != nil {
// dna.Log("Transaction No:" + dna.Sprintf("%v", globalSqlTransactoNo).String() + err.Error() + " Could not close\n")
// }
// }
}
// err = tx.Commit()
// if err != nil {
// dna.Log("Transaction No:" + dna.Sprintf("%v", globalSqlTransactoNo).String() + err.Error() + " Could not commit transaction\n")
// }
if err != nil {
errQueries := dna.StringArray(queries.Map(func(val dna.String, idx dna.Int) dna.String {
return "Transaction No:" + dna.Sprintf("%v", globalSqlTransactoNo) + " $$$error$$$" + val + "$$$error$$$"
}).([]dna.String))
return errors.New(err.Error() + errQueries.Join("\n").String())
} else {
return nil
}
}
func getGenresFromDesc(desc dna.String) dna.StringArray {
var ret dna.StringArray
genres := desc.FindAllString(`(?i)genres?(\s+)?:?.+`, 1)
// "Released:" found in album id: 836258
if genres.Length() > 0 {
ret = dna.StringArray(genres[0].ReplaceWithRegexp(`(?mis)genres?(\s+)?:?`, "").ReplaceWithRegexp(`\.?\s*Released:.+`, "").Trim().Split(",").Map(func(val dna.String, idx dna.Int) dna.String {
return val.ReplaceWithRegexp(":", "").Trim()
}).([]dna.String))
if ret.Length() == 1 {
arr := dna.StringArray{}
if ret[0].FindAllString(`(?mis)K-Pop`, 1).Length() > 0 {
arr.Push("Korean Pop")
arr.Push(ret[0].ReplaceWithRegexp(`(?mis)\(?K-Pop\)?`, "").Trim())
ret = arr
}
}
}
return ret.SplitWithRegexp(` > `).SplitWithRegexp(`/`)
}
func (apiaward *APIAward) ToAward() *Award {
award := NewAward()
award.Id = apiaward.Id
award.Title = apiaward.Title
award.Section = ""
award.Year = apiaward.Year
award.Chart = apiaward.Chart
award.Peak = apiaward.Peak
award.Type = apiaward.Type
award.Prize = apiaward.Award
winnerids := dna.IntArray{}
winners := dna.StringArray{}
for _, winner := range apiaward.Winners {
winnerids.Push(winner.Id)
winners.Push(winner.Name)
}
award.Winnerids = winnerids
award.Winners = winners
return award
}
// SelectMissingKeys accepts a table name as an input and a list of keys as a source.
// It returns a new list of keys that does not exist in the destination table
//
// * tblName : a table name
// * srcKeys : a source keys
// * db : a pointer to connected databased
// * Returns a new list of keys which are not from the specified table
//
// Notice: Only applied to a table having a column named "key".
// The column has to be indexed to ensure good performance
//
// The format of sql statement is:
// with dna (key) as (values ('43f3HhhU6DGV'),('uFfgQhKbwAfN'),('RvFDlckJB5QU'),('uIF7rwd5wo4p'),('Kveukbhre1ry'),('oJ1lzAlKwJX6'),('43f3HhhU6DGV'),('uFfgQhKbwAfN'),('hfhtyMdywMau'),('PpZuccjYqy1b'))
// select key from dna where key not in
// (select key from nctalbums where key in ('43f3HhhU6DGV','uFfgQhKbwAfN','RvFDlckJB5QU','uIF7rwd5wo4p','Kveukbhre1ry','oJ1lzAlKwJX6','43f3HhhU6DGV','uFfgQhKbwAfN','hfhtyMdywMau','PpZuccjYqy1b'))
func SelectMissingKeys(tblName dna.String, srcKeys *dna.StringArray, db *sqlpg.DB) (*dna.StringArray, error) {
if srcKeys.Length() > 0 {
val := dna.StringArray(srcKeys.Map(func(val dna.String, idx dna.Int) dna.String {
return `('` + val + `')`
}).([]dna.String))
val1 := dna.StringArray(srcKeys.Map(func(val dna.String, idx dna.Int) dna.String {
return `'` + val + `'`
}).([]dna.String))
selectStmt := "with dna (key) as (values " + val.Join(",") + ") \n"
selectStmt += "select key from dna where key not in \n(select key from " + tblName + " where key in (" + val1.Join(",") + "))"
keys := &[]dna.String{}
err := db.Select(keys, selectStmt)
switch {
case err != nil:
return nil, err
case err == nil && keys != nil:
slice := dna.StringArray(*keys)
return &slice, nil
case err == nil && keys == nil:
return &dna.StringArray{}, nil
default:
panic("Default case triggered. Case is not expected. Cannot select non existed keys")
}
} else {
return nil, errors.New("Empty input array")
}
}
func (apisong *APISong) ToSong() *Song {
song := NewSong()
song.Id = apisong.Id
song.Title = apisong.Title.DecodeHTML()
song.Duration = apisong.Duration
artistids := dna.IntArray{}
artists := dna.StringArray{}
for _, artist := range apisong.Artists {
artistids.Push(artist.Id)
artists.Push(artist.Name)
}
song.Artistids = artistids
song.Artists = artists
composerids := dna.IntArray{}
composers := dna.StringArray{}
for _, composer := range apisong.Composers {
composerids.Push(composer.Id)
composers.Push(composer.Name)
}
song.Composerids = composerids
song.Composers = composers
song.Checktime = time.Now()
return song
}
func (apiSong *APIFullSong) FillSong(song *Song) {
song.Id = apiSong.Track.Id
song.Title = apiSong.Track.Title
artists := dna.StringArray{}
for _, artist := range apiSong.Track.Artists {
artists.Push(artist.Name)
}
song.Artists = artists
artistids := dna.IntArray{}
for _, artist := range apiSong.Track.Artists {
artistids.Push(artist.Id)
}
song.Artistids = artistids
song.Albumid = apiSong.Track.Album.Id
song.AlbumTitle = apiSong.Track.Album.Title
song.Duration = utils.ToSeconds(apiSong.Track.Duration)
song.Instrumental = apiSong.Track.Instrumental
song.Viewable = apiSong.Track.Viewable
song.HasLrc = apiSong.Track.HasLrc
song.LrcVerified = apiSong.Track.LrcVerified
song.Lyricid = apiSong.Track.Lyricid
song.Lyric = apiSong.Track.Lyrics
song.Copyright = apiSong.Track.Copyright
song.Writers = apiSong.Track.Writer
// Mon Jan 2 15:04:05 MST 2006
if apiSong.Track.LastUpdate != "" {
lastUpdate, err := time.Parse("2006-01-02 15:04:05", apiSong.Track.LastUpdate.String())
if err == nil {
song.DateUpdated = lastUpdate
} else {
dna.Log(err.Error(), " Song ID:", song.Id, " GOT:", apiSong.Track.LastUpdate, "\n\n")
}
}
}
func getStringifiedSongUrls(urls dna.StringArray) dna.String {
var baseLink = dna.String("")
songUrls := []SongUrl{}
urls.ForEach(func(val dna.String, idx dna.Int) {
// dna.Log(val)
// Finding bitrate
switch {
case val.Match(`128kbps`) == true:
songUrl := getSongUrl(val, "128kbps")
baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
songUrls = append(songUrls, *songUrl)
case val.Match(`320kbps`) == true:
songUrl := getSongUrl(val, "320kbps")
baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
songUrls = append(songUrls, *songUrl)
case val.Match(`32kbps`) == true:
songUrl := getSongUrl(val, "32kbps")
baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
songUrls = append(songUrls, *songUrl)
case val.Match(`500kbps`) == true:
songUrl := getSongUrl(val, "500kbps")
songUrl.Link = baseLink + "m4a/file-name.m4a"
songUrls = append(songUrls, *songUrl)
case val.Match(`Lossless`) == true:
songUrl := getSongUrl(val, "Lossless")
songUrl.Link = baseLink + "flac/file-name.flac"
songUrls = append(songUrls, *songUrl)
}
})
// http://data.chiasenhac.com/downloads/1184/2/1183017-cfc5f7df/flac/file-name.flac
// replace the link 500kps and lossless with available link, apply for registered user only
// and reduce the link length
var ret = dna.StringArray{}
for _, songUrl := range songUrls {
var br dna.String
if songUrl.Bitrate == "Lossless" {
br = "1411"
} else {
br = songUrl.Bitrate.Replace("kbps", "")
}
t := `"(` + songUrl.Link + "," + songUrl.Type + "," + songUrl.Size.ToString() + "," + br + `)"`
ret.Push(t)
}
// dna.Log(`{` + ret.Join(",") + `}`)
return `{` + ret.Join(",") + `}`
}
func (sf *APISongFreaksTrack) Save(db *sqlpg.DB) error {
var queries = dna.StringArray{}
album, artist, song, videos := sf.Convert()
if artist != nil {
queries.Push(getInsertStmt(artist, dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v)", getTableName(artist), artist.Id)))
}
if album != nil {
queries.Push(getInsertStmt(album, dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v)", getTableName(album), album.Id)))
}
if song != nil {
queries.Push(getInsertStmt(song, dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v)", getTableName(song), song.Id)))
}
for _, video := range videos {
queries.Push(getInsertStmt(video, dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE songid=%v AND youtube_id='%v')", getTableName(video), video.Songid, video.YoutubeId)))
}
// dna.Log(queries.Join("\n"))
// dna.Log(queries.Join("\n"))
// return nil
return sqlpg.ExecQueriesInTransaction(db, &queries)
}
func getStringifiedVideoUrls(urls dna.StringArray) dna.String {
var baseLink = dna.String("")
videoUrls := []VideoUrl{}
urls.ForEach(func(val dna.String, idx dna.Int) {
// dna.Log(val)
// Finding bitrate
switch {
case val.Match(`MV 360p`) == true:
songUrl := getVideoUrl(val, "360p")
baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
videoUrls = append(videoUrls, *songUrl)
case val.Match(`MV 480p`) == true:
songUrl := getVideoUrl(val, "480p")
baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
videoUrls = append(videoUrls, *songUrl)
case val.Match(`MV 180p`) == true:
songUrl := getVideoUrl(val, "180p")
baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
videoUrls = append(videoUrls, *songUrl)
case val.Match(`HD 720p`) == true:
songUrl := getVideoUrl(val, "720p")
songUrl.Link = baseLink + "m4a/file-name.mp4"
videoUrls = append(videoUrls, *songUrl)
case val.Match(`HD 1080p`) == true:
songUrl := getVideoUrl(val, "1080p")
songUrl.Link = baseLink + "flac/file-name.mp4"
videoUrls = append(videoUrls, *songUrl)
}
})
var ret = dna.StringArray{}
for _, videoUrl := range videoUrls {
t := `"(` + videoUrl.Link + "," + videoUrl.Type + "," + videoUrl.Size.ToString() + "," + videoUrl.Resolution.Replace("p", "") + `)"`
ret.Push(t)
}
// dna.Log(`{` + ret.Join(",") + `}`)
return `{` + ret.Join(",") + `}`
}
func getInsertStatement(tbName dna.String, structValue interface{}, condStr dna.String, isPrintable dna.Bool) dna.String {
var realKind string
var columnNames, columnValues dna.StringArray
tempintslice := []int{0}
var ielements int
var kind string = reflect.TypeOf(structValue).Kind().String()
if kind == "ptr" {
realKind = reflect.TypeOf(structValue).Elem().Kind().String()
} else {
realKind = reflect.TypeOf(structValue).Kind().String()
}
if realKind != "struct" {
panic("Param has to be struct")
}
if kind == "ptr" {
ielements = reflect.TypeOf(structValue).Elem().NumField()
} else {
ielements = reflect.TypeOf(structValue).NumField()
}
for i := 0; i < ielements; i++ {
tempintslice[0] = i
if kind == "ptr" {
f := reflect.TypeOf(structValue).Elem().FieldByIndex(tempintslice)
v := reflect.ValueOf(structValue).Elem().FieldByIndex(tempintslice)
clName, clValue := getColumn(f, v.Interface())
columnNames.Push(clName)
columnValues.Push(clValue)
} else {
f := reflect.TypeOf(structValue).FieldByIndex(tempintslice)
v := reflect.ValueOf(structValue).FieldByIndex(tempintslice)
clName, clValue := getColumn(f, v.Interface())
columnNames.Push(clName)
columnValues.Push(clValue)
}
}
if isPrintable == true {
return "INSERT INTO " + tbName + "\n(" + columnNames.Join(",") + ")\n" + " SELECT " + columnValues.Join(",\n") + " \n" + condStr
} else {
return "INSERT INTO " + tbName + "(" + columnNames.Join(",") + ")" + " SELECT " + columnValues.Join(",") + " " + condStr
}
}
func getQuestions(cat Category, isFirstPage dna.Bool, pageUrl dna.String, pageLinks *dna.StringArray) <-chan Questions {
channel := make(chan Questions)
questions := Questions{}
go func() {
var url dna.String
if isFirstPage == true {
url = cat.Url
} else {
url = pageUrl
}
link := "http://www.indiabix.com" + url
// dna.Log(link)
result, err := http.Get(link)
if err == nil {
data := &result.Data
if isFirstPage == true {
// getting page links if the first page found
pageLinkArr := data.FindAllString(`<p class="ib-pager">.+</p>`, 1)
if pageLinkArr.Length() > 0 {
links := pageLinkArr[0].FindAllString(`<a href.+?>`, -1)
for _, link := range links {
pageLinks.Push(link.GetTagAttributes("href"))
}
}
}
// Getting direction for the questions
descriptionArr := data.FindAllString(`(?mis)id="divDirectionText".+?<div class="bix-div-container">`, 1)
description := dna.String("")
if descriptionArr.Length() > 0 {
description = descriptionArr[0].ReplaceWithRegexp(`<div class="bix-div-container">$`, "").Trim().ReplaceWithRegexp(`<td style="padding-left:5px" valign="top">$`, "").ReplaceWithRegexp(`id="divDirectionText">?`, "").Trim().ReplaceWithRegexp(`<tr>$`, "").Trim().ReplaceWithRegexp(`</div></div></td></tr>$`, "").Trim().ReplaceWithRegexp(`^<p>`, "").Trim().ReplaceWithRegexp(`</p>$`, "").Trim()
}
questionTables := data.FindAllString(`(?mis)<table class="bix-tbl-container".+?<hr />`, -1)
// if questionTables.Length() > 0 {
// dna.Log("# of questions: ", questionTables.Length())
// }
for _, questionTable := range questionTables {
question := NewQuestion()
question.Cat = cat
question.Description = description
// Getting question content
questionContentArr := questionTable.FindAllString(`(?mis)<td class="bix-td-qtxt".+?<td class="bix-td-miscell"`, 1)
if questionContentArr.Length() > 0 {
question.QuestionContent = questionContentArr[0].ReplaceWithRegexp(`^<td class="bix-td-qtxt.+?>`, "").ReplaceWithRegexp(`<td class="bix-td-miscell"`, "").Trim().ReplaceWithRegexp(`^<p>`, "").Trim().ReplaceWithRegexp(`<tr>$`, "").Trim().ReplaceWithRegexp(`</tr>$`, "").Trim().ReplaceWithRegexp(`</p></td>$`, "").Trim()
}
// Getting question answers
// answersArr is table html contains a list of options for the question
optionArr := questionTable.FindAllString(`(?mis)<td class="bix-td-option.+tdOptionDt.+?<div class="bix-div-answer"`, -1)
getOptionsForQuestion(optionArr[0], question)
// Getting explaination for the question
explantionArr := questionTable.FindAllString(`(?mis)Explanation:</b></span></p>.+?<div class="bix-div-workspace"`, 1)
if explantionArr.Length() > 0 {
question.Explaination = explantionArr[0].ReplaceWithRegexp(`^Explanation:</b></span></p>`, "").Trim().ReplaceWithRegexp(`^<p>`, "").Trim().ReplaceWithRegexp(`<div class="bix-div-workspace"$`, "").Trim().ReplaceWithRegexp(`</div>$`, "").Trim().ReplaceWithRegexp(`</div>$`, "").Trim().ReplaceWithRegexp(`</p>$`, "").Trim()
}
questions = append(questions, *question)
}
// Getting answers for all question
// If length of answers != length of questions, raise an error
answersArr := data.FindAllString(`<input id="hdnAjaxImageCacheKey".+`, -1)
var answers = dna.StringArray{}
if answersArr.Length() > 0 {
answer := answersArr[0].GetTagAttributes("value")
// $('input' + '#' + 'hdn' + 'Ajax' + 'Image' + 'Cache' + 'Key').val().substr(18).split('').reverse().join('').substr(17).toUpperCase().split('');
tmp := answer.Substring(18, answer.Length()).Split("").Reverse().Join("")
answers = tmp.Substring(17, tmp.Length()).ToUpperCase().Split("")
}
if answers.Length().ToPrimitiveValue() != len(questions) {
panic("Answer length differs from question length - " + cat.ToString())
}
for idx, ans := range answers {
questions[idx].Answer = ans
}
}
channel <- questions
}()
return channel
}
// transformCats turns genre name into multiple genres if applicable
func transformCats(cats dna.StringArray) dna.StringArray {
ret := dna.StringArray{}
vnSongs := dna.StringArray{"Nhạc Trẻ", "Nhạc Trữ Tình", "Nhạc Cách Mạng", "Nhạc Trịnh", "Nhạc Tiền Chiến", "Nhạc Dân Tộc", "Nhạc Thiếu Nhi", "Rock Việt", "Nhạc Hải Ngoại", "Nhạc Quê Hương", "Rap Việt - Hiphop"}
for _, cat := range cats {
if vnSongs.IndexOf(cat) > -1 {
ret.Push("Nhạc Việt Nam")
}
switch cat {
case "Pop/Ballad":
ret.Push("Pop")
ret.Push("Ballad")
case "Dance/Electronic":
ret.Push("Dance")
ret.Push("Electronic")
case "Nhạc Spa | Thư Giãn":
ret.Push("Nhạc Spa")
ret.Push("Thư Giãn")
case "Hiphop/Rap":
ret.Push("Hiphop")
ret.Push("Rap")
case "Nhạc Bà Bầu & Baby":
ret.Push("Nhạc Bà Bầu")
ret.Push("Nhạc Baby")
case "Rap Việt - Hiphop":
ret.Push("Rap Việt")
ret.Push("Hiphop")
case "Radio - Cảm Xúc":
ret.Push("Radio")
ret.Push("Cảm Xúc")
default:
ret.Push(cat)
}
}
return ret
}
func (album *APIAlbum) Save(db *sqlpg.DB) error {
var queries = dna.StringArray{}
awards, credits, discos, releases, songs, mainAlbum := album.Convert()
queries.Push(getInsertStmt(mainAlbum, dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v)", getTableName(mainAlbum), album.Id)))
for _, award := range awards {
// dna.Log(award)
queries.Push(getInsertStmt(&award, dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v AND albumid=%v AND chart=$binhdna$%v$binhdna$ AND prize=$binhdna$%v$binhdna$)", getTableName(&award), award.Id, award.Albumid, award.Chart, award.Prize)))
}
for _, credit := range credits {
queries.Push(getInsertStmt(&credit, dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v and albumid=%v)", getTableName(&credit), credit.Id, credit.Albumid)))
}
for _, disco := range discos {
queries.Push(getInsertStmt(&disco, dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v and artistid=%v)", getTableName(&disco), disco.Id, disco.Artistid)).Replace("amdiscographys", "amdiscographies"))
}
for _, release := range releases {
queries.Push(getInsertStmt(&release, dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v and albumid=%v)", getTableName(&release), release.Id, release.Albumid)))
}
for _, song := range songs {
queries.Push(getInsertStmt(&song, dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v and albumid=%v)", getTableName(&song), song.Id, song.Albumid)))
}
// dna.Log(queries.Join("\n"))
// return db.Update(mainAlbum, "id", "coverart")
return sqlpg.ExecQueriesInTransaction(db, &queries)
// return db.InsertIgnore(album)
}
func getCategory(songs *[]*Song, genre Genre, page dna.Int) <-chan bool {
channel := make(chan bool, 1)
go func() {
link := "http://nhacso.net/bai-hat-theo-the-loai-" + genre.Id.ToString() + "/joke-link-2-" + page.ToString() + ".html"
// dna.Log(link)
result, err := http.Get(link)
if err == nil {
data := &result.Data
// transform string {"2":[0,3,5,7,9,11,13,15,29],"10":[1,2,4,6,8,]}
// to map[dna.Int]dna.Int{20:2, 28:2, 4:10, 12:10} Ex: map[29] = 2
temp := data.FindAllStringSubmatch(`getCategory.+'(\{.+\})'`, -1)
mapping := map[dna.Int]dna.Int{}
if len(temp) > 0 && temp[0].Length() > 0 {
vals := temp[0][1].FindAllString(`"[0-9]+":\[[0-9,]+?\]`, -1)
if vals.Length() > 0 {
for _, val := range vals {
target := val.FindAllStringSubmatch(`"(.+)"`, -1)[0][1].ToInt()
arr := val.FindAllStringSubmatch(`\[(.+)\]`, -1)[0][1].Split(",").ToIntArray()
for _, itm := range arr {
mapping[itm] = target
}
}
}
}
// Finding cat id for each song. cats var is 2-dimentional array.
// Each index of it represents the correspondent song, its value is the categories the song belongs to
catStrings := data.FindAllString(`Thể loại :.+`, -1)
cats := []dna.IntArray{}
for _, val := range catStrings {
tagids := dna.IntArray{}
tmp := val.FindAllStringSubmatch(`cate_tag_song_([0-9]+)`, -1)
if len(tmp) > 0 {
for _, el := range tmp {
tagids.Push(el[1].ToInt())
}
}
cats = append(cats, tagids)
}
// Log(cats)
// get songids
temps := data.FindAllStringSubmatch(`play" id="blocksongtag_([0-9]+)`, -1)
songids := dna.IntArray{}
if len(temps) > 0 {
for _, val := range temps {
songids.Push(val[1].ToInt())
}
}
tmpsongs := &[]*Song{}
for idx, songid := range songids {
song := NewSong()
song.Id = songid
category := dna.StringArray{}
for _, val := range cats[idx] {
if mapping[val] > 0 && mapping[val] < CatTags.Length() {
if CatTags[mapping[val]] != "" {
category.Push(CatTags[mapping[val]])
}
} else {
mess := dna.Sprintf("WRONG INDEX AT CATTAGS: %v %v %v - %v", mapping[val], genre, page, link)
panic(mess.String())
}
}
category.Push(genre.Name)
song.Category = transformCats(category.Unique()).Unique()
*tmpsongs = append(*tmpsongs, song)
}
*songs = *tmpsongs
}
channel <- true
}()
return channel
}
// DumpHashTables uses psql command. Set its path to ENV
func DumpHashTables() {
var commands = dna.StringArray{}
for index, table := range GetAlbumTables().Concat(GetSongTables()).Concat(GetVideoTables()) {
shortForm := table.Replace("songs", "").Replace("albums", "").Replace("videos", "")
stmt := dna.Sprintf(`SELECT dna_hash(title,artists), ROW(%v,id) from %v`, ToSiteid(shortForm), table)
command := dna.Sprintf(`psql -c 'COPY (%v) TO STDOUT'`, stmt)
switch {
case table.Match("albums") == true:
command += " >> data/hash_albums.log"
case table.Match("songs") == true:
command += " >> data/hash_songs.log"
case table.Match("videos") == true:
command += " >> data/hash_videos.log"
}
commands.Push(`INTERNAL_TIME=$(date +%s)`)
commands.Push(dna.Sprintf(`printf "%-3v:Extracting hashids from %-12v "`, index+1, table+"..."))
commands.Push(command)
commands.Push(`echo "Completed in $(($(date +%s) - $INTERNAL_TIME)) seconds!"`)
commands.Push("#--------------------------")
}
dna.Log(commands.Join("\n"))
}