func getOptionsForQuestion(tableOptions dna.String, question *Question) {
// dna.Log(tableOptions)
options := tableOptions.Replace(`<div class="bix-div-answer"`, "").Trim().Split("tdOptionDt")
if options.Length() > 6 {
panic("Answer options greater than 5 at: " + question.Cat.ToString())
}
for idx, option := range options {
if idx > 0 {
option := option.ReplaceWithRegexp(`^.+?>`, ``).ReplaceWithRegexp(`(?mis)<td class="bix-td-option".+$`, "").ReplaceWithRegexp(`</td>$`, "").ReplaceWithRegexp(`</td></tr><tr>$`, "").ReplaceWithRegexp(`</td></tr></table><input type="hidden".+$`, "").Trim()
switch idx {
case 1:
question.OptionA = option
case 2:
question.OptionB = option
case 3:
question.OptionC = option
case 4:
question.OptionD = option
case 5:
question.OptionE = option
}
}
}
}
func getGracenoteSongLyric(artist, title dna.String, song *Song) {
link := "http://lyrics.wikia.com/Gracenote:" + artist.Replace(" ", "_") + ":" + title.Replace(" ", "_")
result, err := http.Get(link)
if err == nil {
data := &result.Data
writersArr := data.FindAllString(`Songwriters.+`, 1)
if writersArr.Length() > 0 {
song.GracenoteSongwriters = writersArr[0].GetTags("em")[0].RemoveHtmlTags("").DecodeHTML()
}
publisheraArr := data.FindAllString(`Publishers.+`, 1)
if publisheraArr.Length() > 0 {
song.GracenotePublishers = publisheraArr[0].GetTags("em")[0].RemoveHtmlTags("").DecodeHTML()
}
lyricArr := data.FindAllStringSubmatch(`(?mis)<div class='lyricbox'>(.+?)<\!--`, 1)
if len(lyricArr) > 0 {
song.GracenoteLyric = lyricArr[0][1].Trim().DecodeHTML().ReplaceWithRegexp(`(?mis)^<div.+?</span></div>`, "").Trim().Replace("<br />", "\n")
}
if song.GracenoteLyric != "" {
song.DownloadGracenoteDone = 1
} else {
song.DownloadGracenoteDone = 0
}
}
}
// Update gets item from sites and save them to database
func Update(state *StateHandler) *Counter {
CheckStateHandler(state)
var (
counter *Counter = NewCounterWithStateHandler(state)
idcFormat dna.String
cData dna.String
idc *terminal.Indicator
bar *terminal.ProgressBar
errChannel chan bool = make(chan bool)
tableName dna.String = state.GetTableName()
startupFmt dna.String = "Update %v - Cid:%v - Pat:%v - Ncf:%v - NCon:%v"
)
// 3rd pattern: callind GetCid() wil invoke error
INFO.Println(dna.Sprintf(startupFmt, tableName, state.Cid, state.GetPattern(), state.GetNCFail(), state.SiteConfig.NConcurrent))
if utils.IsValidTable(tableName, state.GetDb()) == false {
tableName = ""
}
if state.GetPattern() == 1 {
idcFormat = " $indicator %v|cid:%v|cf:%v" // cid: current id, cf: continuous failure count
idc = terminal.NewIndicatorWithTheme(terminal.ThemeDefault)
// Getting maxid from an item's table
id, err := utils.GetMaxId(tableName, state.GetDb())
dna.PanicError(err)
state.SetCid(id)
} else {
bar = getUpdateProgressBar(counter.Total, tableName)
}
// Config.NConcurrent
for i := dna.Int(0); i < state.SiteConfig.NConcurrent; i++ {
go atomicUpdate(errChannel, state)
}
for state.IsComplete() == false {
hasError := <-errChannel
counter.Tick(dna.Bool(hasError))
switch state.GetPattern() {
case 1:
if counter.GetCFail() == state.GetNCFail() {
state.SetCompletion()
}
idc.Show(dna.Sprintf(idcFormat, counter, state.GetCid(), counter.GetCFail()))
case 2:
if counter.GetCount() == state.GetRange().Total {
state.SetCompletion()
}
cData = dna.Sprintf("%v | Ncc:%v | Cid:%v", counter, state.GetNcCount(), state.GetCid())
bar.Show(counter.GetCount(), cData, cData.Replace("|", "-"))
case 3:
if counter.GetCount() == state.GetExtSlice().Length() {
state.SetCompletion()
}
cData = dna.Sprintf("%v | Ncc:%v | Cid:%v", counter, state.GetNcCount(), state.GetCid())
bar.Show(counter.GetCount(), cData, cData.Replace("|", "-"))
}
}
if state.GetPattern() == 1 {
idc.Close(dna.Sprintf("$indicator Complete updating %v!", tableName))
}
INFO.Printf("[%v] %v\n", tableName, counter.FinalString())
// Delay 2s to ensure all the goroutines left finish it processed before sqlpg.DB closed
// time.Sleep(2 * time.Second)
return counter
}
func GetSong(artist, title dna.String) (*Song, error) {
link := "http://lyrics.wikia.com/" + artist.Replace(" ", "_") + ":" + title.Replace(" ", "_")
result, err := http.Get(link)
// Log(link)
// Log(result.Data)
song := NewSong()
song.Title = title
song.Artists = artist.ToStringArray()
if err == nil {
data := &result.Data
if data.Match(`class='lyricbox'.+Instrumental.+TrebleClef`) == true {
song.Lyric = "Instrumental"
}
lyricArr := data.FindAllStringSubmatch(`(?mis)<div class='lyricbox'>(.+?)<!--`, 1)
if len(lyricArr) > 0 {
song.Lyric = lyricArr[0][1].Trim().DecodeHTML().ReplaceWithRegexp(`(?mis)^<div.+?</span></div>`, "").Trim().Replace("<br />", "\n")
}
if song.Lyric == "" {
song.DownloadDone = -1
} else {
song.DownloadDone = 1
}
musicBrainzArr := data.FindAllString(`http://musicbrainz.+html`, 1)
if musicBrainzArr.Length() > 0 {
song.MusicbrainzLink = musicBrainzArr[0]
}
allmusicArr := data.FindAllString(`http://www\.allmusic\.com.+`, 1)
if allmusicArr.Length() > 0 {
song.AllmusicLink = allmusicArr[0].ReplaceWithRegexp(`".+$`, "")
}
youtubeArr := data.FindAllString(`http://youtube\.com.+`, 1)
if youtubeArr.Length() > 0 {
song.YoutubeLink = youtubeArr[0].ReplaceWithRegexp(`".+$`, "")
}
if data.Match("View the Gracenote") {
song.IsGracenote = "1"
}
languageArr := data.FindAllStringSubmatch(`category normal" data-name="Language/(.+)" data-namespace`, 1)
if len(languageArr) > 0 {
song.Language = languageArr[0][1]
}
albumArr := data.FindAllStringSubmatch(`appears on the album.+">(.+)</a></i>`, 1)
if len(albumArr) > 0 {
song.AlbumTitle = albumArr[0][1]
}
if song.AlbumTitle.Match(`[0-9]+`) {
song.Year = song.AlbumTitle.FindAllStringSubmatch(`([0-9]+)`, 1)[0][1]
}
if data.Match(`It has been suggested that Gracenote`) {
song.IsGracenote = "1"
getGracenoteSongLyric(artist, title, song)
}
return song, nil
} else {
return nil, err
}
}