// CheckKey returns true if key is valid
func CheckKey(key dna.String) dna.Bool {
for _, v := range key.Split("") {
if a.IndexOf(v) == -1 {
return false
}
}
return true
}
func getAnchorTagsData(data dna.String) dna.StringArray {
return dna.StringArray(data.Split("|").Map(func(val dna.String, idx dna.Int) dna.String {
return val.RemoveHtmlTags("").Trim()
}).([]dna.String)).Filter(func(val dna.String, idx dna.Int) dna.Bool {
if val != "" {
return true
} else {
return false
}
})
}
// ToSeconds returns total seconds from the time format "01:02:03"
func ToSeconds(str dna.String) dna.Int {
if str == "" {
return 0
} else {
intervals := dna.IntArray(str.Split(":").Map(func(val dna.String, idx dna.Int) dna.Int {
return val.ToInt()
}).([]dna.Int))
switch intervals.Length() {
case 3:
return intervals[0]*3600 + intervals[1]*60 + intervals[2]
case 2:
return intervals[0]*60 + intervals[1]
case 1:
return intervals[0]
default:
return 0
}
}
}
func isSongFormat(str dna.String) dna.Bool {
if str.EndsWith("mp3") == true || str.EndsWith("m4a") == true {
return true
} else {
return false
}
}
func getOptionsForQuestion(tableOptions dna.String, question *Question) {
// dna.Log(tableOptions)
options := tableOptions.Replace(`<div class="bix-div-answer"`, "").Trim().Split("tdOptionDt")
if options.Length() > 6 {
panic("Answer options greater than 5 at: " + question.Cat.ToString())
}
for idx, option := range options {
if idx > 0 {
option := option.ReplaceWithRegexp(`^.+?>`, ``).ReplaceWithRegexp(`(?mis)<td class="bix-td-option".+$`, "").ReplaceWithRegexp(`</td>$`, "").ReplaceWithRegexp(`</td></tr><tr>$`, "").ReplaceWithRegexp(`</td></tr></table><input type="hidden".+$`, "").Trim()
switch idx {
case 1:
question.OptionA = option
case 2:
question.OptionB = option
case 3:
question.OptionC = option
case 4:
question.OptionD = option
case 5:
question.OptionE = option
}
}
}
}
// To Source converts string format such as "(12,123123)" to Source Type
func ToSource(sourceStr dna.String) Source {
if sourceStr.Match(`\([0-9]+,[0-9]+\)`) == false {
panic("Wrong Source format")
}
sourArr := sourceStr.ReplaceWithRegexp(`^\(|\)$`, "").Split(",")
return Source{sourArr[0].ToInt(), sourArr[1].ToInt()}
}
func irrcrpt(_arg1 dna.String, _arg2 dna.Int) dna.String {
var _local5 dna.Int
var _local3 dna.String = ""
var _local4 dna.Int
for _local4 < _arg1.Length() {
_local5 = _arg1.CharCodeAt(_local4)
if (_local5 >= 48) && (_local5 <= 57) {
_local5 = ((_local5 - _arg2) - 48)
if _local5 < 0 {
_local5 = (_local5 + ((57 - 48) + 1))
}
_local5 = ((_local5 % ((57 - 48) + 1)) + 48)
} else {
if (_local5 >= 65) && (_local5 <= 90) {
_local5 = ((_local5 - _arg2) - 65)
if _local5 < 0 {
_local5 = (_local5 + ((90 - 65) + 1))
}
_local5 = ((_local5 % ((90 - 65) + 1)) + 65)
} else {
if (_local5 >= 97) && (_local5 <= 122) {
_local5 = ((_local5 - _arg2) - 97)
if _local5 < 0 {
_local5 = (_local5 + ((122 - 97) + 1))
}
_local5 = ((_local5 % ((122 - 97) + 1)) + 97)
}
}
}
_local3 = (_local3 + dna.FromCharCode(_local5))
_local4++
}
return (_local3)
}
func amGetErrIds(inputFile dna.String, mode dna.Int) *dna.IntArray {
var ret = dna.IntArray{}
b, err := ioutil.ReadFile(inputFile.String())
if err != nil {
panic(err)
}
data := dna.String(string(b))
lines := data.Split("\n")
for _, line := range lines {
switch mode {
case 1:
idArr := line.FindAllStringSubmatch(`([0-9]+) Post.+no such host`, 1)
if len(idArr) > 0 {
ret.Push(idArr[0][1].ToInt())
}
idArr = line.FindAllStringSubmatch(`Timeout.+at id :([0-9]+)`, 1)
if len(idArr) > 0 {
ret.Push(idArr[0][1].ToInt())
}
case 2:
ret.Push(line.ToInt())
}
}
if mode == 1 {
err = ioutil.WriteFile(inputFile.String(), []byte{}, 0644)
if err != nil {
dna.Log("Cannot write to file1:", err.Error())
}
}
ret = ret.Unique()
return &ret
}
func getGracenoteSongLyric(artist, title dna.String, song *Song) {
link := "http://lyrics.wikia.com/Gracenote:" + artist.Replace(" ", "_") + ":" + title.Replace(" ", "_")
result, err := http.Get(link)
if err == nil {
data := &result.Data
writersArr := data.FindAllString(`Songwriters.+`, 1)
if writersArr.Length() > 0 {
song.GracenoteSongwriters = writersArr[0].GetTags("em")[0].RemoveHtmlTags("").DecodeHTML()
}
publisheraArr := data.FindAllString(`Publishers.+`, 1)
if publisheraArr.Length() > 0 {
song.GracenotePublishers = publisheraArr[0].GetTags("em")[0].RemoveHtmlTags("").DecodeHTML()
}
lyricArr := data.FindAllStringSubmatch(`(?mis)<div class='lyricbox'>(.+?)<\!--`, 1)
if len(lyricArr) > 0 {
song.GracenoteLyric = lyricArr[0][1].Trim().DecodeHTML().ReplaceWithRegexp(`(?mis)^<div.+?</span></div>`, "").Trim().Replace("<br />", "\n")
}
if song.GracenoteLyric != "" {
song.DownloadGracenoteDone = 1
} else {
song.DownloadGracenoteDone = 0
}
}
}
func IsValidTable(tblName dna.String, db *sqlpg.DB) dna.Bool {
_, err := db.Exec("select * from " + tblName.String() + " limit 0")
if err == nil {
return true
} else {
return false
}
}
func Post(url, bodyStr dna.String) (*http.Result, error) {
mutex.Lock()
http.DefaulHeader = Header
http.DefaulHeader.Set("Cookie", Cookie)
http.DefaulHeader.Set("Content-Length", string(bodyStr.Length().ToString()))
mutex.Unlock()
// dna.Log(http.DefaulHeader)
return http.Post(url, bodyStr)
}
func getNames(data, from, to dna.String) dna.StringArray {
pattern := "(?mis)" + from + "(.+?)" + to
arr := data.FindAllStringSubmatch(pattern, 1)
if len(arr) > 0 {
return getAnchorTagsData(arr[0][1])
} else {
return dna.StringArray{}
}
}
// getSongFromMainPage returns song from main page
func getSongFromMainPage(song *Song) <-chan bool {
channel := make(chan bool, 1)
go func() {
link := "http://nhacso.net/nghe-nhac/google-bot." + GetKey(song.Id) + "==.html"
result, err := http.Get(link)
// Log(link)
// Log(result.Data)
if err == nil && !result.Data.Match("Rất tiếc, chúng tôi không tìm thấy thông tin bạn yêu cầu!") {
data := &result.Data
if data.Match("official") {
song.Official = 1
}
bitrate := data.FindAllString(`\d+kb\/s`, 1)[0]
if !bitrate.IsBlank() {
song.Bitrate = bitrate.FindAllString(`\d+`, 1)[0].ToInt()
}
plays := data.FindAllString("total_listen_song_detail_\\d+.+", 1)[0]
if !plays.IsBlank() {
song.Plays = plays.ReplaceWithRegexp("<\\/span>.+$", "").ReplaceWithRegexp("^.+>", "").ReplaceWithRegexp("\\.", "").ToInt()
}
topics := data.FindAllString("<li><a\\shref\\=\\\"http\\:\\/\\/nhacso\\.net\\/the-loai.+", 1)[0]
if !topics.IsBlank() {
topics = topics.ReplaceWithRegexp("^.+\\\">|<\\/a><\\/li>", "")
song.Topics = topics.ToStringArray().SplitWithRegexp(" - ").SplitWithRegexp("/")
temp := data.FindAllString("<a.+class=\"casi\".+>(.+?)<\\/a>", 1)
var singer dna.String
if temp.Length() > 0 {
singer = temp[0]
} else {
singer = ""
}
if topics.Match("Nhạc Hoa") && singer.Match(` / `) {
song.SameArtist = 1
}
}
lyric := data.FindAllString(`(?mis)txtlyric.+Bạn chưa nhập nội bài hát`, 1)[0]
if !lyric.IsBlank() {
song.Islyric = 1
song.Lyric = lyric.ReplaceWithRegexp("(?mis)<\\/textarea>.+$", "").ReplaceWithRegexp("^.+>", "")
if song.Lyric.Match("Hãy đóng góp lời bài hát chính xác cho Nhacso nhé") {
song.Lyric = ``
song.Islyric = 0
}
}
}
channel <- true
}()
return channel
}
func getVideoUrl(str, resolution dna.String) *VideoUrl {
su := NewVideoUrl()
su.Resolution = resolution
su.Type = getType(str)
su.Link = str.GetTagAttributes("href").ReplaceWithRegexp(`(http.+/).+(\..+$)`, "${1}file-name${2}")
size := str.FindAllString(`[0-9\.]+ MB`, -1)
if size.Length() > 0 {
su.Size = size[0].ParseBytesFormat() / 1000
}
return su
}
// DecodePath decodes encoded string such as "MjAxMyUyRjExJTJGMDUlMkYwJTJGMiUyRjAyN2UzN2M4NDUwMWFlOTEwNGNkZjgyMDZjYWE4OTkzLm1wMyU3QzI="
// into its real path on server such as "/2013/11/05/0/2/027e37c84501ae9104cdf8206caa8993.mp3|2"
func DecodePath(encodedPath dna.String) dna.String {
ret, err := base64.StdEncoding.DecodeString(encodedPath.String())
if err == nil {
escape, err := url.QueryUnescape(string(ret))
if err == nil {
return dna.String(escape)
} else {
return ""
}
} else {
return ""
}
}
func startVPN(vpnAppName, vpnAppUrl dna.String) chan *exec.Cmd {
c := make(chan *exec.Cmd)
dna.Log("Starting", vpnAppName)
go func() {
cmd := exec.Command(vpnAppUrl.String())
c <- cmd
err := cmd.Run()
if err != nil {
dna.Log(err.Error())
}
}()
return c
}
// Get impliments getting site with basic properties.
// Enable gzip, deflat by default to reduce network data, redirect to new location from response.
// It returns data (String type) and error
// if err is nil then data is "" (empty).
func Get(url dna.String) (*Result, error) {
req, err := http.NewRequest("GET", url.ToPrimitiveValue(), nil)
req.Header = DefaulHeader
// req.Header.Add("Accept-Encoding", "gzip,deflate")
// req.Header.Add("Accept-Language", "en-US,en")
// req.Header.Add("Cache-Control", "max-age=0")
// req.Header.Add("Connection", "keep-alive")
req.Header.Add("Host", url.ToPrimitiveValue())
// req.Header.Add("User-Agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
// req.Header.Add("Cookie", "")
// dna.Log(req.Header)
res, err := client.Do(req)
if err != nil {
return new(Result), err
}
var data []byte
var myErr error
switch res.Header.Get("Content-Encoding") {
case "gzip":
var reader io.ReadCloser
reader, err := gzip.NewReader(res.Body)
if err != nil {
return new(Result), err
}
data, myErr = ioutil.ReadAll(reader)
reader.Close()
case "deflate":
// Logv("sdsafsd")
reader, err := zlib.NewReader(res.Body)
if err != nil {
return new(Result), err
}
data, myErr = ioutil.ReadAll(reader)
reader.Close()
default:
data, myErr = ioutil.ReadAll(res.Body)
}
if myErr != nil {
return new(Result), myErr
}
res.Body.Close()
return NewResult(dna.Int(res.StatusCode), dna.String(res.Status), res.Header, dna.String(data)), nil
}
func getGenresFromDesc(desc dna.String) dna.StringArray {
var ret dna.StringArray
genres := desc.FindAllString(`(?i)genres?(\s+)?:?.+`, 1)
// "Released:" found in album id: 836258
if genres.Length() > 0 {
ret = dna.StringArray(genres[0].ReplaceWithRegexp(`(?mis)genres?(\s+)?:?`, "").ReplaceWithRegexp(`\.?\s*Released:.+`, "").Trim().Split(",").Map(func(val dna.String, idx dna.Int) dna.String {
return val.ReplaceWithRegexp(":", "").Trim()
}).([]dna.String))
if ret.Length() == 1 {
arr := dna.StringArray{}
if ret[0].FindAllString(`(?mis)K-Pop`, 1).Length() > 0 {
arr.Push("Korean Pop")
arr.Push(ret[0].ReplaceWithRegexp(`(?mis)\(?K-Pop\)?`, "").Trim())
ret = arr
}
}
}
return ret.SplitWithRegexp(` > `).SplitWithRegexp(`/`)
}
func refineAuthorsOrArtists(str dna.String) dna.StringArray {
tmp := str.ToStringArray().SplitWithRegexp(` / `).SplitWithRegexp(` - `).SplitWithRegexp(` – `)
tmp = tmp.SplitWithRegexp(`, `).SplitWithRegexp(` ft `).SplitWithRegexp(` feat `).SplitWithRegexp(` ft. `)
tmp = tmp.SplitWithRegexp(` feat. `).SplitWithRegexp(` Feat. `).SplitWithRegexp(` Ft. `)
tmp = tmp.SplitWithRegexp(` & `).SplitWithRegexp(` vs. `).SplitWithRegexp(`- `).SplitWithRegexp(` & `)
tmp = dna.StringArray(tmp.Map(func(val dna.String, idx dna.Int) dna.String {
rv := val.Replace(`Đang Cập Nhật...`, ``).Replace(`Đang Cập Nhật (QT)`, ``)
rv = rv.Replace(`Đang Cập Nhật (VN)`, ``).Replace(`Nhạc Phim QT`, `Nhạc Phim Quốc Tế`)
rv = rv.Replace(`Đang cập nhật`, ``).Replace(`Nhiều Ca Sỹ`, `Various Artists`)
return rv
}).([]dna.String)).Filter(func(val dna.String, idx dna.Int) dna.Bool {
if val != "" {
return true
} else {
return false
}
})
return tmp
}
// ForeachLine loops through every line a file.
// An anynomous input function has line, index as params
func ForeachLine(filePath dna.String, lineFunc func(dna.String, dna.Int)) {
var err error
var line []byte
f, err := os.Open(filePath.String())
if err != nil {
dna.Log("error opening file: %v\n", err)
os.Exit(1)
}
defer f.Close()
r := bufio.NewReaderSize(f, 4*1024)
i := 0
err = nil
for err == nil {
i += 1
line, _, err = r.ReadLine()
lineFunc(dna.String(string(line)), dna.Int(i))
}
}
//DecryptLRC returns LRC string from encrypted string.
//
//For example: Given a song id 2882720 with a key 9Fd4zVvPMIbf
//and an XML source file http://www.nhaccuatui.com/flash/xml?key1=8bfe9db992afaff5dc931dffca1b5c7b
//
//The encrypted lyric file url lies in <lyric> tags
// http://lrc.nct.nixcdn.com/2013/12/29/0/a/5/b/1388320374102.lrc
func DecryptLRC(data dna.String) (dna.String, error) {
// CODE_SPECIAL "M z s 2 d k v t u 5 o d u" separated by space
keyStr := irrcrpt(dna.Sprintf("%s", []byte{0x4d, 0x7a, 0x73, 0x32, 0x64, 0x6b, 0x76, 0x74, 0x75, 0x35, 0x6f, 0x64, 0x75}), 1)
keyStrInHex := hex.EncodeToString(keyStr.ToBytes())
keyStrInBytes, err := hex.DecodeString(keyStrInHex)
if err != nil {
return "", err
}
ret, err := hex.DecodeString(data.String())
if err != nil {
return "", err
}
cipher, err := rc4.NewCipher(keyStrInBytes)
if err != nil {
return "", err
} else {
cipher.XORKeyStream(ret, ret)
return dna.String(string(ret)), nil
}
}
func isVideoFormat(str dna.String) dna.Bool {
if str.EndsWith("mp4") == true || str.EndsWith("mpg") == true || str.EndsWith("flv") == true {
return true
} else {
return false
}
}
// RecoverSQLLogError re-executes failed sql queries in sql error log file from specified path.
// It returns the number of failed -reexec queries, and new failed
// queries will be written to the file
//
// The format of error file is:
// Error description - $$$error$$$SQL_QUERY$$$error$$$
// Therefore only get statements enclosed by special `$$$error$$$`
func RecoverSQLLogError(sqlErrFilePath dna.String, db *sqlpg.DB) dna.Int {
var errCount = 0
var errStrings = dna.StringArray{}
b, err := ioutil.ReadFile(sqlErrFilePath.String())
if err != nil {
panic(err)
}
data := dna.String(string(b))
// dna.Log("\n", data.Length())
sqlArr := data.FindAllString(`(?mis)\$\$\$error\$\$\$.+?\$\$\$error\$\$\$`, -1)
// dna.Log("\nTOTAL SQL STATEMENTS FOUND:", sqlArr.Length())
for _, val := range sqlArr {
sqlStmtArr := val.FindAllStringSubmatch(`(?mis)\$\$\$error\$\$\$(.+?)\$\$\$error\$\$\$`, -1)
if len(sqlStmtArr) > 0 {
_, err := db.Exec(sqlStmtArr[0][1].String())
if err != nil {
if dna.String(err.Error()).Contains(`duplicate key value violates unique constraint`) == false {
errCount += 1
errStrings.Push("$$$error$$$" + sqlStmtArr[0][1] + "$$$error$$$")
}
}
}
}
if errCount == 0 {
err = ioutil.WriteFile(sqlErrFilePath.String(), []byte{}, 0644)
} else {
err = ioutil.WriteFile(sqlErrFilePath.String(), []byte(errStrings.Join("\n").String()), 0644)
}
if err != nil {
panic(err)
}
return dna.Int(errCount)
}
// ToSiteid returns a siteid id from a short form.
// For example: "ns" is for NhacSo
func ToSiteid(shortForm dna.String) dna.Int {
switch shortForm {
case "ns":
return NhacSo
case "zi":
return Zing
case "nct":
return NhacCuaTui
case "cc":
return ChaCha
case "nv":
return NhacVui
case "csn":
return ChiaSeNhac
case "hdv":
return HDViet
case "ke":
return Keeng
case "sf":
return SongFreaks
case "am":
return AllMusic
case "lw":
return LyricWiki
case "ml":
return MetroLyrics
case "lf":
return LyricFind
case "vg":
return VietGiaiTri
case "mv":
return MusicVNN
default:
panic("Cannot convert shortform to siteid - GOT:" + shortForm.String())
}
}
func (users *Users) Save(db *sqlpg.DB) error {
var queries dna.String = ""
// var err error
// for _, user := range users.List {
// if user.IsValid() == true {
// err = db.InsertIgnore(user)
// }
// }
// return err
for _, user := range users.List {
if user.IsValid() == true {
queries += sqlpg.GetInsertStatement("ziusers", user, false) + "\n"
}
}
mutex.Lock()
n, err := SongFile.WriteString(queries.String())
if err != nil {
dna.Log("Cannot write to file while getting song", err.Error(), "\n\n")
}
TotalBytes += dna.Int(n)
mutex.Unlock()
return nil
}
func getTSGM(data *dna.String, kind dna.String) dna.String {
var itemArr dna.StringArray
switch kind {
case "genres":
itemArr = data.FindAllString(`(?mis)<h4>Genre</h4>(.+?)</div>`, 1)
case "styles":
itemArr = data.FindAllString(`(?mis)<h4>Styles</h4>(.+?)</div>`, 1)
case "moods":
itemArr = data.FindAllString(`(?mis)<h4>Album Moods</h4>(.+?)</div>`, 1)
case "themes":
itemArr = data.FindAllString(`(?mis)<h4>Themes</h4>(.+?)</div>`, 1)
default:
panic("Wrong kind!!!")
}
if itemArr.Length() > 0 {
catArr := itemArr[0].FindAllString(`<a href=.+?</a>`, -1)
categories := catArr.Map(func(val dna.String, idx dna.Int) Category {
var idArr []dna.StringArray
var id dna.Int = 0
name := val.RemoveHtmlTags("")
if kind == "moods" {
idArr = val.FindAllStringSubmatch(`xa([0-9]+)`, 1)
} else {
idArr = val.FindAllStringSubmatch(`ma([0-9]+)`, 1)
}
if len(idArr) > 0 {
id = idArr[0][1].ToInt()
}
return Category{id, name}
}).([]Category)
if len(categories) > 0 {
bCat, merr := json.Marshal(categories)
if merr != nil {
return "[]"
} else {
return dna.String(string(bCat))
}
} else {
return "[]"
}
} else {
return "[]"
}
}
func Post(url dna.String, bodyStr dna.String) (*Result, error) {
client := &http.Client{}
// dna.Log(bodyStr)
req, err := http.NewRequest("POST", url.String(), strings.NewReader(bodyStr.String()))
req.Header = DefaulHeader
req.Header.Add("Host", url.ToPrimitiveValue())
res, err := client.Do(req)
if err != nil {
return new(Result), err
}
var data []byte
var myErr error
switch res.Header.Get("Content-Encoding") {
case "gzip":
var reader io.ReadCloser
reader, err := gzip.NewReader(res.Body)
if err != nil {
return new(Result), err
}
data, myErr = ioutil.ReadAll(reader)
reader.Close()
case "deflate":
// Logv("sdsafsd")
reader, err := zlib.NewReader(res.Body)
if err != nil {
return new(Result), err
}
data, myErr = ioutil.ReadAll(reader)
reader.Close()
default:
data, myErr = ioutil.ReadAll(res.Body)
}
if myErr != nil {
return new(Result), myErr
}
res.Body.Close()
return NewResult(dna.Int(res.StatusCode), dna.String(res.Status), res.Header, dna.String(data)), nil
}
// Update gets item from sites and save them to database
func Update(state *StateHandler) *Counter {
CheckStateHandler(state)
var (
counter *Counter = NewCounterWithStateHandler(state)
idcFormat dna.String
cData dna.String
idc *terminal.Indicator
bar *terminal.ProgressBar
errChannel chan bool = make(chan bool)
tableName dna.String = state.GetTableName()
startupFmt dna.String = "Update %v - Cid:%v - Pat:%v - Ncf:%v - NCon:%v"
)
// 3rd pattern: callind GetCid() wil invoke error
INFO.Println(dna.Sprintf(startupFmt, tableName, state.Cid, state.GetPattern(), state.GetNCFail(), state.SiteConfig.NConcurrent))
if utils.IsValidTable(tableName, state.GetDb()) == false {
tableName = ""
}
if state.GetPattern() == 1 {
idcFormat = " $indicator %v|cid:%v|cf:%v" // cid: current id, cf: continuous failure count
idc = terminal.NewIndicatorWithTheme(terminal.ThemeDefault)
// Getting maxid from an item's table
id, err := utils.GetMaxId(tableName, state.GetDb())
dna.PanicError(err)
state.SetCid(id)
} else {
bar = getUpdateProgressBar(counter.Total, tableName)
}
// Config.NConcurrent
for i := dna.Int(0); i < state.SiteConfig.NConcurrent; i++ {
go atomicUpdate(errChannel, state)
}
for state.IsComplete() == false {
hasError := <-errChannel
counter.Tick(dna.Bool(hasError))
switch state.GetPattern() {
case 1:
if counter.GetCFail() == state.GetNCFail() {
state.SetCompletion()
}
idc.Show(dna.Sprintf(idcFormat, counter, state.GetCid(), counter.GetCFail()))
case 2:
if counter.GetCount() == state.GetRange().Total {
state.SetCompletion()
}
cData = dna.Sprintf("%v | Ncc:%v | Cid:%v", counter, state.GetNcCount(), state.GetCid())
bar.Show(counter.GetCount(), cData, cData.Replace("|", "-"))
case 3:
if counter.GetCount() == state.GetExtSlice().Length() {
state.SetCompletion()
}
cData = dna.Sprintf("%v | Ncc:%v | Cid:%v", counter, state.GetNcCount(), state.GetCid())
bar.Show(counter.GetCount(), cData, cData.Replace("|", "-"))
}
}
if state.GetPattern() == 1 {
idc.Close(dna.Sprintf("$indicator Complete updating %v!", tableName))
}
INFO.Printf("[%v] %v\n", tableName, counter.FinalString())
// Delay 2s to ensure all the goroutines left finish it processed before sqlpg.DB closed
// time.Sleep(2 * time.Second)
return counter
}
func getSection(section dna.String) APIAwardSection {
var awards = []APIAward{}
var name dna.String = ""
var sectionType dna.String = ""
var awardType dna.Int = 0
nameArr := section.FindAllString(`(?mis)<h2 class="headline">.+?</h2>`, -1)
if nameArr.Length() > 0 {
name = nameArr[0].RemoveHtmlTags("").Trim()
switch {
case name.Match("Singles") == true:
sectionType = "SINGLE"
awardType = 2
case name.Match("APIAlbums") == true:
sectionType = "ALBUM"
awardType = 1
}
}
if name.Match("Grammy Awards") == true {
return getGrammyAPIAwardSection(section)
}
tbody := section.FindAllString(`(?mis)<tbody>.+?</tbody>`, -1)
if tbody.Length() > 0 {
rows := tbody[0].FindAllString(`(?mis)<tr>.+?</tr>`, -1)
awards = rows.Map(func(row dna.String, idx dna.Int) APIAward {
var (
id dna.Int = 0
year dna.Int = 0
chart dna.String = ""
title dna.String = ""
peak dna.Int = 0
winners = []Person{}
award dna.String = ""
)
yearArr := row.FindAllString(`(?mis)<td class="year".+?</div>`, 1)
if yearArr.Length() > 0 {
year = yearArr[0].RemoveHtmlTags("").Trim().ToInt()
}
peakArr := row.FindAllString(`(?mis)<td class="peak".+?</td>`, 1)
if peakArr.Length() > 0 {
peak = peakArr[0].RemoveHtmlTags("").Trim().ToInt()
}
charArr := row.FindAllString(`(?mis)<div class="chart-name">.+?</div>`, 1)
if charArr.Length() > 0 {
chart = charArr[0].RemoveHtmlTags("").DecodeHTML().Trim()
}
titleArr := row.FindAllString(`<a href=".+</a>`, 1)
if titleArr.Length() > 0 {
title = titleArr[0].RemoveHtmlTags("").DecodeHTML().Trim()
}
var match dna.String = ""
switch sectionType {
case "SINGLE":
match = `<a href=.+mt([0-9]+)"`
case "ALBUM":
match = `<a href=.+mw([0-9]+)"`
default:
match = `<a href=.+mw([0-9]+)"`
}
idArr := row.FindAllStringSubmatch(match, 1)
if len(idArr) > 0 {
id = idArr[0][1].ToInt()
}
return APIAward{
Id: id,
Title: title,
Year: year,
Chart: chart,
Peak: peak,
Type: awardType,
Winners: winners,
Award: award,
}
}).([]APIAward)
}
return APIAwardSection{Name: name, Type: sectionType, Awards: awards}
}
func getGrammyAPIAwardSection(section dna.String) APIAwardSection {
var awards = []APIAward{}
tbody := section.FindAllString(`(?mis)<tbody>.+?</tbody>`, -1)
if tbody.Length() > 0 {
rows := tbody[0].FindAllString(`(?mis)<tr>.+?</tr>`, -1)
awards = rows.Map(func(row dna.String, idx dna.Int) APIAward {
var (
id dna.Int = 0
year dna.Int = 0
chart dna.String = ""
title dna.String = ""
peak dna.Int = 0
winners = []Person{}
award dna.String = ""
atype dna.Int = 0
)
yearArr := row.FindAllString(`(?mis)<div class="year">.+?</div>`, 1)
if yearArr.Length() > 0 {
year = yearArr[0].RemoveHtmlTags("").Trim().ToInt()
}
atypeArr := row.FindAllString(`(?mis)<div class="type">.+?</div>`, 1)
if atypeArr.Length() > 0 {
switch atypeArr[0].RemoveHtmlTags("").Trim() {
case "T":
atype = 2
case "A":
atype = 1
default:
atype = 0
}
}
awardArr := row.FindAllString(`(?mis)<div class="award-name">.+?</div>`, 1)
if awardArr.Length() > 0 {
award = awardArr[0].RemoveHtmlTags("").Trim()
}
titleArr := row.FindAllString(`(?mis)<td class="title".+?</div>`, 1)
if titleArr.Length() > 0 {
title = titleArr[0].RemoveHtmlTags("").Trim()
}
winnerArr := row.FindAllString(`<a href=".+?</a>`, -1)
winners = winnerArr.Map(func(winner dna.String, idx dna.Int) Person {
var winnerName dna.String = ""
var winnerId dna.Int = 0
winnerName = winner.RemoveHtmlTags("").Trim()
winnerIdArr := winner.FindAllStringSubmatch(`<a href=.+mn([0-9]+)`, 1)
if len(winnerIdArr) > 0 {
winnerId = winnerIdArr[0][1].ToInt()
}
return Person{Id: winnerId, Name: winnerName}
}).([]Person)
return APIAward{
Id: id,
Title: title,
Year: year,
Chart: chart,
Peak: peak,
Type: atype,
Winners: winners,
Award: award,
}
}).([]APIAward)
return APIAwardSection{Name: "Grammy Awards", Type: "ALBUMs & SONGS", Awards: awards}
} else {
return APIAwardSection{}
}
}