func getAnchorTagsData(data dna.String) dna.StringArray {
return dna.StringArray(data.Split("|").Map(func(val dna.String, idx dna.Int) dna.String {
return val.RemoveHtmlTags("").Trim()
}).([]dna.String)).Filter(func(val dna.String, idx dna.Int) dna.Bool {
if val != "" {
return true
} else {
return false
}
})
}
// Decrypt decodes a key into an integer ID
func Decrypt(key dna.String) dna.Int {
return dna.ParseInt(dna.StringArray(key.Split("").Map(func(v dna.String, i dna.Int) dna.String {
return b[a.IndexOf(v)]
}).([]dna.String)).Join(""), 16)
}
// SelectMissingKeys accepts a table name as an input and a list of keys as a source.
// It returns a new list of keys that does not exist in the destination table
//
// * tblName : a table name
// * srcKeys : a source keys
// * db : a pointer to connected databased
// * Returns a new list of keys which are not from the specified table
//
// Notice: Only applied to a table having a column named "key".
// The column has to be indexed to ensure good performance
//
// The format of sql statement is:
// with dna (key) as (values ('43f3HhhU6DGV'),('uFfgQhKbwAfN'),('RvFDlckJB5QU'),('uIF7rwd5wo4p'),('Kveukbhre1ry'),('oJ1lzAlKwJX6'),('43f3HhhU6DGV'),('uFfgQhKbwAfN'),('hfhtyMdywMau'),('PpZuccjYqy1b'))
// select key from dna where key not in
// (select key from nctalbums where key in ('43f3HhhU6DGV','uFfgQhKbwAfN','RvFDlckJB5QU','uIF7rwd5wo4p','Kveukbhre1ry','oJ1lzAlKwJX6','43f3HhhU6DGV','uFfgQhKbwAfN','hfhtyMdywMau','PpZuccjYqy1b'))
func SelectMissingKeys(tblName dna.String, srcKeys *dna.StringArray, db *sqlpg.DB) (*dna.StringArray, error) {
if srcKeys.Length() > 0 {
val := dna.StringArray(srcKeys.Map(func(val dna.String, idx dna.Int) dna.String {
return `('` + val + `')`
}).([]dna.String))
val1 := dna.StringArray(srcKeys.Map(func(val dna.String, idx dna.Int) dna.String {
return `'` + val + `'`
}).([]dna.String))
selectStmt := "with dna (key) as (values " + val.Join(",") + ") \n"
selectStmt += "select key from dna where key not in \n(select key from " + tblName + " where key in (" + val1.Join(",") + "))"
keys := &[]dna.String{}
err := db.Select(keys, selectStmt)
switch {
case err != nil:
return nil, err
case err == nil && keys != nil:
slice := dna.StringArray(*keys)
return &slice, nil
case err == nil && keys == nil:
return &dna.StringArray{}, nil
default:
panic("Default case triggered. Case is not expected. Cannot select non existed keys")
}
} else {
return nil, errors.New("Empty input array")
}
}
//GetAlbumFromAPI gets a album from API. It does not get content from main site.
func GetAlbumFromAPI(id dna.Int) (*Album, error) {
var album *Album = NewAlbum()
album.Id = id
apialbum, err := GetAPIAlbum(id)
if err != nil {
return nil, err
} else {
if apialbum.Response.MsgCode == 1 {
if GetKey(apialbum.Id) != GetKey(album.Id) {
errMes := dna.Sprintf("Resulted key and computed key are not match. %v =/= %v , id: %v =/= %v", GetKey(apialbum.Id), GetKey(album.Id), id, apialbum.Id)
panic(errMes.String())
}
album.Title = apialbum.Title
album.Artists = dna.StringArray(apialbum.Artists.Split(" , ").Map(func(val dna.String, idx dna.Int) dna.String {
return val.Trim()
}).([]dna.String)).SplitWithRegexp(",").Filter(func(v dna.String, i dna.Int) dna.Bool {
if v != "" {
return true
} else {
return false
}
})
album.Topics = dna.StringArray(apialbum.Topics.Split(", ").Map(func(val dna.String, idx dna.Int) dna.String {
return val.Trim()
}).([]dna.String)).SplitWithRegexp(" / ").Unique().Filter(func(v dna.String, i dna.Int) dna.Bool {
if v != "" {
return true
} else {
return false
}
})
album.Plays = apialbum.Plays
// album.Songids
// album.Nsongs
// album.EncodedKey
// album.Coverart
// album.DateCreated
album.YearReleased = apialbum.YearReleased
album.Description = apialbum.Description.RemoveHtmlTags("")
album.ArtistIds = apialbum.ArtistIds.Split(",").ToIntArray()
album.IsAlbum = apialbum.IsAlbum
album.IsHit = apialbum.IsHit
album.IsOfficial = apialbum.IsOfficial
album.Likes = apialbum.Likes
album.StatusId = apialbum.StatusId
album.Comments = apialbum.Comments
album.Checktime = time.Now()
return album, nil
} else {
return nil, errors.New("Message code invalid " + apialbum.Response.MsgCode.ToString().String())
}
}
}
// Fill writes all fields found APIUser instance to a User variable.
func (user *User) Fill(apiUser *APIUser) {
user.Id = apiUser.Id
user.Email = apiUser.Email
user.Mobile = dna.StringArray(apiUser.Mobile)
user.ProfilePoint = apiUser.ProfilePoint
user.StatusWall = apiUser.StatusWall
user.DisplayName = apiUser.DisplayName.Trim()
user.Point = apiUser.Point
user.Username = apiUser.Username
user.Avatar = apiUser.Avatar
user.BirthdayType = apiUser.BirthdayType
user.VipTotal = apiUser.Vip.Total
user.VipBlock = apiUser.Vip.Block
user.VipAvatar = apiUser.Vip.AvatarVip
user.IsNull = apiUser.Benull
user.Gender = apiUser.Gender
user.Status = apiUser.Status
user.Birthday = apiUser.Birthday
user.FeedWriteWallAll = apiUser.Feed.WriteWallAll
user.FeedViewWallAll = apiUser.Feed.ViewWallAll
user.CoverUrl = apiUser.CoverUrl
user.GoogleId = apiUser.GoogleId
user.YahooId = apiUser.YahooId
user.FriendTotal = apiUser.Friend.Total
user.FriendBlock = apiUser.Friend.Block
user.FriendAvatar = apiUser.Avatar
}
func (apiSong *APIFullSong) Save(db *sqlpg.DB) error {
var queries = dna.StringArray{}
var err error
// Getting artist queries
artists := apiSong.ToArtists()
for _, artist := range artists {
queries.Push(sqlpg.GetInsertIgnoreStatement(sqlpg.GetTableName(artist), artist, "id", artist.Id, false))
}
// Getting album query
album := apiSong.ToAlbum()
queries.Push(sqlpg.GetInsertIgnoreStatement(sqlpg.GetTableName(album), album, "id", album.Id, false))
// Getting song query
song := apiSong.ToSong()
queries.Push(sqlpg.GetInsertStatement(sqlpg.GetTableName(song), song, false))
for _, query := range queries {
_, err = db.Exec(query.String())
}
if err != nil {
errQueries := dna.StringArray(queries.Map(func(val dna.String, idx dna.Int) dna.String {
return "$$$error$$$" + val + "$$$error$$$"
}).([]dna.String))
return errors.New(err.Error() + errQueries.Join("\n").String())
} else {
return nil
}
}
func getCipherText(id dna.Int, tailArray dna.IntArray) dna.String {
rand.Seed(time.Now().UnixNano())
return dna.StringArray(dna.IntArray{1, 0, 8, 0, 10}.Concat((id - 307843200).ToString().Split("").ToIntArray()).Concat(tailArray).Map(
func(v dna.Int, i dna.Int) dna.String {
return c[v].Split("")[rand.Intn(len(c[v]))]
}).([]dna.String)).Join("")
}
// SelectMissingIds accepts a table name as an input and a list of ids as a source.
// It returns a new list of ids that does not exist in the destination table
//
// * tblName : a table name
// * srcIds : a source ids
// * db : a pointer to connected databased
// * Returns a new list of ids which are not from the specified table
//
// The format of sql statement is:
// WITH dna (id) AS (VALUES (5),(6),(7),(8),(9))
// SELECT id FROM dna WHERE NOT EXISTS
// (SELECT 1 from ziartists WHERE id=dna.id)
func SelectMissingIds(tblName dna.String, srcIds *dna.IntArray, db *sqlpg.DB) (*dna.IntArray, error) {
if srcIds.Length() > 0 {
val := dna.StringArray(srcIds.Map(func(val dna.Int, idx dna.Int) dna.String {
return "(" + val.ToString() + ")"
}).([]dna.String))
selectStmt := "with dna (id) as (values " + val.Join(",") + ") \n"
selectStmt += "SELECT id FROM dna WHERE NOT EXISTS\n (SELECT 1 from " + tblName + " WHERE id=dna.id)"
ids := &[]dna.Int{}
err := db.Select(ids, selectStmt)
switch {
case err != nil:
return nil, err
case err == nil && ids != nil:
slice := dna.IntArray(*ids)
return &slice, nil
case err == nil && ids == nil:
return &dna.IntArray{}, nil
default:
panic("Default case triggered. Case is not expected. Cannot select non existed ids")
}
} else {
return nil, errors.New("Empty input array")
}
}
// splitAndTruncateArtists splits stringarray by the key "feat:"
// and filter only string elements not equal to empty string.
func splitAndTruncateArtists(artists dna.StringArray) dna.StringArray {
return dna.StringArray(artists.SplitWithRegexp("feat:").Map(func(val dna.String, idx dna.Int) dna.String {
return val.Trim()
}).([]dna.String)).Filter(func(val dna.String, idx dna.Int) dna.Bool {
if val != "" {
return true
} else {
return false
}
})
}
func getGenresFromDesc(desc dna.String) dna.StringArray {
var ret dna.StringArray
genres := desc.FindAllString(`(?i)genres?(\s+)?:?.+`, 1)
// "Released:" found in album id: 836258
if genres.Length() > 0 {
ret = dna.StringArray(genres[0].ReplaceWithRegexp(`(?mis)genres?(\s+)?:?`, "").ReplaceWithRegexp(`\.?\s*Released:.+`, "").Trim().Split(",").Map(func(val dna.String, idx dna.Int) dna.String {
return val.ReplaceWithRegexp(":", "").Trim()
}).([]dna.String))
if ret.Length() == 1 {
arr := dna.StringArray{}
if ret[0].FindAllString(`(?mis)K-Pop`, 1).Length() > 0 {
arr.Push("Korean Pop")
arr.Push(ret[0].ReplaceWithRegexp(`(?mis)\(?K-Pop\)?`, "").Trim())
ret = arr
}
}
}
return ret.SplitWithRegexp(` > `).SplitWithRegexp(`/`)
}
// ExecQueriesInTransaction executes queries in a transaction.
// If one statement fails, the whole queries cannot commit.
//
// The returned error is nil if there is no error.
// If an error occurs, each statement will be enclosed in
// format $$$error$$$.
// $$$error$$$ Your Custom Query $$$error$$$
//
// This function is seen in songfreaks and allmusic sites.
func ExecQueriesInTransaction(db *DB, queries *dna.StringArray) error {
var err error
globalSqlTransactoNo += 1
// tx, err := db.Begin()
// if err != nil {
// dna.Log("Transaction No:" + dna.Sprintf("%v", globalSqlTransactoNo).String() + err.Error() + " Could not create transaction\n")
// }
for idx, query := range *queries {
_, err = db.Exec(query.String())
// _, err = tx.Exec(query.String())
if err != nil {
dna.Log(dna.Sprintf("DNAError: Query series No: %v - %v - %v - %v\n", dna.Sprintf("%v", globalSqlTransactoNo), idx, err.Error(), "Could not execute the statement"))
}
// stmt, err := tx.Prepare(query.String())
// if err != nil {
// dna.Log(dna.Sprintf("DNAError Transaction No: %v - %v - %v - %v \n", dna.Sprintf("%v", globalSqlTransactoNo), idx, err.Error(), "Could not prepare"))
// } else {
// _, err = stmt.Exec()
// if err != nil {
// dna.Log(dna.Sprintf("DNAError: Transaction No: %v - %v - %v - %v\n", dna.Sprintf("%v", globalSqlTransactoNo), idx, err.Error(), "Could not execute the prepared statement"))
// }
// err = stmt.Close()
// if err != nil {
// dna.Log("Transaction No:" + dna.Sprintf("%v", globalSqlTransactoNo).String() + err.Error() + " Could not close\n")
// }
// }
}
// err = tx.Commit()
// if err != nil {
// dna.Log("Transaction No:" + dna.Sprintf("%v", globalSqlTransactoNo).String() + err.Error() + " Could not commit transaction\n")
// }
if err != nil {
errQueries := dna.StringArray(queries.Map(func(val dna.String, idx dna.Int) dna.String {
return "Transaction No:" + dna.Sprintf("%v", globalSqlTransactoNo) + " $$$error$$$" + val + "$$$error$$$"
}).([]dna.String))
return errors.New(err.Error() + errQueries.Join("\n").String())
} else {
return nil
}
}
func refineAuthorsOrArtists(str dna.String) dna.StringArray {
tmp := str.ToStringArray().SplitWithRegexp(` / `).SplitWithRegexp(` - `).SplitWithRegexp(` – `)
tmp = tmp.SplitWithRegexp(`, `).SplitWithRegexp(` ft `).SplitWithRegexp(` feat `).SplitWithRegexp(` ft. `)
tmp = tmp.SplitWithRegexp(` feat. `).SplitWithRegexp(` Feat. `).SplitWithRegexp(` Ft. `)
tmp = tmp.SplitWithRegexp(` & `).SplitWithRegexp(` vs. `).SplitWithRegexp(`- `).SplitWithRegexp(` & `)
tmp = dna.StringArray(tmp.Map(func(val dna.String, idx dna.Int) dna.String {
rv := val.Replace(`Đang Cập Nhật...`, ``).Replace(`Đang Cập Nhật (QT)`, ``)
rv = rv.Replace(`Đang Cập Nhật (VN)`, ``).Replace(`Nhạc Phim QT`, `Nhạc Phim Quốc Tế`)
rv = rv.Replace(`Đang cập nhật`, ``).Replace(`Nhiều Ca Sỹ`, `Various Artists`)
return rv
}).([]dna.String)).Filter(func(val dna.String, idx dna.Int) dna.Bool {
if val != "" {
return true
} else {
return false
}
})
return tmp
}
func getSongFormats(song *Song) <-chan bool {
channel := make(chan bool, 1)
go func() {
link := "http://download.chiasenhac.com/google-bot~" + song.Id.ToString() + "_download.html"
result, err := http.Get(link)
// dna.Log(link)
if err == nil {
data := &result.Data
urlsArr := data.FindAllString(`<a href.+(Link|Mobile) Download.+`, -1)
if urlsArr.Length() > 0 {
// dna.Log(urlsArr.Length())
song.Formats, song.Type = GetFormats(urlsArr)
// dna.Log(song.Formats)
}
topicsArr := data.FindAllStringSubmatch(`<div class="plt-text">(.+)`, 1)
if len(topicsArr) > 0 {
topics := topicsArr[0][1].RemoveHtmlTags("").Trim().Split(`->`)
topics.Pop()
song.Topics = dna.StringArray(topics.Map(func(val dna.String, idx dna.Int) dna.String {
return val.Replace("...", "").Replace("Nước khác", "Nhạc Các Nước Khác").Replace("Âu, Mỹ", "Âu Mỹ").Trim().Title()
}).([]dna.String)).SplitWithRegexp(`, `).Unique()
}
hrefArr := data.FindAllString(`Download: <a.+`, 1)
if hrefArr.Length() > 0 {
song.Href = "http://chiasenhac.com/" + hrefArr[0].GetTagAttributes("href").Trim()
titleArtists := hrefArr[0].Replace("Download:", "").Trim().RemoveHtmlTags("").Split(" - ")
song.Artists = refineAuthorsOrArtists(titleArtists[titleArtists.Length()-1])
titleArtists.Pop()
song.Title = titleArtists.Join(" - ")
<-getSongFromMainPage(song)
}
dlArr := data.FindAllStringSubmatch(`([0-9]+) downloads`, 1)
if len(dlArr) > 0 {
song.Downloads = dlArr[0][1].Trim().Replace(".", "").ToInt()
}
dateCreatedArr := data.FindAllStringSubmatch(`<img src="images/tain5.gif" title="Upload at (.+?)" />`, 1)
if len(dateCreatedArr) > 0 {
tmp := dateCreatedArr[0][1].Trim()
switch {
case tmp.Match(`Hôm qua`) == true:
song.DateCreated = time.Now().AddDate(0, 0, -1)
case tmp.Match(`Hôm nay`) == true:
song.DateCreated = time.Now()
default:
val := tmp.ReplaceWithRegexp(`(\d+)/(\d+)/(\d+) (\d+):(\d+)`, "${3}-${2}-${1}T${4}:${5}:00Z")
parsedDate, err := time.Parse(time.RFC3339, val.String())
if err == nil {
song.DateCreated = parsedDate
}
}
}
}
channel <- true
}()
return channel
}
func getVideoFromMainPage(video *Video) <-chan bool {
channel := make(chan bool, 1)
go func() {
link := "http://nhacso.net/xem-video/google-bot." + GetKey(video.Id) + "=.html"
// Log(link)
result, err := http.Get(link)
if err == nil && !result.Data.Match("Rất tiếc, chúng tôi không tìm thấy thông tin bạn yêu cầu!") {
data := &result.Data
temp := data.FindAllString(`(?mis)<p class="title_video.+Đăng bởi`, 1)
if temp.Length() > 0 {
title := temp[0].FindAllString(`<h1 class="title">.+</h1>`, 1)
if title.Length() > 0 {
video.Title = title[0].RemoveHtmlTags("").Trim()
}
if temp[0].Match(`official`) {
video.Official = 1
}
artists := temp[0].FindAllString(`<h2>.+</h2>`, -1)
if artists.Length() > 0 {
video.Artists = dna.StringArray(artists.Map(func(val dna.String, idx dna.Int) dna.String {
return val.RemoveHtmlTags("").Trim()
}).([]dna.String)).SplitWithRegexp(` / `).Unique()
}
}
topics := data.FindAllString(`<li><a href="http://nhacso.net/the-loai-video/.+</a></li>`, 1)
if topics.Length() > 0 {
video.Topics = topics[0].RemoveHtmlTags("").ToStringArray().SplitWithRegexp(` - `).SplitWithRegexp(`/`)
}
plays := data.FindAllString(`<span>.+</span><ins> lượt xem</ins>`, 1)
if plays.Length() > 0 {
video.Plays = plays[0].GetTags("span")[0].RemoveHtmlTags("").Trim().Replace(".", "").ToInt()
}
thumbLink := data.FindAllString(`poster="(.+)" src="(.+)" data`, 1)
if thumbLink.Length() > 0 {
video.Thumbnail = thumbLink[0].FindAllStringSubmatch(`poster="(.+?)" `, 1)[0][1]
video.Link = thumbLink[0].FindAllStringSubmatch(`src="(.+?)" `, 1)[0][1]
if video.Link != "" {
ts := video.Link.FindAllStringSubmatch(`([0-9]+)_`, 1)
if len(ts) > 0 {
secs := float64(ts[0][1].ToInt()) * math.Pow10(13-len(ts[0][1]))
// Log(secs)
video.DateCreated = dna.Float(secs / 1000).ToInt().ToTime()
}
}
}
producerid := data.FindAllStringSubmatch(`getProducerByListIds\('(\d+)', '#producer_'\);`, 1)
if len(producerid) > 0 {
video.Producerid = producerid[0][1].ToInt()
}
}
channel <- true
}()
return channel
}
// getVideoFromMainPage returns song from main page
func getVideoFromMainPage(video *Video) <-chan bool {
channel := make(chan bool, 1)
go func() {
link := "http://mp3.zing.vn/video-clip/google-bot/" + GetKey(video.Id) + ".html"
result, err := http.Get(link)
// Log(link)
// Log(result.Data)
data := &result.Data
// dna.Log(data.Match("<title>Thông báo</title>"))
if err == nil && !data.Match("<title>Thông báo</title>") {
topicsArr := data.FindAllStringSubmatch(`Thể loại:(.+)\|`, -1)
if len(topicsArr) > 0 {
video.Topics = topicsArr[0][1].RemoveHtmlTags("").Trim().Split(", ").SplitWithRegexp(` / `).Unique()
}
playsArr := data.FindAllStringSubmatch(`Lượt xem:(.+)</p>`, -1)
if len(playsArr) > 0 {
video.Plays = playsArr[0][1].Trim().Replace(".", "").ToInt()
}
titleArr := data.FindAllStringSubmatch(`<h1 class="detail-title">(.+?)</h1>`, -1)
if len(titleArr) > 0 {
video.Title = titleArr[0][1].RemoveHtmlTags("").Trim()
}
artistsArr := data.FindAllStringSubmatch(`<h1 class="detail-title">.+(<a.+)`, -1)
if len(artistsArr) > 0 {
video.Artists = dna.StringArray(artistsArr[0][1].RemoveHtmlTags("").Trim().Split(" ft. ").Unique().Map(func(val dna.String, idx dna.Int) dna.String {
return val.Trim()
}).([]dna.String))
}
thumbnailArr := data.FindAllString(`<meta property="og:image".+`, -1)
if thumbnailArr.Length() > 0 {
video.Thumbnail = thumbnailArr[0].GetTagAttributes("content")
// datecreatedArr := video.Thumbnail.FindAllStringSubmatch(`_([0-9]+)\..+$`, -1)
// if len(datecreatedArr) > 0 {
// // Log(int64(datecreatedArr[0][1].ToInt()))
// video.DateCreated = time.Unix(int64(datecreatedArr[0][1].ToInt()), 0)
// }
datecreatedArr := video.Thumbnail.FindAllStringSubmatch(`_([0-9]+)\..+$`, -1)
if len(datecreatedArr) > 0 && datecreatedArr[0][1].Length() > 6 {
video.DateCreated = time.Unix(int64(datecreatedArr[0][1].ToInt()), 0)
} else {
dateCreatedArr := video.Thumbnail.FindAllStringSubmatch(`/?(\d{4}/\d{2}/\d{2})`, -1)
if len(dateCreatedArr) > 0 {
year := dateCreatedArr[0][1].FindAllStringSubmatch(`^(\d{4})/\d{2}/\d{2}`, -1)[0][1].ToInt()
month := dateCreatedArr[0][1].FindAllStringSubmatch(`^\d{4}/(\d{2})/\d{2}`, -1)[0][1].ToInt()
day := dateCreatedArr[0][1].FindAllStringSubmatch(`^\d{4}/\d{2}/(\d{2})`, -1)[0][1].ToInt()
video.DateCreated = time.Date(int(year), time.Month(month), int(day), 0, 0, 0, 0, time.UTC)
}
}
}
video.Link = video.GetDirectLink(Resolution360p)
lyricArr := data.FindAllStringSubmatch(`(?mis)<p class="_lyricContent.+</span></span>(.+?)<p class="seo">`, -1)
if len(lyricArr) > 0 {
video.Lyric = lyricArr[0][1].ReplaceWithRegexp(`(?mis)<p class="seo">.+`, "").Trim().Replace("</p>\r\n\t</div>\r\n\t\t</div>", "").Trim()
}
}
channel <- true
}()
return channel
}
//GetVideoFromAPI gets a video from API. It does not get content from main site.
func GetVideoFromAPI(id dna.Int) (*Video, error) {
var video *Video = NewVideo()
video.Id = id
apivideo, err := GetAPIVideo(id)
if err != nil {
return nil, err
} else {
if apivideo.Response.MsgCode == 1 {
video.Title = apivideo.Title
video.Artists = dna.StringArray(apivideo.Artists.Split(" , ").Map(func(val dna.String, idx dna.Int) dna.String {
return val.Trim()
}).([]dna.String)).Filter(func(v dna.String, i dna.Int) dna.Bool {
if v != "" {
return true
} else {
return false
}
})
video.Topics = dna.StringArray(apivideo.Topics.Split(", ").Map(func(val dna.String, idx dna.Int) dna.String {
return val.Trim()
}).([]dna.String)).SplitWithRegexp(" / ").Unique().Filter(func(v dna.String, i dna.Int) dna.Bool {
if v != "" {
return true
} else {
return false
}
})
video.Plays = apivideo.Plays
video.Thumbnail = "http://image.mp3.zdn.vn/" + apivideo.Thumbnail
// video.Lyric =
datecreatedArr := video.Thumbnail.FindAllStringSubmatch(`_([0-9]+)\..+$`, -1)
if len(datecreatedArr) > 0 && datecreatedArr[0][1].Length() > 6 {
video.DateCreated = time.Unix(int64(datecreatedArr[0][1].ToInt()), 0)
} else {
dateCreatedArr := video.Thumbnail.FindAllStringSubmatch(`/?(\d{4}/\d{2}/\d{2})`, -1)
if len(dateCreatedArr) > 0 {
year := dateCreatedArr[0][1].FindAllStringSubmatch(`^(\d{4})/\d{2}/\d{2}`, -1)[0][1].ToInt()
month := dateCreatedArr[0][1].FindAllStringSubmatch(`^\d{4}/(\d{2})/\d{2}`, -1)[0][1].ToInt()
day := dateCreatedArr[0][1].FindAllStringSubmatch(`^\d{4}/\d{2}/(\d{2})`, -1)[0][1].ToInt()
video.DateCreated = time.Date(int(year), time.Month(month), int(day), 0, 0, 0, 0, time.UTC)
}
}
video.Plays = apivideo.Plays
video.ArtistIds = apivideo.ArtistIds.Split(",").ToIntArray()
flags := 0
for key, val := range apivideo.Source {
video.Link = val
switch {
case key == "240" && val != "":
flags = flags | LRe240p
case key == "360" && val != "":
flags = flags | LRe360p
case key == "480" && val != "":
flags = flags | LRe480p
case key == "720" && val != "":
flags = flags | LRe720p
case key == "1080" && val != "":
flags = flags | LRe1080p
}
}
video.ResolutionFlags = dna.Int(flags)
video.Duration = apivideo.Duration
video.Likes = apivideo.Likes
video.StatusId = apivideo.StatusId
video.Comments = apivideo.Comments
video.Checktime = time.Now()
return video, nil
} else {
return nil, errors.New("Message code invalid " + apivideo.Response.MsgCode.ToString().String())
}
}
}
// Encrypt encodes an integer ID into a key
func Encrypt(id dna.Int) dna.String {
return dna.StringArray(id.ToHex().Split("").Map(
func(v dna.String, i dna.Int) dna.String {
return a[b.IndexOf(v)]
}).([]dna.String)).Join("")
}
// This function will encode the Id of nhacso into cipher text.
func Encrypt(id dna.Int) dna.String {
return dna.StringArray(id.ToString().Split("").Map(
func(v dna.String, i dna.Int) dna.String {
return ns[6-i][v.ToInt()]
}).([]dna.String)).Join("")
}
//GetSongFromAPI gets a song from API. It does not get content from main site.
func GetSongFromAPI(id dna.Int) (*Song, error) {
var song *Song = NewSong()
song.Id = id
asong, err := GetAPISong(id)
if err != nil {
return nil, err
} else {
if asong.Response.MsgCode == 1 {
if asong.Key != GetKey(song.Id) {
panic("Resulted key and computed key are not match.")
}
song.Key = asong.Key
song.Title = asong.Title
song.Artists = dna.StringArray(asong.Artists.Split(" , ").Map(func(val dna.String, idx dna.Int) dna.String {
return val.Trim()
}).([]dna.String)).SplitWithRegexp(",").Filter(func(v dna.String, i dna.Int) dna.Bool {
if v != "" {
return true
} else {
return false
}
})
song.ArtistIds = asong.ArtistIds.Split(",").ToIntArray()
song.Authors = asong.Authors.Split(", ").Filter(func(v dna.String, i dna.Int) dna.Bool {
if v != "" {
return true
} else {
return false
}
})
song.Plays = asong.Plays
song.Topics = dna.StringArray(asong.Topics.Split(", ").Map(func(val dna.String, idx dna.Int) dna.String {
return val.Trim()
}).([]dna.String)).SplitWithRegexp(" / ").Unique().Filter(func(v dna.String, i dna.Int) dna.Bool {
if v != "" {
return true
} else {
return false
}
})
// song.Link
// song.Path
// song.Lyric
// song.DateCreated
// song.Checktime = time.Time{}
if asong.Video.Id > 0 {
song.VideoId = asong.Video.Id + 307843200
}
if asong.AlbumId > 0 {
song.AlbumId = asong.AlbumId + 307843200
}
song.IsHit = asong.IsHit
song.IsOfficial = asong.IsOfficial
song.DownloadStatus = asong.DownloadStatus
song.Copyright = asong.Copyright
flags := 0
for key, val := range asong.Source {
switch {
case key == "128" && val != "":
flags = flags | LBr128
case key == "320" && val != "":
flags = flags | LBr320
case key == "lossless" && val != "":
flags = flags | LBrLossless
}
}
song.BitrateFlags = dna.Int(flags)
song.Likes = asong.Likes
song.Comments = asong.Comments
song.Thumbnail = asong.Thumbnail
song.Checktime = time.Now()
return song, nil
} else {
return nil, errors.New("Message code invalid " + asong.Response.MsgCode.ToString().String())
}
}
}
func getAlbumFromMainPage(album *Album) <-chan bool {
channel := make(chan bool, 1)
go func() {
link := "http://nhacso.net/nghe-album/ab." + GetKey(album.Id) + ".html"
// Log(link)
result, err := http.Get(link)
if err == nil && !result.Data.Match("Rất tiếc, chúng tôi không tìm thấy thông tin bạn yêu cầu!") {
data := &result.Data
temp := data.FindAllString(`(?mis)class="intro_album_detail.+id="divPlayer`, 1)[0]
if !temp.IsBlank() {
title := temp.GetTags("strong")[0]
if !title.IsBlank() {
album.Title = title.RemoveHtmlTags("")
}
artists := temp.FindAllString(`strong.+`, 1)[0]
if !artists.IsBlank() {
album.Artists = artists.ReplaceWithRegexp(`^.+>`, "").ToStringArray().SplitWithRegexp(`\|\|`).SplitWithRegexp(` / `).SplitWithRegexp(" - ")
artistid := artists.FindAllString(`\d+\.html`, 1)
if artistid.Length() > 0 {
album.Artistid = artistid[0].ReplaceWithRegexp(`\.html`, "").ToInt()
}
}
// get multiple artists, overwrite the artists var above
newArs := temp.FindAllString(`<p><span>.+?</span></p>`, -1)
if newArs.Length() > 0 {
album.Artists = dna.StringArray(newArs.Map(func(val dna.String, idx dna.Int) dna.String {
return val.RemoveHtmlTags("").Trim()
}).([]dna.String)).SplitWithRegexp(`\|\|`).SplitWithRegexp(` / `).SplitWithRegexp(" - ")
}
coverart := temp.GetTags(`img`)[0]
if !coverart.IsBlank() {
album.Coverart = coverart.GetTagAttributes("src")
}
}
description := data.FindAllString(`<p class="desc".+?</p>`, 1)
if description.Length() > 0 {
album.Description = description[0].Trim().Replace("<br>", "\n").RemoveHtmlTags("")
if album.Description.Match(`thưởng thức nhạc chất lượng cao và chia sẻ cảm xúc với bạn bè tại Nhacso.net`) {
album.Description = ""
}
album.Genres = getGenresFromDesc(album.Description)
album.Label = getLabelFromDesc(album.Description)
}
topics := data.FindAllString(`<li class="bg">.+</li>`, 1)[0]
if !topics.IsBlank() {
album.Topics = topics.RemoveHtmlTags("").ToStringArray().SplitWithRegexp(" / ").SplitWithRegexp(" - ")
}
songids := data.FindAllString(`songid_\d+`, -1)
if songids.Length() > 0 {
songids.ForEach(func(value dna.String, index dna.Int) {
album.Songids.Push(value.ReplaceWithRegexp(`songid_`, "").ToInt())
})
}
}
channel <- true
}()
return channel
}
// getSongFromMainPage returns song from main page
func getSongFromMainPage(song *Song) <-chan bool {
channel := make(chan bool, 1)
go func() {
link := "http://mp3.zing.vn/bai-hat/google-bot/" + song.Key + ".html"
result, err := http.Get(link)
// Log(link)
// Log(result.Data)
if err == nil {
data := &result.Data
titleArr := data.FindAllStringSubmatch(`<h1 class="detail-title">(.+?)</h1>`, -1)
if len(titleArr) > 0 {
song.Title = titleArr[0][1].Trim()
}
artistsArr := data.FindAllStringSubmatch(`<h1 class="detail-title">.+</h1><span>-</span>(.+)`, -1)
if len(artistsArr) > 0 {
artists := artistsArr[0][1].RemoveHtmlTags("").Trim().Split(", ").SplitWithRegexp("ft. ")
song.Artists = dna.StringArray(artists.Map(func(val dna.String, i dna.Int) dna.String {
return val.Trim()
}).([]dna.String))
}
playsArr := data.FindAllStringSubmatch(`Lượt nghe: (.+)</p>`, -1)
if len(playsArr) > 0 {
song.Plays = playsArr[0][1].Replace(".", "").ToInt()
}
authorsArr := data.FindAllStringSubmatch(`Sáng tác:(.+?)\|`, -1)
if len(authorsArr) > 0 {
authors := authorsArr[0][1].RemoveHtmlTags("").Trim().Split(", ").SplitWithRegexp(" / ").SplitWithRegexp(" & ")
song.Authors = dna.StringArray(authors.Map(func(val dna.String, idx dna.Int) dna.String {
switch {
case val == "Đang Cập Nhật":
return ""
case val == "Đang cập nhật":
return ""
default:
return val
}
}).([]dna.String)).Filter(func(v dna.String, i dna.Int) dna.Bool {
if v != "" {
return true
} else {
return false
}
})
}
topicsArr := data.FindAllStringSubmatch(`Thể loại:(.+?)\|`, -1)
if len(topicsArr) > 0 {
song.Topics = topicsArr[0][1].RemoveHtmlTags("").Trim().Split(", ").SplitWithRegexp(" / ").Unique()
}
lyricsArr := data.FindAllStringSubmatch(`(?mis)<p class="_lyricContent.+</span></span>(.+?)</p>`, -1)
if len(lyricsArr) > 0 {
song.Lyric = lyricsArr[0][1].Trim()
}
}
channel <- true
}()
return channel
}
// getVideoFromMainPage returns video from main page
func getVideoFromMainPage(video *Video) <-chan bool {
channel := make(chan bool, 1)
go func() {
link := "http://www.nhaccuatui.com/video/google-bot." + video.Key + ".html"
result, err := http.Get(link)
if err == nil {
data := &result.Data
topicsArr := data.FindAllStringSubmatch(`<strong>Thể loại</strong></p>[\n\t\r]+(.+)`, 1)
if len(topicsArr) > 0 {
video.Topics = topicsArr[0][1].RemoveHtmlTags("").Trim().Split(", ")
}
linkkeyArr := data.FindAllStringSubmatch(`"flashPlayer", ".+", "(.+?)"`, 1)
if len(linkkeyArr) > 0 {
video.LinkKey = linkkeyArr[0][1].Trim()
}
relativesArr := data.FindAllString("<a href=.+[\n\t\r]+.+video_relative", -1)
video.Relatives = dna.StringArray(relativesArr.Map(func(val dna.String, idx dna.Int) dna.String {
valArr := val.GetTagAttributes("href").Replace(".html", "").ReplaceWithRegexp(`^.+\.`, "")
return valArr
}).([]dna.String))
// new version in June 2013 does not support lyric of videos
// idArr := data.FindAllStringSubmatch(`value="(.+)" id="inpHiddenId"`, 1)
// if len(idArr) > 0 {
// video.Id = idArr[0][1].ToInt()
// }
// titleArr := data.FindAllStringSubmatch(`<h1 itemprop="name">(.+?)</h1>`, 1)
// if len(titleArr) > 0 {
// video.Title = titleArr[0][1].Trim().SplitWithRegexp(" - ", 2)[0].Trim()
// }
// artistsArr := data.FindAllStringSubmatch(`<h1 itemprop="name">(.+?)</h1>`, 1)
// if len(artistsArr) > 0 {
// artists := artistsArr[0][1].RemoveHtmlTags("").SplitWithRegexp(" - ", 2)
// if artists.Length() == 2 {
// video.Artists = artists[1].Split(", ").Filter(func(v dna.String, i dna.Int) dna.Bool {
// if v != "" {
// return true
// } else {
// return false
// }
// })
// }
// }
// durationArr := data.FindAllStringSubmatch(`<meta itemprop="duration" content="(.+)" />`, 1)
// if len(durationArr) > 0 {
// durationF := durationArr[0][1].Replace("PT", "").Replace("H", "h").Replace("M", "m").Replace("S", "s")
// duration, perr := time.ParseDuration(durationF.String())
// if perr == nil {
// video.Duration = dna.Float(duration.Seconds()).Round()
// }
// }
// thumbArr := data.FindAllString(`<link rel="image_src".+`, 1)
// if thumbArr.Length() > 0 {
// video.Thumbnail = thumbArr[0].GetTagAttributes("href").Trim()
// datecreatedArr := video.Thumbnail.FindAllStringSubmatch(`/([0-9]+)_[0-9]+\..+$`, -1)
// if len(datecreatedArr) > 0 {
// // Log(int64(datecreatedArr[0][1].ToInt()))
// video.DateCreated = time.Unix(int64(datecreatedArr[0][1].ToInt()/1000), 0)
// } else {
// dateCreatedArr := video.Thumbnail.FindAllStringSubmatch(`/?(\d{4}/\d{2}/\d{2})`, -1)
// if len(dateCreatedArr) > 0 {
// year := dateCreatedArr[0][1].FindAllStringSubmatch(`(\d{4})/\d{2}/\d{2}`, -1)[0][1].ToInt()
// month := dateCreatedArr[0][1].FindAllStringSubmatch(`\d{4}/(\d{2})/\d{2}`, -1)[0][1].ToInt()
// day := dateCreatedArr[0][1].FindAllStringSubmatch(`\d{4}/\d{2}/(\d{2})`, -1)[0][1].ToInt()
// video.DateCreated = time.Date(int(year), time.Month(month), int(day), 0, 0, 0, 0, time.UTC)
// }
// }
// }
// Find params for the number of video plays
// itemIdArr := data.FindAllStringSubmatch(`NCTWidget.hitCounter\('(.+?)'.+`, 1)
// timeArr := data.FindAllStringSubmatch(`NCTWidget.hitCounter\('.+?', '(.+?)'.+\);`, 1)
// signArr := data.FindAllStringSubmatch(`NCTWidget.hitCounter\('.+?', '.+?', '(.+?)'.+;`, 1)
// typeArr := data.FindAllStringSubmatch(`NCTWidget.hitCounter\('.+?', '.+?', '.+?', "(.+?)"\);`, 1)
// if len(itemIdArr) > 0 && len(timeArr) > 0 && len(signArr) > 0 && len(typeArr) > 0 {
// // boday has post form:
// // item_id=2870710&time=1389009424631&sign=2499ab08f6662842a02b06aad603d8ab&type=video
// body := dna.Sprintf(`item_id=%v&time=%v&sign=%v&type=%v`, itemIdArr[0][1], timeArr[0][1], signArr[0][1], typeArr[0][1])
// getVideoPlays(video, body)
// video.Type = typeArr[0][1].Trim()
// if video.Type == "video" {
// video.Type = "mv"
// }
// video.Id = itemIdArr[0][1].ToInt()
// }
// GetRelevantPortions(&result.Data)
}
channel <- true
}()
return channel
}