// ToAlbum converts APIAlbum to Album type.
func (apiAlbum *APIAlbum) ToAlbum() *Album {
album := NewAlbum()
album.Id = apiAlbum.Id
keyArr := apiAlbum.Url.FindAllStringSubmatch(`.+/(.+?)\.html`, 1)
if len(keyArr) > 0 {
album.Key = keyArr[0][1]
}
album.Title = apiAlbum.Title
album.Artists = apiAlbum.Artists.Split(" ft ").FilterEmpty()
album.Plays = apiAlbum.Plays
songids := dna.IntArray{}
for _, song := range apiAlbum.SongList {
songids.Push(song.Id)
}
album.Songids = songids
album.Nsongs = dna.Int(len(apiAlbum.SongList))
album.Description = ""
album.Coverart = apiAlbum.Coverart
dateCreatedArr := album.Coverart.FindAllStringSubmatch(`/?(\d{4}/\d{2}/\d{2})`, -1)
if len(dateCreatedArr) > 0 {
year := dateCreatedArr[0][1].FindAllStringSubmatch(`(\d{4})/\d{2}/\d{2}`, -1)[0][1].ToInt()
month := dateCreatedArr[0][1].FindAllStringSubmatch(`\d{4}/(\d{2})/\d{2}`, -1)[0][1].ToInt()
day := dateCreatedArr[0][1].FindAllStringSubmatch(`\d{4}/\d{2}/(\d{2})`, -1)[0][1].ToInt()
album.DateCreated = time.Date(int(year), time.Month(month), int(day), 0, 0, 0, 0, time.UTC)
}
album.Checktime = time.Now()
return album
}
func (sf *APISongFreaksAlbum) ToAlbum() (*Album, error) {
if sferr := sf.HasError(); sferr != nil {
return nil, sferr
}
album := NewAlbum()
album.Id = sf.Album.Id
album.AMG = sf.Album.AMG
album.UrlSlug = sf.Album.UrlSlug
album.Year = sf.Album.Year
album.Coverart = sf.Album.Coverart
album.CoverartLarge = sf.Album.CoverartLarge
album.Title = sf.Album.Title
album.Ratings = dna.IntArray{sf.Album.Rating.AverageRating.ToInt(), sf.Album.Rating.UserRating, sf.Album.Rating.TotalRatings}
album.Artistid = sf.Album.Artist.Id
album.Artists = sf.Album.Artist.Name.ToStringArray()
album.Link = sf.Album.Link
songids := dna.IntArray{}
for _, track := range sf.Album.Tracks {
songids.Push(track.Id)
}
album.Songids = songids
album.ReviewAuthor = sf.Album.Review.Author
album.Review = sf.Album.Review.Value.DecodeHTML()
album.Checktime = time.Now()
return album, nil
}
// SelectMissingIds accepts a table name as an input and a list of ids as a source.
// It returns a new list of ids that does not exist in the destination table
//
// * tblName : a table name
// * srcIds : a source ids
// * db : a pointer to connected databased
// * Returns a new list of ids which are not from the specified table
//
// The format of sql statement is:
// WITH dna (id) AS (VALUES (5),(6),(7),(8),(9))
// SELECT id FROM dna WHERE NOT EXISTS
// (SELECT 1 from ziartists WHERE id=dna.id)
func SelectMissingIds(tblName dna.String, srcIds *dna.IntArray, db *sqlpg.DB) (*dna.IntArray, error) {
if srcIds.Length() > 0 {
val := dna.StringArray(srcIds.Map(func(val dna.Int, idx dna.Int) dna.String {
return "(" + val.ToString() + ")"
}).([]dna.String))
selectStmt := "with dna (id) as (values " + val.Join(",") + ") \n"
selectStmt += "SELECT id FROM dna WHERE NOT EXISTS\n (SELECT 1 from " + tblName + " WHERE id=dna.id)"
ids := &[]dna.Int{}
err := db.Select(ids, selectStmt)
switch {
case err != nil:
return nil, err
case err == nil && ids != nil:
slice := dna.IntArray(*ids)
return &slice, nil
case err == nil && ids == nil:
return &dna.IntArray{}, nil
default:
panic("Default case triggered. Case is not expected. Cannot select non existed ids")
}
} else {
return nil, errors.New("Empty input array")
}
}
func amGetErrIds(inputFile dna.String, mode dna.Int) *dna.IntArray {
var ret = dna.IntArray{}
b, err := ioutil.ReadFile(inputFile.String())
if err != nil {
panic(err)
}
data := dna.String(string(b))
lines := data.Split("\n")
for _, line := range lines {
switch mode {
case 1:
idArr := line.FindAllStringSubmatch(`([0-9]+) Post.+no such host`, 1)
if len(idArr) > 0 {
ret.Push(idArr[0][1].ToInt())
}
idArr = line.FindAllStringSubmatch(`Timeout.+at id :([0-9]+)`, 1)
if len(idArr) > 0 {
ret.Push(idArr[0][1].ToInt())
}
case 2:
ret.Push(line.ToInt())
}
}
if mode == 1 {
err = ioutil.WriteFile(inputFile.String(), []byte{}, 0644)
if err != nil {
dna.Log("Cannot write to file1:", err.Error())
}
}
ret = ret.Unique()
return &ret
}
func (apiAlbum *APIAlbum) FillAlbum(album *Album) {
var (
nSongs, nVideos dna.Int = 0, 0
)
album.Id = apiAlbum.Id
album.Key = apiAlbum.Key
album.Title = apiAlbum.Title
album.Artists = apiAlbum.Artist.Split(", ")
album.Topics = apiAlbum.Genre.ToStringArray()
album.Likes = apiAlbum.Likes
album.Plays = apiAlbum.Plays
ids := dna.IntArray{}
for _, song := range apiAlbum.Listsong {
if isSongFormat(song.StreamUrl) == true {
nSongs += 1
}
if isVideoFormat(song.StreamUrl) == true {
nVideos += 1
}
ids.Push(song.Id)
}
// dna.Log("NSONGS AND VIDEOS:", nSongs, nVideos)
album.Songids = ids
album.Nsongs = dna.Int(len(apiAlbum.Listsong))
album.Description = apiAlbum.Description
album.Coverart = apiAlbum.Coverart
album.LinkShare = apiAlbum.Linkshare
switch {
case nSongs == album.Nsongs:
album.Type = "PLAYLIST_SONG"
case nVideos == album.Nsongs:
album.Type = "PLAYLIST_VIDEO"
case nVideos+nSongs == album.Nsongs:
album.Type = "PLAYLIST_MIXED"
default:
album.Type = "PLAYLIST"
}
datecreatedArr := album.Coverart.FindAllStringSubmatch(`/([0-9]+)[_500]*\..+$`, -1)
if len(datecreatedArr) > 0 {
// Log(int64(datecreatedArr[0][1].ToInt()))
album.DateCreated = time.Unix(int64(datecreatedArr[0][1].ToInt()/1000), 0)
} else {
dateCreatedArr := album.Coverart.FindAllStringSubmatch(`/?(\d{4}/\d{2}/\d{2})`, -1)
if len(dateCreatedArr) > 0 {
year := dateCreatedArr[0][1].FindAllStringSubmatch(`(\d{4})/\d{2}/\d{2}`, -1)[0][1].ToInt()
month := dateCreatedArr[0][1].FindAllStringSubmatch(`\d{4}/(\d{2})/\d{2}`, -1)[0][1].ToInt()
day := dateCreatedArr[0][1].FindAllStringSubmatch(`\d{4}/\d{2}/(\d{2})`, -1)[0][1].ToInt()
album.DateCreated = time.Date(int(year), time.Month(month), int(day), 0, 0, 0, 0, time.UTC)
}
}
album.Checktime = time.Now()
}
// DecodeEncodedKey decodes an encoded key to a key
func DecodeEncodedKey(key dna.String) dna.String {
var y dna.IntArray = key[5:15].Split("").Map(func(v dna.String, i dna.Int) dna.Int {
for j, val := range c {
for _, char := range val.Split("") {
if char == v {
return dna.Int(j)
}
}
}
return -1
}).([]dna.Int)
return GetKey(y.ToString().ToInt() + 307843200)
}
func convertSongToIntArray(songStr dna.String) dna.IntArray {
var apiSongs []APISong
var ret = dna.IntArray{}
err := json.Unmarshal([]byte(string(songStr)), &apiSongs)
if err != nil {
panic("Invalid song string input")
} else {
for _, song := range apiSongs {
ret.Push(song.Id)
}
}
return ret
}
func (sf *APISongFreaksTrack) ToSong() (*Song, error) {
if sferr := sf.HasError(); sferr != nil {
return nil, sferr
}
song := NewSong()
song.Id = sf.Track.Id
song.TrackGroupId = sf.Track.TrackGroupId
song.AMG = sf.Track.AMG
song.UrlSlug = sf.Track.UrlSlug
song.IsInstrumental = sf.Track.IsInstrumental
song.Viewable = sf.Track.Viewable
song.Duration = utils.ToSeconds(sf.Track.Duration)
song.Lyricid = sf.Track.LyricId
song.HasLrc = sf.Track.HasLrc
song.TrackNumber = sf.Track.TrackNumber
song.DiscNumber = sf.Track.DiscNumber
song.Title = sf.Track.Title
song.Rating = dna.IntArray{sf.Track.Rating.AverageRating.ToInt(), sf.Track.Rating.UserRating, sf.Track.Rating.TotalRatings}
song.Albumid = sf.Track.Album.Id
artistIds := dna.IntArray{}
artists := dna.StringArray{}
for _, artist := range sf.Track.Artists {
artistIds.Push(artist.Id)
artists.Push(artist.Name)
}
song.Artistids = artistIds
song.Artists = artists
if sf.Track.Lrc.Lines != nil && len(sf.Track.Lrc.Lines) > 0 {
lines, err := json.Marshal(sf.Track.Lrc.Lines)
if err == nil {
song.Lrc = dna.String(string(lines))
}
}
song.Link = sf.Track.Link
song.Lyric = sf.Track.Lyrics
if song.Lyric != "" {
song.HasLyric = true
}
song.Copyright = sf.Track.Copyright
song.Writer = sf.Track.Writer
song.SubmittedLyric = sf.Track.SubmittedLyric
song.Checktime = time.Now()
return song, nil
}
func GetNewEpisodeKeys(movieid, currentEps dna.Int) (dna.IntArray, error) {
movie, err := GetMovie(movieid)
// Reset hdv.EpisodeKeyList
EpisodeKeyList = dna.IntArray{}
if err == nil {
if movie.CurrentEps > currentEps {
var ret = dna.IntArray{}
LastestMovieCurrentEps[movieid] = movie.CurrentEps
for i := currentEps + 1; i <= movie.CurrentEps; i++ {
ret.Push(ToEpisodeKey(movieid, i))
}
return ret, nil
} else {
return nil, errors.New(dna.Sprintf("Ep ID: %v has to updated episode", movieid).String())
}
} else {
return nil, err
}
}
func (apiSong *APISong) FillSong(song *Song) {
song.Id = apiSong.Id
song.Title = apiSong.Title
song.Artists = apiSong.Artist.ToStringArray()
song.Url = apiSong.Url
song.Lyric = apiSong.Lyric
song.Status = apiSong.Status
song.Gracenote = apiSong.Gracenote
song.Publishers = apiSong.Publishers
song.Writers = apiSong.Writers
song.LineCount = apiSong.LineCount
lt := dna.IntArray{}
for _, ts := range apiSong.LineTimestamps {
lt.Push(ts.Line)
lt.Push(ts.Timestamp)
}
song.LineTimestamps = lt
}
func (apiaward *APIAward) ToAward() *Award {
award := NewAward()
award.Id = apiaward.Id
award.Title = apiaward.Title
award.Section = ""
award.Year = apiaward.Year
award.Chart = apiaward.Chart
award.Peak = apiaward.Peak
award.Type = apiaward.Type
award.Prize = apiaward.Award
winnerids := dna.IntArray{}
winners := dna.StringArray{}
for _, winner := range apiaward.Winners {
winnerids.Push(winner.Id)
winners.Push(winner.Name)
}
award.Winnerids = winnerids
award.Winners = winners
return award
}
func (alca *AlbumCategory) Save(db *sqlpg.DB) error {
var last error
var aids = dna.IntArray{}
albums := &[]Album{}
for _, album := range *(alca.Albums) {
aids.Push(album.Id)
// dna.Log(album)
}
query := "SELECT id, topics, genres from nsalbums WHERE id IN (" + aids.Join(",") + ")"
// dna.Log(query)
err := db.Select(albums, query)
if err != nil {
dna.Log(query, alca, *alca.Albums)
dna.PanicError(err)
}
for _, album := range *(alca.Albums) {
foundIndex := 0
for j, anotherAlbum := range *(albums) {
if album.Id == anotherAlbum.Id {
foundIndex = j
}
}
if foundIndex < len(*albums) {
cat := album.Category.Concat((*albums)[foundIndex].Topics).Concat((*albums)[foundIndex].Genres).Unique()
album.Category = cat.Filter(func(v dna.String, i dna.Int) dna.Bool {
if v != "" {
return true
} else {
return false
}
})
}
last = db.Update(album, "id", "category")
}
return last
}
func (apisong *APISong) ToSong() *Song {
song := NewSong()
song.Id = apisong.Id
song.Title = apisong.Title.DecodeHTML()
song.Duration = apisong.Duration
artistids := dna.IntArray{}
artists := dna.StringArray{}
for _, artist := range apisong.Artists {
artistids.Push(artist.Id)
artists.Push(artist.Name)
}
song.Artistids = artistids
song.Artists = artists
composerids := dna.IntArray{}
composers := dna.StringArray{}
for _, composer := range apisong.Composers {
composerids.Push(composer.Id)
composers.Push(composer.Name)
}
song.Composerids = composerids
song.Composers = composers
song.Checktime = time.Now()
return song
}
func (apiSong *APIFullSong) FillSong(song *Song) {
song.Id = apiSong.Track.Id
song.Title = apiSong.Track.Title
artists := dna.StringArray{}
for _, artist := range apiSong.Track.Artists {
artists.Push(artist.Name)
}
song.Artists = artists
artistids := dna.IntArray{}
for _, artist := range apiSong.Track.Artists {
artistids.Push(artist.Id)
}
song.Artistids = artistids
song.Albumid = apiSong.Track.Album.Id
song.AlbumTitle = apiSong.Track.Album.Title
song.Duration = utils.ToSeconds(apiSong.Track.Duration)
song.Instrumental = apiSong.Track.Instrumental
song.Viewable = apiSong.Track.Viewable
song.HasLrc = apiSong.Track.HasLrc
song.LrcVerified = apiSong.Track.LrcVerified
song.Lyricid = apiSong.Track.Lyricid
song.Lyric = apiSong.Track.Lyrics
song.Copyright = apiSong.Track.Copyright
song.Writers = apiSong.Track.Writer
// Mon Jan 2 15:04:05 MST 2006
if apiSong.Track.LastUpdate != "" {
lastUpdate, err := time.Parse("2006-01-02 15:04:05", apiSong.Track.LastUpdate.String())
if err == nil {
song.DateUpdated = lastUpdate
} else {
dna.Log(err.Error(), " Song ID:", song.Id, " GOT:", apiSong.Track.LastUpdate, "\n\n")
}
}
}
func getCategory(songs *[]*Song, genre Genre, page dna.Int) <-chan bool {
channel := make(chan bool, 1)
go func() {
link := "http://nhacso.net/bai-hat-theo-the-loai-" + genre.Id.ToString() + "/joke-link-2-" + page.ToString() + ".html"
// dna.Log(link)
result, err := http.Get(link)
if err == nil {
data := &result.Data
// transform string {"2":[0,3,5,7,9,11,13,15,29],"10":[1,2,4,6,8,]}
// to map[dna.Int]dna.Int{20:2, 28:2, 4:10, 12:10} Ex: map[29] = 2
temp := data.FindAllStringSubmatch(`getCategory.+'(\{.+\})'`, -1)
mapping := map[dna.Int]dna.Int{}
if len(temp) > 0 && temp[0].Length() > 0 {
vals := temp[0][1].FindAllString(`"[0-9]+":\[[0-9,]+?\]`, -1)
if vals.Length() > 0 {
for _, val := range vals {
target := val.FindAllStringSubmatch(`"(.+)"`, -1)[0][1].ToInt()
arr := val.FindAllStringSubmatch(`\[(.+)\]`, -1)[0][1].Split(",").ToIntArray()
for _, itm := range arr {
mapping[itm] = target
}
}
}
}
// Finding cat id for each song. cats var is 2-dimentional array.
// Each index of it represents the correspondent song, its value is the categories the song belongs to
catStrings := data.FindAllString(`Thể loại :.+`, -1)
cats := []dna.IntArray{}
for _, val := range catStrings {
tagids := dna.IntArray{}
tmp := val.FindAllStringSubmatch(`cate_tag_song_([0-9]+)`, -1)
if len(tmp) > 0 {
for _, el := range tmp {
tagids.Push(el[1].ToInt())
}
}
cats = append(cats, tagids)
}
// Log(cats)
// get songids
temps := data.FindAllStringSubmatch(`play" id="blocksongtag_([0-9]+)`, -1)
songids := dna.IntArray{}
if len(temps) > 0 {
for _, val := range temps {
songids.Push(val[1].ToInt())
}
}
tmpsongs := &[]*Song{}
for idx, songid := range songids {
song := NewSong()
song.Id = songid
category := dna.StringArray{}
for _, val := range cats[idx] {
if mapping[val] > 0 && mapping[val] < CatTags.Length() {
if CatTags[mapping[val]] != "" {
category.Push(CatTags[mapping[val]])
}
} else {
mess := dna.Sprintf("WRONG INDEX AT CATTAGS: %v %v %v - %v", mapping[val], genre, page, link)
panic(mess.String())
}
}
category.Push(genre.Name)
song.Category = transformCats(category.Unique()).Unique()
*tmpsongs = append(*tmpsongs, song)
}
*songs = *tmpsongs
}
channel <- true
}()
return channel
}
// UpdateHDViet gets lastest items from hdviet.com.
//
// The update process goes through 8 steps and 2 parts:
//
// PART I: FINDING AND UPDATING NEW EPS OF MOVIES IN DB
// Step 1: Initalizing db connection, loading site config and state handler.
// Step 2: Finding all movies possibly having new episodes in DB
// Step 3: Finding all movies found in Step 2 actually having new episodes available from source website.
// Step 4: Checking consitency of movies' new found episodes from the source.
// Step 5: Getting new episode data from the source, save them to DB and update current_eps field of movies having new eps.
//
// PART II: GETTING NEW MOVIES FROM HDVIET SITE.
// Step 6: Find newest movies from the source.
// Step 7: Updating newest episodes from newest found movies in Step 6.
// Step 8: Recovering failed sql statements.
func UpdateHDViet() {
var mvTable = dna.String("hdvmovies")
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("hdv", SiteConfigPath)
dna.PanicError(err)
TIMEOUT_SECS = 100
// PART 1: UPDATING NEW EPISODES OF MOVIES IN DB
// STEP 2: Finding all movies possibly having new episodes in DB.
movieCurrentEps, err := hdv.GetMoviesCurrentEps(db, mvTable)
dna.PanicError(err)
newEpisodeKeys := dna.IntArray{}
for movieid, currentEps := range movieCurrentEps {
newEpisodeKeys.Push(hdv.ToEpisodeKey(movieid, currentEps))
}
newEpisodeKeys.Sort()
// STEP 3: Checking and getting new episodes if available from source website.
state := NewStateHandlerWithExtSlice(new(hdv.EpUpdater), &newEpisodeKeys, siteConf, db)
Update(state)
// STEP 4:Checking consitency of new found episodes of movies from the source.
movieIdList := dna.IntArray{}
for _, epKey := range hdv.LastestEpisodeKeyList {
mvid, _ := hdv.ToMovieIdAndEpisodeId(epKey)
movieIdList.Push(mvid)
}
movieIdList = movieIdList.Unique()
dna.Log("\nNumber of movies: ", movieIdList.Length(), "having new episodes:", hdv.LastestEpisodeKeyList.Length())
if movieIdList.Length() != dna.Int(len(hdv.LastestMovieCurrentEps)) {
dna.Log("LastestEpisodeKeyList & LastestMovieCurrentEps do not match! GOT:", movieIdList.Length(), len(hdv.LastestMovieCurrentEps))
}
// STEP 5: Getting new episode data from the source, save them to DB and update current_eps field of movies having new eps.
if hdv.LastestEpisodeKeyList.Length() > 0 {
state = NewStateHandlerWithExtSlice(new(hdv.Episode), &hdv.LastestEpisodeKeyList, siteConf, db)
Update(state)
RecoverErrorQueries(SqlErrorLogPath, db)
// dna.Log(hdv.LastestEpisodeKeyList)
hdv.SaveLastestMovieCurrentEps(db, mvTable, SQLERROR)
RecoverErrorQueries(SqlErrorLogPath, db)
} else {
dna.Log("No new episodes found. Update operation has been aborted!")
}
// PART 2: UPDATING NEW MOVIES FROM hdv site.
// STEP 6: Find newest movies from sources.
db.Ping()
state = NewStateHandler(new(hdv.Movie), siteConf, db)
Update(state)
// STEP 7: Updating newest episodes from newest found movies in Step 6.
hdv.EpisodeKeyList = hdv.EpisodeKeyList.Unique()
if hdv.EpisodeKeyList.Length() > 0 {
state = NewStateHandlerWithExtSlice(new(hdv.Episode), &hdv.EpisodeKeyList, siteConf, db)
Update(state)
} else {
dna.Log("No new movies found")
}
// STEP 8: Recovering failed sql statements.
RecoverErrorQueries(SqlErrorLogPath, db)
CountDown(3*time.Second, QuittingMessage, EndingMessage)
db.Close()
}