// UpdateSongFreaks gets lastest songs,albums,artists and videos from songfreaks.com
// The update process goes through 4 steps:
// Step 1: Initalizing db connection, loading site config and state handler.
// Step 2: Finding new songs, insert new albums,artists and videos if found.
// Step 3: Updating found new albums in Step 2.
// Step 4: Recovering failed sql statements in Step 2.
func UpdateSongFreaks() {
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("sf", SiteConfigPath)
siteConf.NConcurrent = 20
dna.PanicError(err)
// Update new songs
state := NewStateHandler(new(sf.APISongFreaksTrack), siteConf, db)
state.TableName = "sfsongs"
Update(state)
// Update "ratings", "songids", "review_author", "review" of song
ids := &[]dna.Int{}
query := dna.Sprintf("SELECT id FROM sfalbums where checktime > '%v' AND array_length(songids, 1) is NULL", time.Now().Format("2006-01-02"))
// dna.Log(query)
err = db.Select(ids, query)
if err != nil {
dna.PanicError(err)
}
idsSlice := dna.IntArray(*ids)
if idsSlice.Length() > 0 {
state = NewStateHandlerWithExtSlice(new(sf.APISongFreaksAlbum), &idsSlice, siteConf, db)
Update(state)
} else {
dna.Log("No new albums found")
}
// Recover failed sql statements
RecoverErrorQueries(SqlErrorLogPath, db)
CountDown(3*time.Second, QuittingMessage, EndingMessage)
db.Close()
}
// UpdateNhaccuatui gets lastest items from nhaccuatui.com.
//
// The update process goes through 6 steps:
// Step 1: Initalizing db connection, loading site config and state handler.
// Step 2: Updating new songs.
// Step 3: Updating new albums.
// Step 4: Updating new videos.
// Step 5: Updating new artists.
// Step 6: Recovering failed sql statements.
func UpdateNhaccuatui() {
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("nct", SiteConfigPath)
dna.PanicError(err)
TIMEOUT_SECS = 60 // for safe
// siteConf.NConcurrent = 15
state := NewStateHandler(new(nct.Song), siteConf, db)
Update(state)
state = NewStateHandler(new(nct.Album), siteConf, db)
Update(state)
state = NewStateHandler(new(nct.Video), siteConf, db)
Update(state)
state = NewStateHandler(new(nct.Artist), siteConf, db)
// Because there is no setting for total concurrent artists failed,
// the default value NCSongFail will be returned.
// Now, it is set to new value
state.SiteConfig.NCSongFail = 1000 // Set concurrency for artists
Update(state)
RecoverErrorQueries(SqlErrorLogPath, db)
CountDown(3*time.Second, QuittingMessage, EndingMessage)
db.Close()
}
// UpdateNhacvui gets lastest items from nhacvui.vn.
// The update process goes through 5 steps:
// Step 1: Initalizing db connection, loading site config and state handler.
// Step 2: Updating new songs.
// Step 3: Updating new albums.
// Step 4: Updating new videos from FoundVideos var.
// Step 5: Recovering failed sql statements.
func UpdateNhacvui() {
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("nv", SiteConfigPath)
dna.PanicError(err)
state := NewStateHandler(new(nv.Song), siteConf, db)
Update(state)
// update album
state = NewStateHandler(new(nv.Album), siteConf, db)
Update(state)
if nv.FoundVideos.Length() > 0 {
state = NewStateHandlerWithExtSlice(new(nv.Video), nv.FoundVideos, siteConf, db)
Update(state)
} else {
dna.Log("No videos found!")
}
RecoverErrorQueries(SqlErrorLogPath, db)
time.Sleep(3 * time.Second)
db.Close()
}
func amRecovery() {
// Recovering failed ambum ids
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("am", SiteConfigPath)
siteConf.NConcurrent = 30
dna.PanicError(err)
// r := NewRange(20987, 30000)
ids := amGetErrIds("./log/http_error.log", 1)
tmp := ids.Unique()
ids = &tmp
if ids.Length() > 0 {
state := NewStateHandlerWithExtSlice(new(am.APIAlbum), ids, siteConf, db)
Update(state)
} else {
dna.Log("No need to recover file")
}
// Recover failed SQL statements
RecoverErrorQueries(SqlErrorLogPath, db)
CountDown(3*time.Second, QuittingMessage, EndingMessage)
db.Close()
}
// UpdateNhacso gets lastest items from nhacso.com.
// The update process goes through 7 steps:
// Step 1: Initalizing db connection, loading site config and state handler.
// Step 2: Updating new songs.
// Step 3: Updating new albums.
// Step 4: Updating new videos.
// Step 5: Updating catagories of new songs.
// Step 6: Updating catagories of new albums.
// Step 7: Recovering failed sql statements.
func UpdateNhacso() {
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("ns", SiteConfigPath)
dna.PanicError(err)
// update song
state := NewStateHandler(new(ns.Song), siteConf, db)
Update(state)
// update album
state = NewStateHandler(new(ns.Album), siteConf, db)
Update(state)
// update video
state = NewStateHandler(new(ns.Video), siteConf, db)
Update(state)
r := NewRange(0, dna.Int(len(*ns.SongGenreList))*ns.LastNPages-1)
siteConf.NConcurrent = 10
state = NewStateHandlerWithRange(new(ns.SongCategory), r, siteConf, db)
Update(state)
state = NewStateHandlerWithRange(new(ns.AlbumCategory), r, siteConf, db)
Update(state)
RecoverErrorQueries(SqlErrorLogPath, db)
time.Sleep(3 * time.Second)
CountDown(3*time.Second, QuittingMessage, EndingMessage)
db.Close()
}
func ExampleSelectMissingIds() {
db, err := sqlpg.Connect(sqlpg.NewSQLConfig("./app.ini"))
dna.PanicError(err)
ids, err := SelectMissingIds("ziartists", &dna.IntArray{5, 6, 7, 8, 9}, db)
dna.PanicError(err)
dna.Log(ids)
db.Close()
// Output:
// &dna.IntArray{8}
}
func amUpdateNewAlbums() {
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("am", SiteConfigPath)
siteConf.NConcurrent = 10
dna.PanicError(err)
state := NewStateHandler(new(am.APIAlbum), siteConf, db)
state.TableName = "amalbums"
Update(state)
CountDown(5*time.Second, QuittingMessage, EndingMessage)
db.Close()
}
func ExampleGetMovie() {
movie, err := GetMovie(5561)
dna.PanicError(err)
movie.Checktime = time.Date(2013, time.November, 21, 0, 0, 0, 0, time.UTC)
dna.LogStruct(movie)
// Output:
// Id : 5561
// Title : "The Scandal - Sự Thật Nghiệt Ngã - Tập 36/36"
// AnotherTitle : "The Scandal"
// ForeignTitle : "The Scandal"
// VnTitle : "Sự Thật Nghiệt Ngã"
// Topics : dna.StringArray{"Hàn quốc", "Gia Đình", "Lãng Mạn"}
// Actors : dna.StringArray{"Kim Jae Won", "Jung Yoon Suk", "Kim Hwi Soo"}
// Directors : dna.StringArray{"Kim Jin Man", "Park Jae Bum"}
// Countries : dna.StringArray{"Hàn Quốc"}
// Description : "Bộ phim kể về câu chuyện của một cảnh sát hình sự (Kim Jae Won - Ha Eun Joong) tình cờ biết được sự thật, người cha của anh hiện tại thật ra là kẻ bắt cóc và chính ông đã bắt cóc anh lúc anh còn nhỏ. Cuộc sống của anh đã có nhiều thay đổi khi gặp gỡ người phụ nữ Woo Ah Mi (Jo Yoon Hee), một người mẹ đơn thân 26 tuổi. Kể từ giây phút ấy, anh bắt đầu hành trình hàn gắn nỗi đau trong trái tim mình cùng Woo Ah Mi."
// YearReleased : 2013
// IMDBRating : dna.IntArray{0, 0}
// Similars : dna.IntArray{}
// Thumbnail : "http://t.hdviet.com/thumbs/214x321/ca8507356775119a9f183e2abe95fe02.jpg"
// MaxResolution : 720
// IsSeries : true
// SeasonId : 0
// Seasons : dna.IntArray{}
// Epid : 0
// CurrentEps : 36
// MaxEp : 36
// Checktime : "2013-11-21 00:00:00"
}
func ExampleGetAlbum() {
_, err := GetAlbum(84935)
if err == nil {
panic("Album has to have an error")
} else {
if err.Error() != "Keeng - Album ID: 84935 not found" {
panic("Wrong error message! GOT: " + err.Error())
}
}
album, err := GetAlbum(86694)
dna.PanicError(err)
album.Checktime = time.Date(2013, time.November, 21, 0, 0, 0, 0, time.UTC)
if album.Plays < 36049 {
panic("Wrong play")
}
if album.Coverart == "" {
panic("WRong covert")
}
album.Coverart = "http://media3.keeng.vn:8082/medias/images/images_thumb/f_medias_6/album/image/2013/12/23/b6abfc53bcb1f5ecd1c7a2ee7f6f5292a79e12e2_103_103.jpg"
album.Plays = 36049
dna.LogStruct(album)
// Output:
// Id : 86694
// Key : "09BYXAMW"
// Title : "100 Hit Nhạc Việt 2013 (Part 1)"
// Artists : dna.StringArray{"Mỹ Tâm", "Thu Minh", "The Men", "Hồ Quang Hiếu", "Khởi My", "Miu Lê", "Hồng Dương M4U", "Bảo Thy", "Noo Phước Thịnh", "Đông Nhi"}
// Nsongs : 49
// Plays : 36049
// Coverart : "http://media3.keeng.vn:8082/medias/images/images_thumb/f_medias_6/album/image/2013/12/23/b6abfc53bcb1f5ecd1c7a2ee7f6f5292a79e12e2_103_103.jpg"
// Description : ""
// Songids : dna.IntArray{1771701, 1907293, 1922861, 1786476, 1764324, 1794339, 1944931, 1944729, 1759706, 1787817, 1944090, 1802630, 1907296, 1790298, 1771702, 1776165, 1779905, 1963988, 1962277, 1788903, 1965592, 1954418, 1790245, 1788955, 1963617, 1771761, 1965428, 1968399, 1966605, 1821302, 1787318, 1959758, 1747128, 1968036, 1760361, 1967779, 1794784, 1967780, 1919261, 1922843, 1965180, 1785077, 1958497, 1794641, 1966916, 1780127, 1773989, 1967851, 1958909}
// DateCreated : "2013-12-23 00:00:00"
// Checktime : "2013-11-21 00:00:00"
}
// UpdateZing gets lastest items from mp3.zing.vn.
// The update process goes through 8 steps:
// Step 1: Initalizing db connection, loading site config and state handler
// Step 1: Updating new songs
// Step 3: Updating new albums
// Step 4: Updating new videos
// Step 5: Updating new artists
// Step 6: Updating new songids found in new albums
// Step 7: Updating new tvs
// Step 8: Recovering failed sql statements
func UpdateZing() {
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("zi", SiteConfigPath)
dna.PanicError(err)
// update song
state := NewStateHandler(new(zi.Song), siteConf, db)
Update(state)
// update album
state = NewStateHandler(new(zi.Album), siteConf, db)
Update(state)
// update video
state = NewStateHandler(new(zi.Video), siteConf, db)
Update(state)
// update artist
state = NewStateHandler(new(zi.Artist), siteConf, db)
Update(state)
// update new songids found in albums
dna.Log("Update new songs from albums")
ids := utils.SelectNewSidsFromAlbums("zialbums", time.Now(), db)
nids, err := utils.SelectMissingIds("zisongs", ids, db)
if err != nil {
dna.Log(err.Error())
} else {
if nids != nil && nids.Length() > 0 {
state = NewStateHandlerWithExtSlice(new(zi.Song), nids, siteConf, db)
Update(state)
} else {
dna.Log("No new songs found")
}
}
state = NewStateHandler(new(zi.TV), siteConf, db)
Update(state)
RecoverErrorQueries(SqlErrorLogPath, db)
time.Sleep(3 * time.Second)
db.Close()
}
// UpdateChacha updates chacha.vn.
// The update process goes through 5 steps:
// Step 1: Initalizing db connection, loading site config and state handler.
// Step 1: Updating new songs.
// Step 3: Updating new albums.
// Step 4: Updating new videos.
// Step 5: Recovering failed sql statements.
func UpdateChacha() {
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("cc", SiteConfigPath)
dna.PanicError(err)
state := NewStateHandler(new(cc.Song), siteConf, db)
Update(state)
// update album
state = NewStateHandler(new(cc.Album), siteConf, db)
Update(state)
state = NewStateHandler(new(cc.Video), siteConf, db)
Update(state)
RecoverErrorQueries(SqlErrorLogPath, db)
time.Sleep(3 * time.Second)
db.Close()
}
// UpdateKeeng gets lastest items from keeng.com.
// The update process goes through 6 steps:
// Step 1: Initalizing db connection, loading site config and state handler.
// Step 2: Updating new artists.
// Step 3: Updating new songs.
// Step 4: Updating new albums.
// Step 5: Updating new videos.
// Step 6: Recovering failed sql statements.
func UpdateKeeng() {
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("ke", SiteConfigPath)
dna.PanicError(err)
state := NewStateHandler(new(ke.Artist), siteConf, db)
Update(state)
state = NewStateHandler(new(ke.Song), siteConf, db)
Update(state)
state = NewStateHandler(new(ke.Album), siteConf, db)
Update(state)
state = NewStateHandler(new(ke.Video), siteConf, db)
Update(state)
RecoverErrorQueries(SqlErrorLogPath, db)
CountDown(3*time.Second, QuittingMessage, EndingMessage)
db.Close()
}
func UpdateLyricFind(vpnAppName, vpnAppUrl dna.String, estConDuration time.Duration) {
cmd := <-startVPN(vpnAppName, vpnAppUrl)
CountDown(estConDuration*time.Second, "Establishing VPN connection. Estimated time remaining:", "")
err := testingSongs()
if err != nil {
dna.Log("Error occurs: ", err.Error())
dna.Log("Operation aborted!")
} else {
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("lf", SiteConfigPath)
dna.PanicError(err)
state := NewStateHandler(new(lf.Song), siteConf, db)
Update(state)
RecoverErrorQueries(SqlErrorLogPath, db)
CountDown(3*time.Second, QuittingMessage, EndingMessage)
db.Close()
}
stopVPN(vpnAppName, cmd)
}
func LoadLastedChecktime(filePath dna.String) (map[dna.String]time.Time, error) {
ret := make(map[dna.String]time.Time)
cf, err := cfg.LoadConfigFile(filePath)
if err != nil {
return nil, err
}
for site, _ := range RefTables {
section, err := cf.GetSection(site)
if err != nil {
dna.PanicError(err)
}
for postfix, timeStr := range section {
if timeStr != "" {
t, err := time.Parse(utils.DefaultTimeLayout, timeStr.String())
if err == nil {
ret[site+postfix] = t
} else {
dna.PanicError(err)
}
}
}
}
return ret, nil
}
func ExampleGetUsersWithInitialId() {
UsersMagnitude = 10
users, err := GetUsersWithInitialId(1)
if err != nil {
dna.PanicError(err)
} else {
if len(users.List) != 9 {
dna.Log("Complete")
} else {
dna.Log("Not complete")
}
}
// Output:
// Complete
}
func ExampleGetLyric() {
_, err := GetLyric(1972114)
if err == nil {
panic("Lyric has to have an error")
} else {
if err.Error() != "Keeng - Lyric ID: 1972114 not found" {
panic("Wrong error message!")
}
}
album, err := GetLyric(1966613)
dna.PanicError(err)
dna.LogStruct(album)
// Output:
// Id : 1966613
// Content : "<p>\r\n\tNgày ấy ta yêu nhau,em đã biết mang thương đau<br />\r\n\tNhưng trái tim em lỡ yêu anh rồi<br />\r\n\tAnh chỉ muốn lấp khoảng trống trong lòng<br />\r\n\tNào có yêu thương em gì đâu.<br />\r\n\t<br />\r\n\tGiờ đây anh ra đi, không một câu chia ly<br />\r\n\tAnh chỉ xem em giống như nhân tình<br />\r\n\tKhông thể ở bên nhau suốt đời<br />\r\n\tChỉ tìm vui nhau trong phút giây.<br />\r\n\t<br />\r\n\tNgười yêu ơi, hãy nói em nghe<br />\r\n\tSao anh vô tình lặng thinh quay bước<br />\r\n\tNgười yêu ơi, phút chốc cô đơn bơ vơ<br />\r\n\tĐôi chân lạc loài về đâu?<br />\r\n\t<br />\r\n\t[ĐK:]<br />\r\n\tLòng em tự hỏi quá yếu mềm hay em đã yêu cuồng si<br />\r\n\tĐã trao cho anh tất cả tình yêu<br />\r\n\t<span style=\"font-size: 10%; line-height: 1px; color: #EEFFFF;\">Xin Một Lần Được Yêu (Ballad Version) lyrics on ChiaSeNhac.com</span><br />\r\n\tGiấc mơ đôi ta xây nơi thiên đường<br />\r\n\tGiờ mình em nghe mưa mang anh rời xa.<br />\r\n\t<br />\r\n\tNhiều khi muốn níu lấy tình, nhưng đã quá xa tầm tay<br />\r\n\tGió ơi hãy mang hết đi buồn đau<br />\r\n\tNếu em được quay về phút ban đầu<br />\r\n\tEm xin chấp nhận lấy thêm một niềm đau.</p>"
}
func ExampleGetVideo() {
_, err := GetVideo(203952)
if err == nil {
panic("Video has to have an error")
} else {
if err.Error() != "Keeng - Video ID: 203952 not found" {
panic("Wrong error message!")
}
}
video, err := GetVideo(215236)
dna.PanicError(err)
video.Checktime = time.Date(2013, time.November, 21, 0, 0, 0, 0, time.UTC)
if video.Plays < 6688 {
panic("Wrong play")
}
if video.Thumbnail == "" {
panic("WRong covert")
}
video.Plays = 6688
video.Thumbnail = "http://media3.keeng.vn:8082/medias/images/images_thumb/f_medias_6/video/images/2013/12/24/e2d4ef038c1922a23d4f82b7d4c06972638e8753_147_83.jpg"
dna.LogStruct(video)
// Output:
// Id : 215236
// Key : "Y45E8EPS"
// Title : "Ông Bà Già Noel"
// Artists : dna.StringArray{"Kevin Sôcôla", "Thân Nhật Huy"}
// Plays : 6688
// ListenType : 0
// Link : "http://media2.keeng.vn/medias/video/2013/12/24/bc2137b2029cccf185b657aa88fc1164f5d1d605_mp4_640_360.mp4"
// IsDownload : 0
// DownloadUrl : "http://media2.keeng.vn/medias/video/2013/12/24/bc2137b2029cccf185b657aa88fc1164f5d1d605.mp4"
// RingbacktoneCode : ""
// RingbacktonePrice : 0
// Price : 0
// Copyright : 0
// CrbtId : 0
// Thumbnail : "http://media3.keeng.vn:8082/medias/images/images_thumb/f_medias_6/video/images/2013/12/24/e2d4ef038c1922a23d4f82b7d4c06972638e8753_147_83.jpg"
// DateCreated : "2013-12-24 00:00:00"
// Checktime : "2013-11-21 00:00:00"
}
func (alca *AlbumCategory) Save(db *sqlpg.DB) error {
var last error
var aids = dna.IntArray{}
albums := &[]Album{}
for _, album := range *(alca.Albums) {
aids.Push(album.Id)
// dna.Log(album)
}
query := "SELECT id, topics, genres from nsalbums WHERE id IN (" + aids.Join(",") + ")"
// dna.Log(query)
err := db.Select(albums, query)
if err != nil {
dna.Log(query, alca, *alca.Albums)
dna.PanicError(err)
}
for _, album := range *(alca.Albums) {
foundIndex := 0
for j, anotherAlbum := range *(albums) {
if album.Id == anotherAlbum.Id {
foundIndex = j
}
}
if foundIndex < len(*albums) {
cat := album.Category.Concat((*albums)[foundIndex].Topics).Concat((*albums)[foundIndex].Genres).Unique()
album.Category = cat.Filter(func(v dna.String, i dna.Int) dna.Bool {
if v != "" {
return true
} else {
return false
}
})
}
last = db.Update(album, "id", "category")
}
return last
}
// SelectNewSidsFromAlbums returns a slice of songids from a table since the last specified time.
// The table has to be album type and has a column called songids.
func SelectNewSidsFromAlbums(tblName dna.String, lastTime time.Time, db *sqlpg.DB) *dna.IntArray {
idsArrays := &[]dna.IntArray{}
year := dna.Sprintf("%v", lastTime.Year())
month := dna.Sprintf("%d", lastTime.Month())
day := dna.Sprintf("%v", lastTime.Day())
checktime := dna.Sprintf("'%v-%v-%v'", year, month, day)
query := dna.Sprintf("SELECT songids FROM %s WHERE checktime >= %s", tblName, checktime)
// dna.Log(query)
err := db.Select(idsArrays, query)
dna.PanicError(err)
ids := &dna.IntArray{}
if idsArrays != nil {
for _, val := range *idsArrays {
for _, id := range val {
ids.Push(id)
}
}
return ids
} else {
return nil
}
}
// UpdateChiasenhac updates new songs, videos and albums.
// New albums are created from new songs fetched.
// After getting new songs or videos, it runs re-fetching
// procedure until there is no missing songs, videos or
// the procesure runs 10 times.
//
// fillMode : only update records whose titles are empty.
//
// The update process goes through 6 steps:
// Step 1: Initializing db connection, site config, state handler and finding max id
// Step 2: Getting new songs from max id found in the table. If videos are found, then it inserts the videos into a video table.
// Step 3: Getting missing ids from last 10000 songids.
// Step 4: Refetching errors of songs having empty titles.
// Step 5: Recovering fail sql statement.
// Step 6: Creating new albums from new songs found.
func UpdateChiasenhac(fillMode dna.Bool) {
// note: songid 1172662 1172663 1172664 are not continuos
var errCount = 0
var missingCount = 0
var lastTotalMissing dna.Int = 0
var done = false
TIMEOUT_SECS = 240
dna.Log("TIMEOUT:", TIMEOUT_SECS)
// Step 1
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("csn", SiteConfigPath)
dna.PanicError(err)
// Getting LastSongId for SaveNewAlbums func
csn.LastSongId, err = utils.GetMaxId("csnsongs", db)
dna.PanicError(err)
dna.Log("Max ID:", csn.LastSongId)
// Step 2: Getting both new songs and videos and
// inserting into appropriate tables respectively.
if fillMode == false {
state := NewStateHandler(new(csn.Song), siteConf, db)
Update(state)
// Step 3: Fetching missing ids.
// It stops when total loop is 3 or the last total ids equals new total ones
dna.Log(dna.String("\nGetting missing ids from last 10000 songids").ToUpperCase())
for missingCount < 3 && done == false {
temp := updateMissingIds(db, siteConf, 10000)
if temp == lastTotalMissing {
done = true
}
lastTotalMissing = temp
missingCount += 1
}
}
// SET LAST SONGID
// csn.LastSongId = 1203603
// Step 4: Re-fetching err songs
db.Ping()
dna.Log(dna.String("\nRe-fetching err from last 10000 songs having EMPTY titles & ID > " + csn.LastSongId.ToString()).ToUpperCase())
for false == updateEmptyTitles(db, siteConf, csn.LastSongId) && errCount < 10 {
db.Ping()
errCount += 1
dna.Log("RE-FETCHING ROUND:", errCount)
}
dna.Log("Re-fetching error done!")
// Step 5: Recovering failed sql statments
RecoverErrorQueries(SqlErrorLogPath, db)
if fillMode == false {
// Step 6: Saving new abums
dna.Log("Finding and saving new albums from last songid:", csn.LastSongId)
nAlbums, err := csn.SaveNewAlbums(db)
if err != nil {
dna.Log(err.Error())
} else {
dna.Log("New albums inserted:", nAlbums)
}
}
dna.Log("SET TIMEOUT TO DEFAULT: 8s")
TIMEOUT_SECS = 8
time.Sleep(2 * time.Second)
db.Close()
}
// UpdateHDViet gets lastest items from hdviet.com.
//
// The update process goes through 8 steps and 2 parts:
//
// PART I: FINDING AND UPDATING NEW EPS OF MOVIES IN DB
// Step 1: Initalizing db connection, loading site config and state handler.
// Step 2: Finding all movies possibly having new episodes in DB
// Step 3: Finding all movies found in Step 2 actually having new episodes available from source website.
// Step 4: Checking consitency of movies' new found episodes from the source.
// Step 5: Getting new episode data from the source, save them to DB and update current_eps field of movies having new eps.
//
// PART II: GETTING NEW MOVIES FROM HDVIET SITE.
// Step 6: Find newest movies from the source.
// Step 7: Updating newest episodes from newest found movies in Step 6.
// Step 8: Recovering failed sql statements.
func UpdateHDViet() {
var mvTable = dna.String("hdvmovies")
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("hdv", SiteConfigPath)
dna.PanicError(err)
TIMEOUT_SECS = 100
// PART 1: UPDATING NEW EPISODES OF MOVIES IN DB
// STEP 2: Finding all movies possibly having new episodes in DB.
movieCurrentEps, err := hdv.GetMoviesCurrentEps(db, mvTable)
dna.PanicError(err)
newEpisodeKeys := dna.IntArray{}
for movieid, currentEps := range movieCurrentEps {
newEpisodeKeys.Push(hdv.ToEpisodeKey(movieid, currentEps))
}
newEpisodeKeys.Sort()
// STEP 3: Checking and getting new episodes if available from source website.
state := NewStateHandlerWithExtSlice(new(hdv.EpUpdater), &newEpisodeKeys, siteConf, db)
Update(state)
// STEP 4:Checking consitency of new found episodes of movies from the source.
movieIdList := dna.IntArray{}
for _, epKey := range hdv.LastestEpisodeKeyList {
mvid, _ := hdv.ToMovieIdAndEpisodeId(epKey)
movieIdList.Push(mvid)
}
movieIdList = movieIdList.Unique()
dna.Log("\nNumber of movies: ", movieIdList.Length(), "having new episodes:", hdv.LastestEpisodeKeyList.Length())
if movieIdList.Length() != dna.Int(len(hdv.LastestMovieCurrentEps)) {
dna.Log("LastestEpisodeKeyList & LastestMovieCurrentEps do not match! GOT:", movieIdList.Length(), len(hdv.LastestMovieCurrentEps))
}
// STEP 5: Getting new episode data from the source, save them to DB and update current_eps field of movies having new eps.
if hdv.LastestEpisodeKeyList.Length() > 0 {
state = NewStateHandlerWithExtSlice(new(hdv.Episode), &hdv.LastestEpisodeKeyList, siteConf, db)
Update(state)
RecoverErrorQueries(SqlErrorLogPath, db)
// dna.Log(hdv.LastestEpisodeKeyList)
hdv.SaveLastestMovieCurrentEps(db, mvTable, SQLERROR)
RecoverErrorQueries(SqlErrorLogPath, db)
} else {
dna.Log("No new episodes found. Update operation has been aborted!")
}
// PART 2: UPDATING NEW MOVIES FROM hdv site.
// STEP 6: Find newest movies from sources.
db.Ping()
state = NewStateHandler(new(hdv.Movie), siteConf, db)
Update(state)
// STEP 7: Updating newest episodes from newest found movies in Step 6.
hdv.EpisodeKeyList = hdv.EpisodeKeyList.Unique()
if hdv.EpisodeKeyList.Length() > 0 {
state = NewStateHandlerWithExtSlice(new(hdv.Episode), &hdv.EpisodeKeyList, siteConf, db)
Update(state)
} else {
dna.Log("No new movies found")
}
// STEP 8: Recovering failed sql statements.
RecoverErrorQueries(SqlErrorLogPath, db)
CountDown(3*time.Second, QuittingMessage, EndingMessage)
db.Close()
}
func ExampleGetSong() {
_, err := GetSong(1966782)
if err == nil {
panic("Song has to have an error")
} else {
if err.Error() != "Keeng - Song ID: 1966782 not found" {
panic("Wrong error message!")
}
}
song, err := GetSong(1967829)
dna.PanicError(err)
song.Checktime = time.Date(2013, time.November, 21, 0, 0, 0, 0, time.UTC)
if song.Plays < 126222 {
panic("Wrong play")
}
if song.Link == "" {
panic("song link is empty")
}
if song.MediaUrlMono == "" {
panic("song media url is empty")
}
if song.MediaUrlPre == "" {
panic("song medial url pre is empty")
}
if song.DownloadUrl == "" {
panic("song download url is empty")
}
if song.Coverart == "" {
panic("WRong covert")
}
if song.Coverart310 == "" {
panic("WRong coverart 310")
}
song.Link = "http://media2.keeng.vn/medias/audio/2013/12/19/a5cc9183b876c288f099e97aecc189f004b2137a_128.mp3"
song.MediaUrlMono = "http://media2.keeng.vn/medias/audio/2013/12/09/54ec35cd058b422c1b4b9f812cdb5726dee66f9c_24.mp3"
song.MediaUrlPre = "http://media2.keeng.vn/medias/audio/2013/12/19/a5cc9183b876c288f099e97aecc189f004b2137a_128.mp3"
song.DownloadUrl = "http://media2.keeng.vn/medias/audio/2013/12/19/a5cc9183b876c288f099e97aecc189f004b2137a.mp3"
song.Plays = 126222
song.Coverart = "http://media3.keeng.vn:8082/medias/images/images_thumb/f_medias2/singer/2013/01/21/673a437247a1f039bbde97119385c16524131f8f_103_103.jpg"
song.Coverart310 = "http://media3.keeng.vn:8082/medias/images/images_thumb/f_medias2/singer/2013/01/21/673a437247a1f039bbde97119385c16524131f8f_310_310.jpg"
song.DateCreated = time.Date(2013, time.December, 9, 0, 0, 0, 0, time.UTC)
dna.LogStruct(song)
// Output:
// Id : 1967829
// Key : "LKD633FY"
// Title : "Em Phải Làm Sao"
// Artists : dna.StringArray{"Mỹ Tâm"}
// Plays : 126222
// ListenType : 0
// HasLyric : true
// Lyric : "<p>\r\n\t<span class=\"lyric\">Làm sao để cho tình ta giờ đây vui như lúc đầu<br />\r\n\tLàm sao để cho nụ hôn nồng say thôi mang nỗi sầu<br />\r\n\tLàm sao để cho từng đêm mình em thôi không đớn đau vì nhau<br />\r\n\tLàm sao để anh sẽ tin rằng em chỉ yêu mỗi anh.<br />\r\n\t<br />\r\n\tLàm sao để cho tình anh giờ đây thôi không hững hờ<br />\r\n\tLàm sao để cho tình không là mơ em thôi thẫn thờ<br />\r\n\tVì em ngày xưa từng mang lầm lỡ nên đâu dám mơ mộng gì<br />\r\n\tLàm sao để cho thời gian đừng bôi xóa đi ngày thơ.<br />\r\n\t<br />\r\n\t[ĐK:]<br />\r\n\tNgười hỡi, người có biết em vẫn luôn cần anh<br />\r\n\tĐể sưởi ấm cho trái tim mỏng manh<br />\r\n\tMùa đông có anh sẽ qua thật nhanh<br />\r\n\tLời yêu thương đó, cố giữ trên môi<br />\r\n\tMà sao tiếng yêu trong anh như mây chiều trôi<br />\r\n\tTình mình sao cứ mãi xa xôi...<br />\r\n\t<br />\r\n\tNgười hỡi, người có biết em vẫn luôn còn đây<br />\r\n\tDù bao giấc mơ cuốn theo làn mây<br />\r\n\tDù anh có quên hết bao nồng say<br />\r\n\tTình em vẫn thế, dẫu chẳng ai hay<br />\r\n\tVà bao ước mơ bên anh nay xa tầm tay<br />\r\n\tNguyện rằng em sẽ mãi yêu không đổi thay.</span></p>"
// Link : "http://media2.keeng.vn/medias/audio/2013/12/19/a5cc9183b876c288f099e97aecc189f004b2137a_128.mp3"
// MediaUrlMono : "http://media2.keeng.vn/medias/audio/2013/12/09/54ec35cd058b422c1b4b9f812cdb5726dee66f9c_24.mp3"
// MediaUrlPre : "http://media2.keeng.vn/medias/audio/2013/12/19/a5cc9183b876c288f099e97aecc189f004b2137a_128.mp3"
// DownloadUrl : "http://media2.keeng.vn/medias/audio/2013/12/19/a5cc9183b876c288f099e97aecc189f004b2137a.mp3"
// IsDownload : 1
// RingbacktoneCode : "7333357"
// RingbacktonePrice : 5000
// Price : 1000
// Copyright : 1
// CrbtId : 0
// Coverart : "http://media3.keeng.vn:8082/medias/images/images_thumb/f_medias2/singer/2013/01/21/673a437247a1f039bbde97119385c16524131f8f_103_103.jpg"
// Coverart310 : "http://media3.keeng.vn:8082/medias/images/images_thumb/f_medias2/singer/2013/01/21/673a437247a1f039bbde97119385c16524131f8f_310_310.jpg"
// DateCreated : "2013-12-09 00:00:00"
// Checktime : "2013-11-21 00:00:00"
}
func ExampleGetEpisode() {
// Only to renew ACCESS_TOKEN_KEY
_, err := GetMovie(5571)
dna.PanicError(err)
episode, err := GetEpisode(5571, 1)
dna.PanicError(err)
episode.Checktime = time.Date(2013, time.November, 21, 0, 0, 0, 0, time.UTC)
if episode.PlaylistM3u8.Match(`(?mis)^#EXTM3U\r\n#EXT-X-VERSION:3.+480_hdv_app.m3u8\r\n$`) == false {
panic("PlaylistM3u8 is wrong")
} else {
episode.PlaylistM3u8 = ""
}
if episode.ViSrt.Match(`^77u/MQ0KMDA6MDA.+OgIHLhu5NpLg0KDQo=$`) == false {
panic("Vietnamese subtitle is wrong")
} else {
episode.ViSrt = ""
}
if episode.EnSrt.Match(`^77u/MQ0KMDA6MDA6MDAsNTg.+MA0KKioqDQoNCg==$`) == false {
panic("English subtitle is wrong")
} else {
episode.EnSrt = ""
}
if episode.EpisodeM3u8.Match(`(?mis)^#EXTM3U\r\n#EXT-X-VERSION:3.+#EXT-X-ENDLIST\r\n$`) == false {
panic("M3u8 file is wrong")
} else {
episode.EpisodeM3u8 = ""
}
if episode.LinkPlayBackup.Match(`playlist_480_hdv_app.m3u8$`) == false {
panic("Wrong link backup")
} else {
episode.LinkPlayBackup = "http://f-11.vn-hd.com/c52447879ebfd67168d82f16f1cb374a/onair_2014/Grimm_S03_HDTV_AC3/E001/playlist_480_hdv_app.m3u8"
}
if episode.Link.Match(`playlist_480_hdv_app.m3u8$`) == false {
panic("Wrong link backup")
} else {
episode.Link = "http://f-11.vn-hd.com/c52447879ebfd67168d82f16f1cb374a/onair_2014/Grimm_S03_HDTV_AC3/E001/playlist_480_hdv_app.m3u8"
}
dna.LogStruct(episode)
// Output:
// MovieId : 5571
// EpId : 1
// Title : "Grimm (Season 3) - Tập 1"
// LinkPlayBackup : "http://f-11.vn-hd.com/c52447879ebfd67168d82f16f1cb374a/onair_2014/Grimm_S03_HDTV_AC3/E001/playlist_480_hdv_app.m3u8"
// Link : "http://f-11.vn-hd.com/c52447879ebfd67168d82f16f1cb374a/onair_2014/Grimm_S03_HDTV_AC3/E001/playlist_480_hdv_app.m3u8"
// LinkPlayOther : "http://125.212.216.74/vod/_definst_/smil:mp4_02/store_01_2014/onair_2014/Grimm_S03_HDTV_AC3/E001/Grimm_S03_HDTV_AC3_E001.smil/Manifest"
// SubtitleExt : dna.StringArray{"http://s.vn-hd.com/mp4_02/store_01_2014/onair_2014/Grimm_S03_HDTV_AC3/E001/Grimm_S03_HDTV_AC3_E001_VIE.srt", "http://s.vn-hd.com/mp4_02/store_01_2014/onair_2014/Grimm_S03_HDTV_AC3/E001/Grimm_S03_HDTV_AC3_E001_ENG.srt"}
// SubtitleExtSe : dna.StringArray{"http://s.vn-hd.com/mp4_02/store_01_2014/onair_2014/Grimm_S03_HDTV_AC3/E001/Grimm_S03_HDTV_AC3_E001_VIE.srt", "http://s.vn-hd.com/mp4_02/store_01_2014/onair_2014/Grimm_S03_HDTV_AC3/E001/Grimm_S03_HDTV_AC3_E001_ENG.srt"}
// Subtitle : dna.StringArray{"", "http://s.vn-hd.com/mp4_02/store_01_2014/onair_2014/Grimm_S03_HDTV_AC3/E001/Grimm_S03_HDTV_AC3_E001_ENG.srt"}
// EpisodeId : 17
// Audiodub : 0
// Audio : 0
// Season : "[{\"MovieID\":\"473\",\"Name\":\"Grimm (Season 1)\"},{\"MovieID\":\"2364\",\"Name\":\"Grimm (Season 2)\"}]"
// PlaylistM3u8 : ""
// ViSrt : ""
// EnSrt : ""
// EpisodeM3u8 : ""
// Checktime : "2013-11-21 00:00:00"
}
func ExampleGetUsers() {
users, err := GetUsers(dna.IntArray{438})
if err != nil {
dna.PanicError(err)
} else {
user := users.List[0]
user.Checktime = time.Date(2013, time.November, 21, 0, 0, 0, 0, time.UTC)
if user.FriendTotal < 694 {
panic("Wrong user's FriendTotal")
}
if user.Point < 22932 {
panic("Wrong user's Point")
}
if user.VipTotal < 310 {
panic("Wrong user's VipTotal")
}
if user.FriendAvatar == "" {
panic("Wrong user's FriendAvatar")
}
if user.Avatar == "" {
panic("Wrong user's Avatar")
}
if user.VipAvatar == "" {
panic("Wrong user's VipAvatar")
}
if user.CoverUrl == "" {
panic("Wrong user's CoverUrl")
}
user.FriendTotal = 694
user.Point = 22932
user.VipTotal = 310
user.FriendAvatar = "http://s120.avatar.zdn.vn/6/7/8/a/dzobanbe_100_131.jpg"
user.Avatar = "http://s120.avatar.zdn.vn/6/7/8/a/dzobanbe_100_131.jpg"
user.VipAvatar = "http://s120.avatar.zdn.vn/9/4/3/c/macma2005_100_262.jpg"
user.CoverUrl = "http://stc.thongdiephay.zdn.vn/bd54328839-9927.jpg"
dna.LogStruct(user)
}
// Output:
// Id : 438
// Email : "*****@*****.**"
// Username : "******"
// DisplayName : "Nguyễn Văn Đức Trọng"
// Birthday : "1981-07-28"
// BirthdayType : 1
// Gender : 0
// Mobile : dna.StringArray{"84937388269"}
// FriendTotal : 694
// FriendBlock : "false"
// FriendAvatar : "http://s120.avatar.zdn.vn/6/7/8/a/dzobanbe_100_131.jpg"
// ProfilePoint : 0
// StatusWall : "Tình yêu của anh đối với em giống như giá xăng vậy càng lúc càng tăng."
// GoogleId : "dzobanbe"
// YahooId : "dzobanbe"
// Point : 22932
// Avatar : "http://s120.avatar.zdn.vn/6/7/8/a/dzobanbe_100_131.jpg"
// VipTotal : 310
// VipBlock : "false"
// VipAvatar : "http://s120.avatar.zdn.vn/9/4/3/c/macma2005_100_262.jpg"
// IsNull : false
// Status : 0
// FeedWriteWallAll : false
// FeedViewWallAll : true
// CoverUrl : "http://stc.thongdiephay.zdn.vn/bd54328839-9927.jpg"
// Checktime : "2013-11-21 00:00:00"
}
// Update gets item from sites and save them to database
func Update(state *StateHandler) *Counter {
CheckStateHandler(state)
var (
counter *Counter = NewCounterWithStateHandler(state)
idcFormat dna.String
cData dna.String
idc *terminal.Indicator
bar *terminal.ProgressBar
errChannel chan bool = make(chan bool)
tableName dna.String = state.GetTableName()
startupFmt dna.String = "Update %v - Cid:%v - Pat:%v - Ncf:%v - NCon:%v"
)
// 3rd pattern: callind GetCid() wil invoke error
INFO.Println(dna.Sprintf(startupFmt, tableName, state.Cid, state.GetPattern(), state.GetNCFail(), state.SiteConfig.NConcurrent))
if utils.IsValidTable(tableName, state.GetDb()) == false {
tableName = ""
}
if state.GetPattern() == 1 {
idcFormat = " $indicator %v|cid:%v|cf:%v" // cid: current id, cf: continuous failure count
idc = terminal.NewIndicatorWithTheme(terminal.ThemeDefault)
// Getting maxid from an item's table
id, err := utils.GetMaxId(tableName, state.GetDb())
dna.PanicError(err)
state.SetCid(id)
} else {
bar = getUpdateProgressBar(counter.Total, tableName)
}
// Config.NConcurrent
for i := dna.Int(0); i < state.SiteConfig.NConcurrent; i++ {
go atomicUpdate(errChannel, state)
}
for state.IsComplete() == false {
hasError := <-errChannel
counter.Tick(dna.Bool(hasError))
switch state.GetPattern() {
case 1:
if counter.GetCFail() == state.GetNCFail() {
state.SetCompletion()
}
idc.Show(dna.Sprintf(idcFormat, counter, state.GetCid(), counter.GetCFail()))
case 2:
if counter.GetCount() == state.GetRange().Total {
state.SetCompletion()
}
cData = dna.Sprintf("%v | Ncc:%v | Cid:%v", counter, state.GetNcCount(), state.GetCid())
bar.Show(counter.GetCount(), cData, cData.Replace("|", "-"))
case 3:
if counter.GetCount() == state.GetExtSlice().Length() {
state.SetCompletion()
}
cData = dna.Sprintf("%v | Ncc:%v | Cid:%v", counter, state.GetNcCount(), state.GetCid())
bar.Show(counter.GetCount(), cData, cData.Replace("|", "-"))
}
}
if state.GetPattern() == 1 {
idc.Close(dna.Sprintf("$indicator Complete updating %v!", tableName))
}
INFO.Printf("[%v] %v\n", tableName, counter.FinalString())
// Delay 2s to ensure all the goroutines left finish it processed before sqlpg.DB closed
// time.Sleep(2 * time.Second)
return counter
}