func amRecovery() {
// Recovering failed ambum ids
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("am", SiteConfigPath)
siteConf.NConcurrent = 30
dna.PanicError(err)
// r := NewRange(20987, 30000)
ids := amGetErrIds("./log/http_error.log", 1)
tmp := ids.Unique()
ids = &tmp
if ids.Length() > 0 {
state := NewStateHandlerWithExtSlice(new(am.APIAlbum), ids, siteConf, db)
Update(state)
} else {
dna.Log("No need to recover file")
}
// Recover failed SQL statements
RecoverErrorQueries(SqlErrorLogPath, db)
CountDown(3*time.Second, QuittingMessage, EndingMessage)
db.Close()
}
// UpdateNhacso gets lastest items from nhacso.com.
// The update process goes through 7 steps:
// Step 1: Initalizing db connection, loading site config and state handler.
// Step 2: Updating new songs.
// Step 3: Updating new albums.
// Step 4: Updating new videos.
// Step 5: Updating catagories of new songs.
// Step 6: Updating catagories of new albums.
// Step 7: Recovering failed sql statements.
func UpdateNhacso() {
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("ns", SiteConfigPath)
dna.PanicError(err)
// update song
state := NewStateHandler(new(ns.Song), siteConf, db)
Update(state)
// update album
state = NewStateHandler(new(ns.Album), siteConf, db)
Update(state)
// update video
state = NewStateHandler(new(ns.Video), siteConf, db)
Update(state)
r := NewRange(0, dna.Int(len(*ns.SongGenreList))*ns.LastNPages-1)
siteConf.NConcurrent = 10
state = NewStateHandlerWithRange(new(ns.SongCategory), r, siteConf, db)
Update(state)
state = NewStateHandlerWithRange(new(ns.AlbumCategory), r, siteConf, db)
Update(state)
RecoverErrorQueries(SqlErrorLogPath, db)
time.Sleep(3 * time.Second)
CountDown(3*time.Second, QuittingMessage, EndingMessage)
db.Close()
}
// UpdateNhacvui gets lastest items from nhacvui.vn.
// The update process goes through 5 steps:
// Step 1: Initalizing db connection, loading site config and state handler.
// Step 2: Updating new songs.
// Step 3: Updating new albums.
// Step 4: Updating new videos from FoundVideos var.
// Step 5: Recovering failed sql statements.
func UpdateNhacvui() {
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("nv", SiteConfigPath)
dna.PanicError(err)
state := NewStateHandler(new(nv.Song), siteConf, db)
Update(state)
// update album
state = NewStateHandler(new(nv.Album), siteConf, db)
Update(state)
if nv.FoundVideos.Length() > 0 {
state = NewStateHandlerWithExtSlice(new(nv.Video), nv.FoundVideos, siteConf, db)
Update(state)
} else {
dna.Log("No videos found!")
}
RecoverErrorQueries(SqlErrorLogPath, db)
time.Sleep(3 * time.Second)
db.Close()
}
// UpdateSongFreaks gets lastest songs,albums,artists and videos from songfreaks.com
// The update process goes through 4 steps:
// Step 1: Initalizing db connection, loading site config and state handler.
// Step 2: Finding new songs, insert new albums,artists and videos if found.
// Step 3: Updating found new albums in Step 2.
// Step 4: Recovering failed sql statements in Step 2.
func UpdateSongFreaks() {
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("sf", SiteConfigPath)
siteConf.NConcurrent = 20
dna.PanicError(err)
// Update new songs
state := NewStateHandler(new(sf.APISongFreaksTrack), siteConf, db)
state.TableName = "sfsongs"
Update(state)
// Update "ratings", "songids", "review_author", "review" of song
ids := &[]dna.Int{}
query := dna.Sprintf("SELECT id FROM sfalbums where checktime > '%v' AND array_length(songids, 1) is NULL", time.Now().Format("2006-01-02"))
// dna.Log(query)
err = db.Select(ids, query)
if err != nil {
dna.PanicError(err)
}
idsSlice := dna.IntArray(*ids)
if idsSlice.Length() > 0 {
state = NewStateHandlerWithExtSlice(new(sf.APISongFreaksAlbum), &idsSlice, siteConf, db)
Update(state)
} else {
dna.Log("No new albums found")
}
// Recover failed sql statements
RecoverErrorQueries(SqlErrorLogPath, db)
CountDown(3*time.Second, QuittingMessage, EndingMessage)
db.Close()
}
// UpdateNhaccuatui gets lastest items from nhaccuatui.com.
//
// The update process goes through 6 steps:
// Step 1: Initalizing db connection, loading site config and state handler.
// Step 2: Updating new songs.
// Step 3: Updating new albums.
// Step 4: Updating new videos.
// Step 5: Updating new artists.
// Step 6: Recovering failed sql statements.
func UpdateNhaccuatui() {
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("nct", SiteConfigPath)
dna.PanicError(err)
TIMEOUT_SECS = 60 // for safe
// siteConf.NConcurrent = 15
state := NewStateHandler(new(nct.Song), siteConf, db)
Update(state)
state = NewStateHandler(new(nct.Album), siteConf, db)
Update(state)
state = NewStateHandler(new(nct.Video), siteConf, db)
Update(state)
state = NewStateHandler(new(nct.Artist), siteConf, db)
// Because there is no setting for total concurrent artists failed,
// the default value NCSongFail will be returned.
// Now, it is set to new value
state.SiteConfig.NCSongFail = 1000 // Set concurrency for artists
Update(state)
RecoverErrorQueries(SqlErrorLogPath, db)
CountDown(3*time.Second, QuittingMessage, EndingMessage)
db.Close()
}
func ExampleSelectMissingIds() {
db, err := sqlpg.Connect(sqlpg.NewSQLConfig("./app.ini"))
dna.PanicError(err)
ids, err := SelectMissingIds("ziartists", &dna.IntArray{5, 6, 7, 8, 9}, db)
dna.PanicError(err)
dna.Log(ids)
db.Close()
// Output:
// &dna.IntArray{8}
}
func amUpdateNewAlbums() {
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("am", SiteConfigPath)
siteConf.NConcurrent = 10
dna.PanicError(err)
state := NewStateHandler(new(am.APIAlbum), siteConf, db)
state.TableName = "amalbums"
Update(state)
CountDown(5*time.Second, QuittingMessage, EndingMessage)
db.Close()
}
// UpdateZing gets lastest items from mp3.zing.vn.
// The update process goes through 8 steps:
// Step 1: Initalizing db connection, loading site config and state handler
// Step 1: Updating new songs
// Step 3: Updating new albums
// Step 4: Updating new videos
// Step 5: Updating new artists
// Step 6: Updating new songids found in new albums
// Step 7: Updating new tvs
// Step 8: Recovering failed sql statements
func UpdateZing() {
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("zi", SiteConfigPath)
dna.PanicError(err)
// update song
state := NewStateHandler(new(zi.Song), siteConf, db)
Update(state)
// update album
state = NewStateHandler(new(zi.Album), siteConf, db)
Update(state)
// update video
state = NewStateHandler(new(zi.Video), siteConf, db)
Update(state)
// update artist
state = NewStateHandler(new(zi.Artist), siteConf, db)
Update(state)
// update new songids found in albums
dna.Log("Update new songs from albums")
ids := utils.SelectNewSidsFromAlbums("zialbums", time.Now(), db)
nids, err := utils.SelectMissingIds("zisongs", ids, db)
if err != nil {
dna.Log(err.Error())
} else {
if nids != nil && nids.Length() > 0 {
state = NewStateHandlerWithExtSlice(new(zi.Song), nids, siteConf, db)
Update(state)
} else {
dna.Log("No new songs found")
}
}
state = NewStateHandler(new(zi.TV), siteConf, db)
Update(state)
RecoverErrorQueries(SqlErrorLogPath, db)
time.Sleep(3 * time.Second)
db.Close()
}
// UpdateChacha updates chacha.vn.
// The update process goes through 5 steps:
// Step 1: Initalizing db connection, loading site config and state handler.
// Step 1: Updating new songs.
// Step 3: Updating new albums.
// Step 4: Updating new videos.
// Step 5: Recovering failed sql statements.
func UpdateChacha() {
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("cc", SiteConfigPath)
dna.PanicError(err)
state := NewStateHandler(new(cc.Song), siteConf, db)
Update(state)
// update album
state = NewStateHandler(new(cc.Album), siteConf, db)
Update(state)
state = NewStateHandler(new(cc.Video), siteConf, db)
Update(state)
RecoverErrorQueries(SqlErrorLogPath, db)
time.Sleep(3 * time.Second)
db.Close()
}
// UpdateKeeng gets lastest items from keeng.com.
// The update process goes through 6 steps:
// Step 1: Initalizing db connection, loading site config and state handler.
// Step 2: Updating new artists.
// Step 3: Updating new songs.
// Step 4: Updating new albums.
// Step 5: Updating new videos.
// Step 6: Recovering failed sql statements.
func UpdateKeeng() {
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("ke", SiteConfigPath)
dna.PanicError(err)
state := NewStateHandler(new(ke.Artist), siteConf, db)
Update(state)
state = NewStateHandler(new(ke.Song), siteConf, db)
Update(state)
state = NewStateHandler(new(ke.Album), siteConf, db)
Update(state)
state = NewStateHandler(new(ke.Video), siteConf, db)
Update(state)
RecoverErrorQueries(SqlErrorLogPath, db)
CountDown(3*time.Second, QuittingMessage, EndingMessage)
db.Close()
}
func UpdateLyricFind(vpnAppName, vpnAppUrl dna.String, estConDuration time.Duration) {
cmd := <-startVPN(vpnAppName, vpnAppUrl)
CountDown(estConDuration*time.Second, "Establishing VPN connection. Estimated time remaining:", "")
err := testingSongs()
if err != nil {
dna.Log("Error occurs: ", err.Error())
dna.Log("Operation aborted!")
} else {
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("lf", SiteConfigPath)
dna.PanicError(err)
state := NewStateHandler(new(lf.Song), siteConf, db)
Update(state)
RecoverErrorQueries(SqlErrorLogPath, db)
CountDown(3*time.Second, QuittingMessage, EndingMessage)
db.Close()
}
stopVPN(vpnAppName, cmd)
}
// UpdateChiasenhac updates new songs, videos and albums.
// New albums are created from new songs fetched.
// After getting new songs or videos, it runs re-fetching
// procedure until there is no missing songs, videos or
// the procesure runs 10 times.
//
// fillMode : only update records whose titles are empty.
//
// The update process goes through 6 steps:
// Step 1: Initializing db connection, site config, state handler and finding max id
// Step 2: Getting new songs from max id found in the table. If videos are found, then it inserts the videos into a video table.
// Step 3: Getting missing ids from last 10000 songids.
// Step 4: Refetching errors of songs having empty titles.
// Step 5: Recovering fail sql statement.
// Step 6: Creating new albums from new songs found.
func UpdateChiasenhac(fillMode dna.Bool) {
// note: songid 1172662 1172663 1172664 are not continuos
var errCount = 0
var missingCount = 0
var lastTotalMissing dna.Int = 0
var done = false
TIMEOUT_SECS = 240
dna.Log("TIMEOUT:", TIMEOUT_SECS)
// Step 1
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("csn", SiteConfigPath)
dna.PanicError(err)
// Getting LastSongId for SaveNewAlbums func
csn.LastSongId, err = utils.GetMaxId("csnsongs", db)
dna.PanicError(err)
dna.Log("Max ID:", csn.LastSongId)
// Step 2: Getting both new songs and videos and
// inserting into appropriate tables respectively.
if fillMode == false {
state := NewStateHandler(new(csn.Song), siteConf, db)
Update(state)
// Step 3: Fetching missing ids.
// It stops when total loop is 3 or the last total ids equals new total ones
dna.Log(dna.String("\nGetting missing ids from last 10000 songids").ToUpperCase())
for missingCount < 3 && done == false {
temp := updateMissingIds(db, siteConf, 10000)
if temp == lastTotalMissing {
done = true
}
lastTotalMissing = temp
missingCount += 1
}
}
// SET LAST SONGID
// csn.LastSongId = 1203603
// Step 4: Re-fetching err songs
db.Ping()
dna.Log(dna.String("\nRe-fetching err from last 10000 songs having EMPTY titles & ID > " + csn.LastSongId.ToString()).ToUpperCase())
for false == updateEmptyTitles(db, siteConf, csn.LastSongId) && errCount < 10 {
db.Ping()
errCount += 1
dna.Log("RE-FETCHING ROUND:", errCount)
}
dna.Log("Re-fetching error done!")
// Step 5: Recovering failed sql statments
RecoverErrorQueries(SqlErrorLogPath, db)
if fillMode == false {
// Step 6: Saving new abums
dna.Log("Finding and saving new albums from last songid:", csn.LastSongId)
nAlbums, err := csn.SaveNewAlbums(db)
if err != nil {
dna.Log(err.Error())
} else {
dna.Log("New albums inserted:", nAlbums)
}
}
dna.Log("SET TIMEOUT TO DEFAULT: 8s")
TIMEOUT_SECS = 8
time.Sleep(2 * time.Second)
db.Close()
}
// UpdateHDViet gets lastest items from hdviet.com.
//
// The update process goes through 8 steps and 2 parts:
//
// PART I: FINDING AND UPDATING NEW EPS OF MOVIES IN DB
// Step 1: Initalizing db connection, loading site config and state handler.
// Step 2: Finding all movies possibly having new episodes in DB
// Step 3: Finding all movies found in Step 2 actually having new episodes available from source website.
// Step 4: Checking consitency of movies' new found episodes from the source.
// Step 5: Getting new episode data from the source, save them to DB and update current_eps field of movies having new eps.
//
// PART II: GETTING NEW MOVIES FROM HDVIET SITE.
// Step 6: Find newest movies from the source.
// Step 7: Updating newest episodes from newest found movies in Step 6.
// Step 8: Recovering failed sql statements.
func UpdateHDViet() {
var mvTable = dna.String("hdvmovies")
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("hdv", SiteConfigPath)
dna.PanicError(err)
TIMEOUT_SECS = 100
// PART 1: UPDATING NEW EPISODES OF MOVIES IN DB
// STEP 2: Finding all movies possibly having new episodes in DB.
movieCurrentEps, err := hdv.GetMoviesCurrentEps(db, mvTable)
dna.PanicError(err)
newEpisodeKeys := dna.IntArray{}
for movieid, currentEps := range movieCurrentEps {
newEpisodeKeys.Push(hdv.ToEpisodeKey(movieid, currentEps))
}
newEpisodeKeys.Sort()
// STEP 3: Checking and getting new episodes if available from source website.
state := NewStateHandlerWithExtSlice(new(hdv.EpUpdater), &newEpisodeKeys, siteConf, db)
Update(state)
// STEP 4:Checking consitency of new found episodes of movies from the source.
movieIdList := dna.IntArray{}
for _, epKey := range hdv.LastestEpisodeKeyList {
mvid, _ := hdv.ToMovieIdAndEpisodeId(epKey)
movieIdList.Push(mvid)
}
movieIdList = movieIdList.Unique()
dna.Log("\nNumber of movies: ", movieIdList.Length(), "having new episodes:", hdv.LastestEpisodeKeyList.Length())
if movieIdList.Length() != dna.Int(len(hdv.LastestMovieCurrentEps)) {
dna.Log("LastestEpisodeKeyList & LastestMovieCurrentEps do not match! GOT:", movieIdList.Length(), len(hdv.LastestMovieCurrentEps))
}
// STEP 5: Getting new episode data from the source, save them to DB and update current_eps field of movies having new eps.
if hdv.LastestEpisodeKeyList.Length() > 0 {
state = NewStateHandlerWithExtSlice(new(hdv.Episode), &hdv.LastestEpisodeKeyList, siteConf, db)
Update(state)
RecoverErrorQueries(SqlErrorLogPath, db)
// dna.Log(hdv.LastestEpisodeKeyList)
hdv.SaveLastestMovieCurrentEps(db, mvTable, SQLERROR)
RecoverErrorQueries(SqlErrorLogPath, db)
} else {
dna.Log("No new episodes found. Update operation has been aborted!")
}
// PART 2: UPDATING NEW MOVIES FROM hdv site.
// STEP 6: Find newest movies from sources.
db.Ping()
state = NewStateHandler(new(hdv.Movie), siteConf, db)
Update(state)
// STEP 7: Updating newest episodes from newest found movies in Step 6.
hdv.EpisodeKeyList = hdv.EpisodeKeyList.Unique()
if hdv.EpisodeKeyList.Length() > 0 {
state = NewStateHandlerWithExtSlice(new(hdv.Episode), &hdv.EpisodeKeyList, siteConf, db)
Update(state)
} else {
dna.Log("No new movies found")
}
// STEP 8: Recovering failed sql statements.
RecoverErrorQueries(SqlErrorLogPath, db)
CountDown(3*time.Second, QuittingMessage, EndingMessage)
db.Close()
}