// UpdateChiasenhac updates new songs, videos and albums.
// New albums are created from new songs fetched.
// After getting new songs or videos, it runs re-fetching
// procedure until there is no missing songs, videos or
// the procesure runs 10 times.
//
// fillMode : only update records whose titles are empty.
//
// The update process goes through 6 steps:
// Step 1: Initializing db connection, site config, state handler and finding max id
// Step 2: Getting new songs from max id found in the table. If videos are found, then it inserts the videos into a video table.
// Step 3: Getting missing ids from last 10000 songids.
// Step 4: Refetching errors of songs having empty titles.
// Step 5: Recovering fail sql statement.
// Step 6: Creating new albums from new songs found.
func UpdateChiasenhac(fillMode dna.Bool) {
// note: songid 1172662 1172663 1172664 are not continuos
var errCount = 0
var missingCount = 0
var lastTotalMissing dna.Int = 0
var done = false
TIMEOUT_SECS = 240
dna.Log("TIMEOUT:", TIMEOUT_SECS)
// Step 1
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("csn", SiteConfigPath)
dna.PanicError(err)
// Getting LastSongId for SaveNewAlbums func
csn.LastSongId, err = utils.GetMaxId("csnsongs", db)
dna.PanicError(err)
dna.Log("Max ID:", csn.LastSongId)
// Step 2: Getting both new songs and videos and
// inserting into appropriate tables respectively.
if fillMode == false {
state := NewStateHandler(new(csn.Song), siteConf, db)
Update(state)
// Step 3: Fetching missing ids.
// It stops when total loop is 3 or the last total ids equals new total ones
dna.Log(dna.String("\nGetting missing ids from last 10000 songids").ToUpperCase())
for missingCount < 3 && done == false {
temp := updateMissingIds(db, siteConf, 10000)
if temp == lastTotalMissing {
done = true
}
lastTotalMissing = temp
missingCount += 1
}
}
// SET LAST SONGID
// csn.LastSongId = 1203603
// Step 4: Re-fetching err songs
db.Ping()
dna.Log(dna.String("\nRe-fetching err from last 10000 songs having EMPTY titles & ID > " + csn.LastSongId.ToString()).ToUpperCase())
for false == updateEmptyTitles(db, siteConf, csn.LastSongId) && errCount < 10 {
db.Ping()
errCount += 1
dna.Log("RE-FETCHING ROUND:", errCount)
}
dna.Log("Re-fetching error done!")
// Step 5: Recovering failed sql statments
RecoverErrorQueries(SqlErrorLogPath, db)
if fillMode == false {
// Step 6: Saving new abums
dna.Log("Finding and saving new albums from last songid:", csn.LastSongId)
nAlbums, err := csn.SaveNewAlbums(db)
if err != nil {
dna.Log(err.Error())
} else {
dna.Log("New albums inserted:", nAlbums)
}
}
dna.Log("SET TIMEOUT TO DEFAULT: 8s")
TIMEOUT_SECS = 8
time.Sleep(2 * time.Second)
db.Close()
}
// Update gets item from sites and save them to database
func Update(state *StateHandler) *Counter {
CheckStateHandler(state)
var (
counter *Counter = NewCounterWithStateHandler(state)
idcFormat dna.String
cData dna.String
idc *terminal.Indicator
bar *terminal.ProgressBar
errChannel chan bool = make(chan bool)
tableName dna.String = state.GetTableName()
startupFmt dna.String = "Update %v - Cid:%v - Pat:%v - Ncf:%v - NCon:%v"
)
// 3rd pattern: callind GetCid() wil invoke error
INFO.Println(dna.Sprintf(startupFmt, tableName, state.Cid, state.GetPattern(), state.GetNCFail(), state.SiteConfig.NConcurrent))
if utils.IsValidTable(tableName, state.GetDb()) == false {
tableName = ""
}
if state.GetPattern() == 1 {
idcFormat = " $indicator %v|cid:%v|cf:%v" // cid: current id, cf: continuous failure count
idc = terminal.NewIndicatorWithTheme(terminal.ThemeDefault)
// Getting maxid from an item's table
id, err := utils.GetMaxId(tableName, state.GetDb())
dna.PanicError(err)
state.SetCid(id)
} else {
bar = getUpdateProgressBar(counter.Total, tableName)
}
// Config.NConcurrent
for i := dna.Int(0); i < state.SiteConfig.NConcurrent; i++ {
go atomicUpdate(errChannel, state)
}
for state.IsComplete() == false {
hasError := <-errChannel
counter.Tick(dna.Bool(hasError))
switch state.GetPattern() {
case 1:
if counter.GetCFail() == state.GetNCFail() {
state.SetCompletion()
}
idc.Show(dna.Sprintf(idcFormat, counter, state.GetCid(), counter.GetCFail()))
case 2:
if counter.GetCount() == state.GetRange().Total {
state.SetCompletion()
}
cData = dna.Sprintf("%v | Ncc:%v | Cid:%v", counter, state.GetNcCount(), state.GetCid())
bar.Show(counter.GetCount(), cData, cData.Replace("|", "-"))
case 3:
if counter.GetCount() == state.GetExtSlice().Length() {
state.SetCompletion()
}
cData = dna.Sprintf("%v | Ncc:%v | Cid:%v", counter, state.GetNcCount(), state.GetCid())
bar.Show(counter.GetCount(), cData, cData.Replace("|", "-"))
}
}
if state.GetPattern() == 1 {
idc.Close(dna.Sprintf("$indicator Complete updating %v!", tableName))
}
INFO.Printf("[%v] %v\n", tableName, counter.FinalString())
// Delay 2s to ensure all the goroutines left finish it processed before sqlpg.DB closed
// time.Sleep(2 * time.Second)
return counter
}