Esempio n. 1
0
// UpdateChiasenhac updates new songs, videos and albums.
// New albums are created from new songs fetched.
// After getting new songs or videos, it runs re-fetching
// procedure until there is no missing songs, videos or
// the procesure runs 10 times.
//
// 	fillMode : only update records whose titles are empty.
//
// The update process goes through 6 steps:
// 	Step 1: Initializing db connection, site config, state handler and finding max id
// 	Step 2: Getting new songs from max id found in the table. If videos are found, then it inserts the videos into a video table.
// 	Step 3: Getting missing ids from last 10000 songids.
// 	Step 4: Refetching errors of songs having empty titles.
// 	Step 5: Recovering fail sql statement.
// 	Step 6: Creating new albums from new songs found.
func UpdateChiasenhac(fillMode dna.Bool) {
	// note: songid 1172662 1172663 1172664 are not continuos
	var errCount = 0
	var missingCount = 0
	var lastTotalMissing dna.Int = 0
	var done = false

	TIMEOUT_SECS = 240
	dna.Log("TIMEOUT:", TIMEOUT_SECS)
	// Step 1
	db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
	dna.PanicError(err)
	siteConf, err := LoadSiteConfig("csn", SiteConfigPath)
	dna.PanicError(err)
	// Getting LastSongId for SaveNewAlbums func
	csn.LastSongId, err = utils.GetMaxId("csnsongs", db)
	dna.PanicError(err)
	dna.Log("Max ID:", csn.LastSongId)

	// Step 2: Getting both new songs and videos and
	// inserting into appropriate tables respectively.
	if fillMode == false {
		state := NewStateHandler(new(csn.Song), siteConf, db)
		Update(state)

		// Step 3: Fetching missing ids.
		// It stops when total loop is 3 or the last total ids equals new total ones
		dna.Log(dna.String("\nGetting missing ids from last 10000 songids").ToUpperCase())
		for missingCount < 3 && done == false {
			temp := updateMissingIds(db, siteConf, 10000)
			if temp == lastTotalMissing {
				done = true
			}
			lastTotalMissing = temp
			missingCount += 1
		}

	}

	// SET LAST SONGID
	// csn.LastSongId = 1203603

	// Step 4: Re-fetching err songs
	db.Ping()
	dna.Log(dna.String("\nRe-fetching err from last 10000 songs having EMPTY titles & ID > " + csn.LastSongId.ToString()).ToUpperCase())
	for false == updateEmptyTitles(db, siteConf, csn.LastSongId) && errCount < 10 {
		db.Ping()
		errCount += 1
		dna.Log("RE-FETCHING ROUND:", errCount)
	}
	dna.Log("Re-fetching error done!")

	// Step 5: Recovering failed sql statments
	RecoverErrorQueries(SqlErrorLogPath, db)

	if fillMode == false {
		// Step 6: Saving new abums
		dna.Log("Finding and saving new albums from last songid:", csn.LastSongId)
		nAlbums, err := csn.SaveNewAlbums(db)
		if err != nil {
			dna.Log(err.Error())
		} else {
			dna.Log("New albums inserted:", nAlbums)
		}
	}

	dna.Log("SET TIMEOUT TO DEFAULT: 8s")
	TIMEOUT_SECS = 8
	time.Sleep(2 * time.Second)
	db.Close()
}
Esempio n. 2
0
File: site.go Progetto: olragon/dna
// Update gets item from sites and save them to database
func Update(state *StateHandler) *Counter {

	CheckStateHandler(state)
	var (
		counter    *Counter = NewCounterWithStateHandler(state)
		idcFormat  dna.String
		cData      dna.String
		idc        *terminal.Indicator
		bar        *terminal.ProgressBar
		errChannel chan bool  = make(chan bool)
		tableName  dna.String = state.GetTableName()
		startupFmt dna.String = "Update %v - Cid:%v - Pat:%v - Ncf:%v - NCon:%v"
	)

	// 3rd pattern: callind GetCid() wil invoke error
	INFO.Println(dna.Sprintf(startupFmt, tableName, state.Cid, state.GetPattern(), state.GetNCFail(), state.SiteConfig.NConcurrent))

	if utils.IsValidTable(tableName, state.GetDb()) == false {
		tableName = ""
	}

	if state.GetPattern() == 1 {
		idcFormat = " $indicator %v|cid:%v|cf:%v" // cid: current id, cf: continuous failure count
		idc = terminal.NewIndicatorWithTheme(terminal.ThemeDefault)
		// Getting maxid from an item's table
		id, err := utils.GetMaxId(tableName, state.GetDb())
		dna.PanicError(err)
		state.SetCid(id)
	} else {
		bar = getUpdateProgressBar(counter.Total, tableName)
	}

	// Config.NConcurrent
	for i := dna.Int(0); i < state.SiteConfig.NConcurrent; i++ {
		go atomicUpdate(errChannel, state)
	}

	for state.IsComplete() == false {
		hasError := <-errChannel
		counter.Tick(dna.Bool(hasError))
		switch state.GetPattern() {
		case 1:
			if counter.GetCFail() == state.GetNCFail() {
				state.SetCompletion()
			}
			idc.Show(dna.Sprintf(idcFormat, counter, state.GetCid(), counter.GetCFail()))
		case 2:
			if counter.GetCount() == state.GetRange().Total {
				state.SetCompletion()
			}
			cData = dna.Sprintf("%v | Ncc:%v | Cid:%v", counter, state.GetNcCount(), state.GetCid())
			bar.Show(counter.GetCount(), cData, cData.Replace("|", "-"))
		case 3:
			if counter.GetCount() == state.GetExtSlice().Length() {
				state.SetCompletion()
			}
			cData = dna.Sprintf("%v | Ncc:%v | Cid:%v", counter, state.GetNcCount(), state.GetCid())
			bar.Show(counter.GetCount(), cData, cData.Replace("|", "-"))
		}

	}
	if state.GetPattern() == 1 {
		idc.Close(dna.Sprintf("$indicator Complete updating %v!", tableName))
	}

	INFO.Printf("[%v] %v\n", tableName, counter.FinalString())
	// Delay 2s to ensure all the goroutines left finish it processed before sqlpg.DB closed
	// time.Sleep(2 * time.Second)
	return counter
}