Exemple #1
1
// CheckKey returns true if key is valid
func CheckKey(key dna.String) dna.Bool {
	for _, v := range key.Split("") {
		if a.IndexOf(v) == -1 {
			return false
		}
	}
	return true
}
Exemple #2
1
func getAnchorTagsData(data dna.String) dna.StringArray {
	return dna.StringArray(data.Split("|").Map(func(val dna.String, idx dna.Int) dna.String {
		return val.RemoveHtmlTags("").Trim()
	}).([]dna.String)).Filter(func(val dna.String, idx dna.Int) dna.Bool {
		if val != "" {
			return true
		} else {
			return false
		}
	})
}
Exemple #3
1
// ToSeconds returns total seconds from the time format "01:02:03"
func ToSeconds(str dna.String) dna.Int {
	if str == "" {
		return 0
	} else {
		intervals := dna.IntArray(str.Split(":").Map(func(val dna.String, idx dna.Int) dna.Int {
			return val.ToInt()
		}).([]dna.Int))
		switch intervals.Length() {
		case 3:
			return intervals[0]*3600 + intervals[1]*60 + intervals[2]
		case 2:
			return intervals[0]*60 + intervals[1]
		case 1:
			return intervals[0]
		default:
			return 0
		}
	}
}
Exemple #4
0
func isSongFormat(str dna.String) dna.Bool {
	if str.EndsWith("mp3") == true || str.EndsWith("m4a") == true {
		return true
	} else {
		return false
	}
}
Exemple #5
0
func getOptionsForQuestion(tableOptions dna.String, question *Question) {
	// dna.Log(tableOptions)

	options := tableOptions.Replace(`<div class="bix-div-answer"`, "").Trim().Split("tdOptionDt")
	if options.Length() > 6 {
		panic("Answer options greater than 5 at: " + question.Cat.ToString())
	}
	for idx, option := range options {
		if idx > 0 {
			option := option.ReplaceWithRegexp(`^.+?>`, ``).ReplaceWithRegexp(`(?mis)<td class="bix-td-option".+$`, "").ReplaceWithRegexp(`</td>$`, "").ReplaceWithRegexp(`</td></tr><tr>$`, "").ReplaceWithRegexp(`</td></tr></table><input type="hidden".+$`, "").Trim()
			switch idx {
			case 1:
				question.OptionA = option
			case 2:
				question.OptionB = option
			case 3:
				question.OptionC = option
			case 4:
				question.OptionD = option
			case 5:
				question.OptionE = option
			}
		}
	}

}
Exemple #6
0
// To Source converts string format such as "(12,123123)" to Source Type
func ToSource(sourceStr dna.String) Source {
	if sourceStr.Match(`\([0-9]+,[0-9]+\)`) == false {
		panic("Wrong Source format")
	}
	sourArr := sourceStr.ReplaceWithRegexp(`^\(|\)$`, "").Split(",")
	return Source{sourArr[0].ToInt(), sourArr[1].ToInt()}
}
Exemple #7
0
func irrcrpt(_arg1 dna.String, _arg2 dna.Int) dna.String {
	var _local5 dna.Int
	var _local3 dna.String = ""
	var _local4 dna.Int
	for _local4 < _arg1.Length() {
		_local5 = _arg1.CharCodeAt(_local4)
		if (_local5 >= 48) && (_local5 <= 57) {
			_local5 = ((_local5 - _arg2) - 48)
			if _local5 < 0 {
				_local5 = (_local5 + ((57 - 48) + 1))
			}
			_local5 = ((_local5 % ((57 - 48) + 1)) + 48)
		} else {
			if (_local5 >= 65) && (_local5 <= 90) {
				_local5 = ((_local5 - _arg2) - 65)
				if _local5 < 0 {
					_local5 = (_local5 + ((90 - 65) + 1))
				}
				_local5 = ((_local5 % ((90 - 65) + 1)) + 65)
			} else {
				if (_local5 >= 97) && (_local5 <= 122) {
					_local5 = ((_local5 - _arg2) - 97)
					if _local5 < 0 {
						_local5 = (_local5 + ((122 - 97) + 1))
					}
					_local5 = ((_local5 % ((122 - 97) + 1)) + 97)
				}
			}
		}
		_local3 = (_local3 + dna.FromCharCode(_local5))
		_local4++
	}
	return (_local3)
}
Exemple #8
0
func amGetErrIds(inputFile dna.String, mode dna.Int) *dna.IntArray {
	var ret = dna.IntArray{}
	b, err := ioutil.ReadFile(inputFile.String())
	if err != nil {
		panic(err)
	}
	data := dna.String(string(b))
	lines := data.Split("\n")
	for _, line := range lines {
		switch mode {
		case 1:
			idArr := line.FindAllStringSubmatch(`([0-9]+) Post.+no such host`, 1)
			if len(idArr) > 0 {
				ret.Push(idArr[0][1].ToInt())
			}
			idArr = line.FindAllStringSubmatch(`Timeout.+at id :([0-9]+)`, 1)
			if len(idArr) > 0 {
				ret.Push(idArr[0][1].ToInt())
			}
		case 2:
			ret.Push(line.ToInt())
		}
	}
	if mode == 1 {
		err = ioutil.WriteFile(inputFile.String(), []byte{}, 0644)
		if err != nil {
			dna.Log("Cannot write to file1:", err.Error())
		}

	}
	ret = ret.Unique()
	return &ret
}
Exemple #9
0
func getGracenoteSongLyric(artist, title dna.String, song *Song) {
	link := "http://lyrics.wikia.com/Gracenote:" + artist.Replace(" ", "_") + ":" + title.Replace(" ", "_")
	result, err := http.Get(link)
	if err == nil {
		data := &result.Data

		writersArr := data.FindAllString(`Songwriters.+`, 1)
		if writersArr.Length() > 0 {
			song.GracenoteSongwriters = writersArr[0].GetTags("em")[0].RemoveHtmlTags("").DecodeHTML()
		}

		publisheraArr := data.FindAllString(`Publishers.+`, 1)
		if publisheraArr.Length() > 0 {
			song.GracenotePublishers = publisheraArr[0].GetTags("em")[0].RemoveHtmlTags("").DecodeHTML()
		}

		lyricArr := data.FindAllStringSubmatch(`(?mis)<div class='lyricbox'>(.+?)<\!--`, 1)
		if len(lyricArr) > 0 {
			song.GracenoteLyric = lyricArr[0][1].Trim().DecodeHTML().ReplaceWithRegexp(`(?mis)^<div.+?</span></div>`, "").Trim().Replace("<br />", "\n")
		}

		if song.GracenoteLyric != "" {
			song.DownloadGracenoteDone = 1
		} else {
			song.DownloadGracenoteDone = 0
		}

	}
}
Exemple #10
0
func IsValidTable(tblName dna.String, db *sqlpg.DB) dna.Bool {
	_, err := db.Exec("select * from " + tblName.String() + " limit 0")
	if err == nil {
		return true
	} else {
		return false
	}
}
Exemple #11
0
func Post(url, bodyStr dna.String) (*http.Result, error) {
	mutex.Lock()
	http.DefaulHeader = Header
	http.DefaulHeader.Set("Cookie", Cookie)
	http.DefaulHeader.Set("Content-Length", string(bodyStr.Length().ToString()))
	mutex.Unlock()
	// dna.Log(http.DefaulHeader)
	return http.Post(url, bodyStr)
}
Exemple #12
0
func getNames(data, from, to dna.String) dna.StringArray {
	pattern := "(?mis)" + from + "(.+?)" + to
	arr := data.FindAllStringSubmatch(pattern, 1)
	if len(arr) > 0 {
		return getAnchorTagsData(arr[0][1])
	} else {
		return dna.StringArray{}
	}
}
Exemple #13
0
// getSongFromMainPage returns song from main page
func getSongFromMainPage(song *Song) <-chan bool {

	channel := make(chan bool, 1)
	go func() {
		link := "http://nhacso.net/nghe-nhac/google-bot." + GetKey(song.Id) + "==.html"
		result, err := http.Get(link)
		// Log(link)
		// Log(result.Data)
		if err == nil && !result.Data.Match("Rất tiếc, chúng tôi không tìm thấy thông tin bạn yêu cầu!") {
			data := &result.Data
			if data.Match("official") {
				song.Official = 1
			}

			bitrate := data.FindAllString(`\d+kb\/s`, 1)[0]
			if !bitrate.IsBlank() {
				song.Bitrate = bitrate.FindAllString(`\d+`, 1)[0].ToInt()
			}

			plays := data.FindAllString("total_listen_song_detail_\\d+.+", 1)[0]
			if !plays.IsBlank() {
				song.Plays = plays.ReplaceWithRegexp("<\\/span>.+$", "").ReplaceWithRegexp("^.+>", "").ReplaceWithRegexp("\\.", "").ToInt()
			}

			topics := data.FindAllString("<li><a\\shref\\=\\\"http\\:\\/\\/nhacso\\.net\\/the-loai.+", 1)[0]
			if !topics.IsBlank() {
				topics = topics.ReplaceWithRegexp("^.+\\\">|<\\/a><\\/li>", "")
				song.Topics = topics.ToStringArray().SplitWithRegexp(" - ").SplitWithRegexp("/")
				temp := data.FindAllString("<a.+class=\"casi\".+>(.+?)<\\/a>", 1)
				var singer dna.String
				if temp.Length() > 0 {
					singer = temp[0]
				} else {
					singer = ""
				}
				if topics.Match("Nhạc Hoa") && singer.Match(` / `) {
					song.SameArtist = 1
				}
			}

			lyric := data.FindAllString(`(?mis)txtlyric.+Bạn chưa nhập nội bài hát`, 1)[0]
			if !lyric.IsBlank() {
				song.Islyric = 1
				song.Lyric = lyric.ReplaceWithRegexp("(?mis)<\\/textarea>.+$", "").ReplaceWithRegexp("^.+>", "")
				if song.Lyric.Match("Hãy đóng góp lời bài hát chính xác cho Nhacso nhé") {
					song.Lyric = ``
					song.Islyric = 0
				}
			}
		}
		channel <- true

	}()
	return channel
}
Exemple #14
0
func getVideoUrl(str, resolution dna.String) *VideoUrl {
	su := NewVideoUrl()
	su.Resolution = resolution
	su.Type = getType(str)
	su.Link = str.GetTagAttributes("href").ReplaceWithRegexp(`(http.+/).+(\..+$)`, "${1}file-name${2}")
	size := str.FindAllString(`[0-9\.]+ MB`, -1)
	if size.Length() > 0 {
		su.Size = size[0].ParseBytesFormat() / 1000
	}
	return su

}
Exemple #15
0
// DecodePath decodes encoded string such as "MjAxMyUyRjExJTJGMDUlMkYwJTJGMiUyRjAyN2UzN2M4NDUwMWFlOTEwNGNkZjgyMDZjYWE4OTkzLm1wMyU3QzI="
// into its real path on server such as "/2013/11/05/0/2/027e37c84501ae9104cdf8206caa8993.mp3|2"
func DecodePath(encodedPath dna.String) dna.String {
	ret, err := base64.StdEncoding.DecodeString(encodedPath.String())
	if err == nil {
		escape, err := url.QueryUnescape(string(ret))
		if err == nil {
			return dna.String(escape)
		} else {
			return ""
		}
	} else {
		return ""
	}
}
Exemple #16
0
func startVPN(vpnAppName, vpnAppUrl dna.String) chan *exec.Cmd {
	c := make(chan *exec.Cmd)
	dna.Log("Starting", vpnAppName)
	go func() {
		cmd := exec.Command(vpnAppUrl.String())
		c <- cmd
		err := cmd.Run()
		if err != nil {
			dna.Log(err.Error())
		}
	}()

	return c
}
Exemple #17
0
// Get impliments getting site with basic properties.
// Enable gzip, deflat by default to reduce  network data, redirect to new location from response.
// It returns data (String type) and error
// if err is nil then data is "" (empty).
func Get(url dna.String) (*Result, error) {
	req, err := http.NewRequest("GET", url.ToPrimitiveValue(), nil)
	req.Header = DefaulHeader
	// req.Header.Add("Accept-Encoding", "gzip,deflate")
	// req.Header.Add("Accept-Language", "en-US,en")
	// req.Header.Add("Cache-Control", "max-age=0")
	// req.Header.Add("Connection", "keep-alive")
	req.Header.Add("Host", url.ToPrimitiveValue())
	// req.Header.Add("User-Agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
	// req.Header.Add("Cookie", "")
	// dna.Log(req.Header)
	res, err := client.Do(req)
	if err != nil {
		return new(Result), err
	}

	var data []byte
	var myErr error

	switch res.Header.Get("Content-Encoding") {
	case "gzip":
		var reader io.ReadCloser
		reader, err := gzip.NewReader(res.Body)
		if err != nil {
			return new(Result), err
		}
		data, myErr = ioutil.ReadAll(reader)
		reader.Close()
	case "deflate":
		// Logv("sdsafsd")
		reader, err := zlib.NewReader(res.Body)
		if err != nil {
			return new(Result), err
		}
		data, myErr = ioutil.ReadAll(reader)
		reader.Close()
	default:
		data, myErr = ioutil.ReadAll(res.Body)
	}

	if myErr != nil {
		return new(Result), myErr
	}

	res.Body.Close()
	return NewResult(dna.Int(res.StatusCode), dna.String(res.Status), res.Header, dna.String(data)), nil
}
Exemple #18
0
func getGenresFromDesc(desc dna.String) dna.StringArray {
	var ret dna.StringArray
	genres := desc.FindAllString(`(?i)genres?(\s+)?:?.+`, 1)
	// "Released:" found in album id: 836258
	if genres.Length() > 0 {
		ret = dna.StringArray(genres[0].ReplaceWithRegexp(`(?mis)genres?(\s+)?:?`, "").ReplaceWithRegexp(`\.?\s*Released:.+`, "").Trim().Split(",").Map(func(val dna.String, idx dna.Int) dna.String {
			return val.ReplaceWithRegexp(":", "").Trim()
		}).([]dna.String))
		if ret.Length() == 1 {
			arr := dna.StringArray{}
			if ret[0].FindAllString(`(?mis)K-Pop`, 1).Length() > 0 {
				arr.Push("Korean Pop")
				arr.Push(ret[0].ReplaceWithRegexp(`(?mis)\(?K-Pop\)?`, "").Trim())
				ret = arr
			}
		}
	}
	return ret.SplitWithRegexp(` > `).SplitWithRegexp(`/`)
}
Exemple #19
0
func refineAuthorsOrArtists(str dna.String) dna.StringArray {
	tmp := str.ToStringArray().SplitWithRegexp(` / `).SplitWithRegexp(` - `).SplitWithRegexp(` – `)
	tmp = tmp.SplitWithRegexp(`, `).SplitWithRegexp(` ft `).SplitWithRegexp(` feat `).SplitWithRegexp(` ft. `)
	tmp = tmp.SplitWithRegexp(` feat. `).SplitWithRegexp(` Feat. `).SplitWithRegexp(` Ft. `)
	tmp = tmp.SplitWithRegexp(` & `).SplitWithRegexp(` vs. `).SplitWithRegexp(`- `).SplitWithRegexp(` & `)
	tmp = dna.StringArray(tmp.Map(func(val dna.String, idx dna.Int) dna.String {
		rv := val.Replace(`Đang Cập Nhật...`, ``).Replace(`Đang Cập Nhật (QT)`, ``)
		rv = rv.Replace(`Đang Cập Nhật (VN)`, ``).Replace(`Nhạc Phim QT`, `Nhạc Phim Quốc Tế`)
		rv = rv.Replace(`Đang cập nhật`, ``).Replace(`Nhiều Ca Sỹ`, `Various Artists`)
		return rv
	}).([]dna.String)).Filter(func(val dna.String, idx dna.Int) dna.Bool {
		if val != "" {
			return true
		} else {
			return false
		}
	})
	return tmp
}
Exemple #20
0
// ForeachLine loops through every line a file.
// An anynomous input function has line, index as params
func ForeachLine(filePath dna.String, lineFunc func(dna.String, dna.Int)) {

	var err error
	var line []byte
	f, err := os.Open(filePath.String())
	if err != nil {
		dna.Log("error opening file: %v\n", err)
		os.Exit(1)
	}
	defer f.Close()
	r := bufio.NewReaderSize(f, 4*1024)

	i := 0
	err = nil
	for err == nil {
		i += 1
		line, _, err = r.ReadLine()
		lineFunc(dna.String(string(line)), dna.Int(i))
	}
}
Exemple #21
0
//DecryptLRC returns LRC string from encrypted string.
//
//For example: Given a song id 2882720 with a key 9Fd4zVvPMIbf
//and an XML source file http://www.nhaccuatui.com/flash/xml?key1=8bfe9db992afaff5dc931dffca1b5c7b
//
//The encrypted lyric file url lies in <lyric> tags
// http://lrc.nct.nixcdn.com/2013/12/29/0/a/5/b/1388320374102.lrc
func DecryptLRC(data dna.String) (dna.String, error) {
	// CODE_SPECIAL "M z s 2 d k v t u 5 o d u" separated by space
	keyStr := irrcrpt(dna.Sprintf("%s", []byte{0x4d, 0x7a, 0x73, 0x32, 0x64, 0x6b, 0x76, 0x74, 0x75, 0x35, 0x6f, 0x64, 0x75}), 1)
	keyStrInHex := hex.EncodeToString(keyStr.ToBytes())

	keyStrInBytes, err := hex.DecodeString(keyStrInHex)
	if err != nil {
		return "", err
	}
	ret, err := hex.DecodeString(data.String())
	if err != nil {
		return "", err
	}
	cipher, err := rc4.NewCipher(keyStrInBytes)
	if err != nil {
		return "", err
	} else {
		cipher.XORKeyStream(ret, ret)
		return dna.String(string(ret)), nil
	}
}
Exemple #22
0
func isVideoFormat(str dna.String) dna.Bool {
	if str.EndsWith("mp4") == true || str.EndsWith("mpg") == true || str.EndsWith("flv") == true {
		return true
	} else {
		return false
	}
}
Exemple #23
0
// RecoverSQLLogError re-executes failed sql queries in sql error log file from specified path.
// It returns the number of failed -reexec queries, and new failed
// queries will be written to the file
//
// The format of error file is:
// 	Error description - $$$error$$$SQL_QUERY$$$error$$$
// Therefore only get statements enclosed by special `$$$error$$$`
func RecoverSQLLogError(sqlErrFilePath dna.String, db *sqlpg.DB) dna.Int {
	var errCount = 0
	var errStrings = dna.StringArray{}
	b, err := ioutil.ReadFile(sqlErrFilePath.String())
	if err != nil {
		panic(err)
	}
	data := dna.String(string(b))
	// dna.Log("\n", data.Length())
	sqlArr := data.FindAllString(`(?mis)\$\$\$error\$\$\$.+?\$\$\$error\$\$\$`, -1)
	// dna.Log("\nTOTAL SQL STATEMENTS FOUND:", sqlArr.Length())
	for _, val := range sqlArr {
		sqlStmtArr := val.FindAllStringSubmatch(`(?mis)\$\$\$error\$\$\$(.+?)\$\$\$error\$\$\$`, -1)
		if len(sqlStmtArr) > 0 {
			_, err := db.Exec(sqlStmtArr[0][1].String())
			if err != nil {
				if dna.String(err.Error()).Contains(`duplicate key value violates unique constraint`) == false {
					errCount += 1
					errStrings.Push("$$$error$$$" + sqlStmtArr[0][1] + "$$$error$$$")
				}
			}
		}
	}
	if errCount == 0 {
		err = ioutil.WriteFile(sqlErrFilePath.String(), []byte{}, 0644)
	} else {
		err = ioutil.WriteFile(sqlErrFilePath.String(), []byte(errStrings.Join("\n").String()), 0644)
	}
	if err != nil {
		panic(err)
	}
	return dna.Int(errCount)
}
Exemple #24
0
// ToSiteid returns a siteid id from a short form.
// For example: "ns" is for NhacSo
func ToSiteid(shortForm dna.String) dna.Int {
	switch shortForm {
	case "ns":
		return NhacSo
	case "zi":
		return Zing
	case "nct":
		return NhacCuaTui
	case "cc":
		return ChaCha
	case "nv":
		return NhacVui
	case "csn":
		return ChiaSeNhac
	case "hdv":
		return HDViet
	case "ke":
		return Keeng
	case "sf":
		return SongFreaks
	case "am":
		return AllMusic
	case "lw":
		return LyricWiki
	case "ml":
		return MetroLyrics
	case "lf":
		return LyricFind
	case "vg":
		return VietGiaiTri
	case "mv":
		return MusicVNN
	default:
		panic("Cannot convert shortform to siteid - GOT:" + shortForm.String())
	}
}
Exemple #25
0
func (users *Users) Save(db *sqlpg.DB) error {
	var queries dna.String = ""
	// var err error
	// for _, user := range users.List {
	// 	if user.IsValid() == true {
	// 		err = db.InsertIgnore(user)
	// 	}
	// }
	// return err

	for _, user := range users.List {
		if user.IsValid() == true {
			queries += sqlpg.GetInsertStatement("ziusers", user, false) + "\n"
		}
	}
	mutex.Lock()
	n, err := SongFile.WriteString(queries.String())
	if err != nil {
		dna.Log("Cannot write to file while getting song", err.Error(), "\n\n")
	}
	TotalBytes += dna.Int(n)
	mutex.Unlock()
	return nil
}
Exemple #26
0
func getTSGM(data *dna.String, kind dna.String) dna.String {
	var itemArr dna.StringArray
	switch kind {
	case "genres":
		itemArr = data.FindAllString(`(?mis)<h4>Genre</h4>(.+?)</div>`, 1)
	case "styles":
		itemArr = data.FindAllString(`(?mis)<h4>Styles</h4>(.+?)</div>`, 1)
	case "moods":
		itemArr = data.FindAllString(`(?mis)<h4>Album Moods</h4>(.+?)</div>`, 1)
	case "themes":
		itemArr = data.FindAllString(`(?mis)<h4>Themes</h4>(.+?)</div>`, 1)
	default:
		panic("Wrong kind!!!")
	}
	if itemArr.Length() > 0 {
		catArr := itemArr[0].FindAllString(`<a href=.+?</a>`, -1)
		categories := catArr.Map(func(val dna.String, idx dna.Int) Category {
			var idArr []dna.StringArray
			var id dna.Int = 0
			name := val.RemoveHtmlTags("")
			if kind == "moods" {
				idArr = val.FindAllStringSubmatch(`xa([0-9]+)`, 1)
			} else {
				idArr = val.FindAllStringSubmatch(`ma([0-9]+)`, 1)
			}
			if len(idArr) > 0 {
				id = idArr[0][1].ToInt()
			}
			return Category{id, name}
		}).([]Category)

		if len(categories) > 0 {
			bCat, merr := json.Marshal(categories)
			if merr != nil {
				return "[]"
			} else {
				return dna.String(string(bCat))
			}
		} else {
			return "[]"
		}
	} else {
		return "[]"
	}

}
Exemple #27
0
func Post(url dna.String, bodyStr dna.String) (*Result, error) {
	client := &http.Client{}
	// dna.Log(bodyStr)
	req, err := http.NewRequest("POST", url.String(), strings.NewReader(bodyStr.String()))
	req.Header = DefaulHeader
	req.Header.Add("Host", url.ToPrimitiveValue())
	res, err := client.Do(req)
	if err != nil {
		return new(Result), err
	}

	var data []byte
	var myErr error

	switch res.Header.Get("Content-Encoding") {
	case "gzip":
		var reader io.ReadCloser
		reader, err := gzip.NewReader(res.Body)
		if err != nil {
			return new(Result), err
		}
		data, myErr = ioutil.ReadAll(reader)
		reader.Close()
	case "deflate":
		// Logv("sdsafsd")
		reader, err := zlib.NewReader(res.Body)
		if err != nil {
			return new(Result), err
		}
		data, myErr = ioutil.ReadAll(reader)
		reader.Close()
	default:
		data, myErr = ioutil.ReadAll(res.Body)
	}

	if myErr != nil {
		return new(Result), myErr
	}

	res.Body.Close()
	return NewResult(dna.Int(res.StatusCode), dna.String(res.Status), res.Header, dna.String(data)), nil
}
Exemple #28
0
// Update gets item from sites and save them to database
func Update(state *StateHandler) *Counter {

	CheckStateHandler(state)
	var (
		counter    *Counter = NewCounterWithStateHandler(state)
		idcFormat  dna.String
		cData      dna.String
		idc        *terminal.Indicator
		bar        *terminal.ProgressBar
		errChannel chan bool  = make(chan bool)
		tableName  dna.String = state.GetTableName()
		startupFmt dna.String = "Update %v - Cid:%v - Pat:%v - Ncf:%v - NCon:%v"
	)

	// 3rd pattern: callind GetCid() wil invoke error
	INFO.Println(dna.Sprintf(startupFmt, tableName, state.Cid, state.GetPattern(), state.GetNCFail(), state.SiteConfig.NConcurrent))

	if utils.IsValidTable(tableName, state.GetDb()) == false {
		tableName = ""
	}

	if state.GetPattern() == 1 {
		idcFormat = " $indicator %v|cid:%v|cf:%v" // cid: current id, cf: continuous failure count
		idc = terminal.NewIndicatorWithTheme(terminal.ThemeDefault)
		// Getting maxid from an item's table
		id, err := utils.GetMaxId(tableName, state.GetDb())
		dna.PanicError(err)
		state.SetCid(id)
	} else {
		bar = getUpdateProgressBar(counter.Total, tableName)
	}

	// Config.NConcurrent
	for i := dna.Int(0); i < state.SiteConfig.NConcurrent; i++ {
		go atomicUpdate(errChannel, state)
	}

	for state.IsComplete() == false {
		hasError := <-errChannel
		counter.Tick(dna.Bool(hasError))
		switch state.GetPattern() {
		case 1:
			if counter.GetCFail() == state.GetNCFail() {
				state.SetCompletion()
			}
			idc.Show(dna.Sprintf(idcFormat, counter, state.GetCid(), counter.GetCFail()))
		case 2:
			if counter.GetCount() == state.GetRange().Total {
				state.SetCompletion()
			}
			cData = dna.Sprintf("%v | Ncc:%v | Cid:%v", counter, state.GetNcCount(), state.GetCid())
			bar.Show(counter.GetCount(), cData, cData.Replace("|", "-"))
		case 3:
			if counter.GetCount() == state.GetExtSlice().Length() {
				state.SetCompletion()
			}
			cData = dna.Sprintf("%v | Ncc:%v | Cid:%v", counter, state.GetNcCount(), state.GetCid())
			bar.Show(counter.GetCount(), cData, cData.Replace("|", "-"))
		}

	}
	if state.GetPattern() == 1 {
		idc.Close(dna.Sprintf("$indicator Complete updating %v!", tableName))
	}

	INFO.Printf("[%v] %v\n", tableName, counter.FinalString())
	// Delay 2s to ensure all the goroutines left finish it processed before sqlpg.DB closed
	// time.Sleep(2 * time.Second)
	return counter
}
Exemple #29
0
func getSection(section dna.String) APIAwardSection {
	var awards = []APIAward{}
	var name dna.String = ""
	var sectionType dna.String = ""
	var awardType dna.Int = 0

	nameArr := section.FindAllString(`(?mis)<h2 class="headline">.+?</h2>`, -1)
	if nameArr.Length() > 0 {
		name = nameArr[0].RemoveHtmlTags("").Trim()
		switch {
		case name.Match("Singles") == true:
			sectionType = "SINGLE"
			awardType = 2
		case name.Match("APIAlbums") == true:
			sectionType = "ALBUM"
			awardType = 1

		}
	}

	if name.Match("Grammy Awards") == true {
		return getGrammyAPIAwardSection(section)
	}

	tbody := section.FindAllString(`(?mis)<tbody>.+?</tbody>`, -1)
	if tbody.Length() > 0 {
		rows := tbody[0].FindAllString(`(?mis)<tr>.+?</tr>`, -1)
		awards = rows.Map(func(row dna.String, idx dna.Int) APIAward {
			var (
				id      dna.Int    = 0
				year    dna.Int    = 0
				chart   dna.String = ""
				title   dna.String = ""
				peak    dna.Int    = 0
				winners            = []Person{}
				award   dna.String = ""
			)

			yearArr := row.FindAllString(`(?mis)<td class="year".+?</div>`, 1)
			if yearArr.Length() > 0 {
				year = yearArr[0].RemoveHtmlTags("").Trim().ToInt()
			}

			peakArr := row.FindAllString(`(?mis)<td class="peak".+?</td>`, 1)
			if peakArr.Length() > 0 {
				peak = peakArr[0].RemoveHtmlTags("").Trim().ToInt()
			}

			charArr := row.FindAllString(`(?mis)<div class="chart-name">.+?</div>`, 1)
			if charArr.Length() > 0 {
				chart = charArr[0].RemoveHtmlTags("").DecodeHTML().Trim()
			}

			titleArr := row.FindAllString(`<a href=".+</a>`, 1)
			if titleArr.Length() > 0 {
				title = titleArr[0].RemoveHtmlTags("").DecodeHTML().Trim()
			}

			var match dna.String = ""
			switch sectionType {
			case "SINGLE":
				match = `<a href=.+mt([0-9]+)"`
			case "ALBUM":
				match = `<a href=.+mw([0-9]+)"`
			default:
				match = `<a href=.+mw([0-9]+)"`
			}
			idArr := row.FindAllStringSubmatch(match, 1)
			if len(idArr) > 0 {
				id = idArr[0][1].ToInt()
			}

			return APIAward{
				Id:      id,
				Title:   title,
				Year:    year,
				Chart:   chart,
				Peak:    peak,
				Type:    awardType,
				Winners: winners,
				Award:   award,
			}
		}).([]APIAward)
	}
	return APIAwardSection{Name: name, Type: sectionType, Awards: awards}
}
Exemple #30
0
func getGrammyAPIAwardSection(section dna.String) APIAwardSection {
	var awards = []APIAward{}
	tbody := section.FindAllString(`(?mis)<tbody>.+?</tbody>`, -1)
	if tbody.Length() > 0 {
		rows := tbody[0].FindAllString(`(?mis)<tr>.+?</tr>`, -1)
		awards = rows.Map(func(row dna.String, idx dna.Int) APIAward {
			var (
				id      dna.Int    = 0
				year    dna.Int    = 0
				chart   dna.String = ""
				title   dna.String = ""
				peak    dna.Int    = 0
				winners            = []Person{}
				award   dna.String = ""
				atype   dna.Int    = 0
			)

			yearArr := row.FindAllString(`(?mis)<div class="year">.+?</div>`, 1)
			if yearArr.Length() > 0 {
				year = yearArr[0].RemoveHtmlTags("").Trim().ToInt()
			}

			atypeArr := row.FindAllString(`(?mis)<div class="type">.+?</div>`, 1)
			if atypeArr.Length() > 0 {
				switch atypeArr[0].RemoveHtmlTags("").Trim() {
				case "T":
					atype = 2
				case "A":
					atype = 1
				default:
					atype = 0
				}
			}

			awardArr := row.FindAllString(`(?mis)<div class="award-name">.+?</div>`, 1)
			if awardArr.Length() > 0 {
				award = awardArr[0].RemoveHtmlTags("").Trim()
			}

			titleArr := row.FindAllString(`(?mis)<td class="title".+?</div>`, 1)
			if titleArr.Length() > 0 {
				title = titleArr[0].RemoveHtmlTags("").Trim()
			}

			winnerArr := row.FindAllString(`<a href=".+?</a>`, -1)
			winners = winnerArr.Map(func(winner dna.String, idx dna.Int) Person {
				var winnerName dna.String = ""
				var winnerId dna.Int = 0
				winnerName = winner.RemoveHtmlTags("").Trim()
				winnerIdArr := winner.FindAllStringSubmatch(`<a href=.+mn([0-9]+)`, 1)
				if len(winnerIdArr) > 0 {
					winnerId = winnerIdArr[0][1].ToInt()
				}
				return Person{Id: winnerId, Name: winnerName}
			}).([]Person)

			return APIAward{
				Id:      id,
				Title:   title,
				Year:    year,
				Chart:   chart,
				Peak:    peak,
				Type:    atype,
				Winners: winners,
				Award:   award,
			}
		}).([]APIAward)
		return APIAwardSection{Name: "Grammy Awards", Type: "ALBUMs & SONGS", Awards: awards}
	} else {
		return APIAwardSection{}
	}

}