示例#1
0
文件: ref.go 项目: olragon/dna
func DumpFiles() {
	var ret = dna.StringArray{}
	for _, table := range GetAlbumTables().Concat(GetSongTables()).Concat(GetVideoTables()) {
		ret.Push(`"` + table + `"`)
	}
	dna.Log(ret.Join(","))
}
示例#2
0
文件: sqlrecover.go 项目: olragon/dna
// RecoverSQLLogError re-executes failed sql queries in sql error log file from specified path.
// It returns the number of failed -reexec queries, and new failed
// queries will be written to the file
//
// The format of error file is:
// 	Error description - $$$error$$$SQL_QUERY$$$error$$$
// Therefore only get statements enclosed by special `$$$error$$$`
func RecoverSQLLogError(sqlErrFilePath dna.String, db *sqlpg.DB) dna.Int {
	var errCount = 0
	var errStrings = dna.StringArray{}
	b, err := ioutil.ReadFile(sqlErrFilePath.String())
	if err != nil {
		panic(err)
	}
	data := dna.String(string(b))
	// dna.Log("\n", data.Length())
	sqlArr := data.FindAllString(`(?mis)\$\$\$error\$\$\$.+?\$\$\$error\$\$\$`, -1)
	// dna.Log("\nTOTAL SQL STATEMENTS FOUND:", sqlArr.Length())
	for _, val := range sqlArr {
		sqlStmtArr := val.FindAllStringSubmatch(`(?mis)\$\$\$error\$\$\$(.+?)\$\$\$error\$\$\$`, -1)
		if len(sqlStmtArr) > 0 {
			_, err := db.Exec(sqlStmtArr[0][1].String())
			if err != nil {
				if dna.String(err.Error()).Contains(`duplicate key value violates unique constraint`) == false {
					errCount += 1
					errStrings.Push("$$$error$$$" + sqlStmtArr[0][1] + "$$$error$$$")
				}
			}
		}
	}
	if errCount == 0 {
		err = ioutil.WriteFile(sqlErrFilePath.String(), []byte{}, 0644)
	} else {
		err = ioutil.WriteFile(sqlErrFilePath.String(), []byte(errStrings.Join("\n").String()), 0644)
	}
	if err != nil {
		panic(err)
	}
	return dna.Int(errCount)
}
示例#3
0
// GetFormats returns proper formatStr for a song or a video.
// If it is a song, IsSong will be set to true. Otherwise, it will set to false false.
func GetFormats(urls dna.StringArray) (formatStr dna.String, IsSong dna.Bool) {
	switch getType(urls.Join("")) {
	case "mp3", "m4a", "flac":
		formatStr, IsSong = getStringifiedSongUrls(urls), IS_SONG
		return formatStr, IsSong
	case "mp4", "flv":
		formatStr, IsSong = getStringifiedVideoUrls(urls), IS_VIDEO
		return formatStr, IsSong
	default:
		panic("Wrong type. Cannot indentify song or video")
	}
	return "", false
}
示例#4
0
文件: funcs.go 项目: olragon/dna
// GetUpdateStatement returns an update statement from specified snake-case columns.
// If columns's names are not found, it will return an error.
// It updates some fields from a struct.
//
// 	* tbName : A name of update table.
// 	* structValue : A struct-typed value being scanned. Its fields have to be dna basic type or time.Time.
// 	* conditionColumn : A snake-case column name in the condition, usually it's an id
// 	* columns : A list of args of column names in the table being updated.
// 	* Returns an update statement.
func GetUpdateStatement(tbName dna.String, structValue interface{}, conditionColumn dna.String, columns ...dna.String) (dna.String, error) {
	if reflect.TypeOf(structValue).Kind() != reflect.Ptr {
		panic("StructValue has to be pointer")
		if reflect.TypeOf(structValue).Elem().Kind() != reflect.Struct {
			panic("StructValue has to be struct type")
		}
	}
	query := "UPDATE " + tbName + " SET\n"
	result := dna.StringArray{}
	for _, column := range columns {
		result.Push(getPairValue(structValue, column))
	}
	conditionRet := "\nWHERE " + getPairValue(structValue, conditionColumn) + ";"
	return query + result.Join(",\n") + conditionRet, nil
}
示例#5
0
func getStringifiedSongUrls(urls dna.StringArray) dna.String {
	var baseLink = dna.String("")
	songUrls := []SongUrl{}
	urls.ForEach(func(val dna.String, idx dna.Int) {
		// dna.Log(val)
		// Finding bitrate
		switch {
		case val.Match(`128kbps`) == true:
			songUrl := getSongUrl(val, "128kbps")
			baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
			songUrls = append(songUrls, *songUrl)
		case val.Match(`320kbps`) == true:
			songUrl := getSongUrl(val, "320kbps")
			baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
			songUrls = append(songUrls, *songUrl)
		case val.Match(`32kbps`) == true:
			songUrl := getSongUrl(val, "32kbps")
			baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
			songUrls = append(songUrls, *songUrl)
		case val.Match(`500kbps`) == true:
			songUrl := getSongUrl(val, "500kbps")
			songUrl.Link = baseLink + "m4a/file-name.m4a"
			songUrls = append(songUrls, *songUrl)
		case val.Match(`Lossless`) == true:
			songUrl := getSongUrl(val, "Lossless")
			songUrl.Link = baseLink + "flac/file-name.flac"
			songUrls = append(songUrls, *songUrl)
		}
	})
	// http://data.chiasenhac.com/downloads/1184/2/1183017-cfc5f7df/flac/file-name.flac
	// replace the link 500kps and lossless with available link,  apply for registered user only
	// and reduce the link length
	var ret = dna.StringArray{}
	for _, songUrl := range songUrls {
		var br dna.String
		if songUrl.Bitrate == "Lossless" {
			br = "1411"
		} else {
			br = songUrl.Bitrate.Replace("kbps", "")
		}
		t := `"(` + songUrl.Link + "," + songUrl.Type + "," + songUrl.Size.ToString() + "," + br + `)"`
		ret.Push(t)
	}
	// dna.Log(`{` + ret.Join(",") + `}`)
	return `{` + ret.Join(",") + `}`

}
示例#6
0
文件: ref.go 项目: olragon/dna
// DumpHashTables uses psql command. Set its path to ENV
func DumpHashTables() {
	var commands = dna.StringArray{}
	for index, table := range GetAlbumTables().Concat(GetSongTables()).Concat(GetVideoTables()) {
		shortForm := table.Replace("songs", "").Replace("albums", "").Replace("videos", "")
		stmt := dna.Sprintf(`SELECT dna_hash(title,artists), ROW(%v,id) from %v`, ToSiteid(shortForm), table)
		command := dna.Sprintf(`psql -c 'COPY (%v) TO STDOUT'`, stmt)
		switch {
		case table.Match("albums") == true:
			command += " >> data/hash_albums.log"
		case table.Match("songs") == true:
			command += " >> data/hash_songs.log"
		case table.Match("videos") == true:
			command += " >> data/hash_videos.log"
		}
		commands.Push(`INTERNAL_TIME=$(date +%s)`)
		commands.Push(dna.Sprintf(`printf "%-3v:Extracting hashids from %-12v "`, index+1, table+"..."))
		commands.Push(command)
		commands.Push(`echo "Completed in $(($(date +%s) - $INTERNAL_TIME)) seconds!"`)
		commands.Push("#--------------------------")
	}
	dna.Log(commands.Join("\n"))
}
示例#7
0
func getStringifiedVideoUrls(urls dna.StringArray) dna.String {
	var baseLink = dna.String("")
	videoUrls := []VideoUrl{}
	urls.ForEach(func(val dna.String, idx dna.Int) {
		// dna.Log(val)
		// Finding bitrate
		switch {
		case val.Match(`MV 360p`) == true:
			songUrl := getVideoUrl(val, "360p")
			baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
			videoUrls = append(videoUrls, *songUrl)
		case val.Match(`MV 480p`) == true:
			songUrl := getVideoUrl(val, "480p")
			baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
			videoUrls = append(videoUrls, *songUrl)
		case val.Match(`MV 180p`) == true:
			songUrl := getVideoUrl(val, "180p")
			baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
			videoUrls = append(videoUrls, *songUrl)
		case val.Match(`HD 720p`) == true:
			songUrl := getVideoUrl(val, "720p")
			songUrl.Link = baseLink + "m4a/file-name.mp4"
			videoUrls = append(videoUrls, *songUrl)
		case val.Match(`HD 1080p`) == true:
			songUrl := getVideoUrl(val, "1080p")
			songUrl.Link = baseLink + "flac/file-name.mp4"
			videoUrls = append(videoUrls, *songUrl)
		}
	})

	var ret = dna.StringArray{}
	for _, videoUrl := range videoUrls {
		t := `"(` + videoUrl.Link + "," + videoUrl.Type + "," + videoUrl.Size.ToString() + "," + videoUrl.Resolution.Replace("p", "") + `)"`
		ret.Push(t)
	}
	// dna.Log(`{` + ret.Join(",") + `}`)
	return `{` + ret.Join(",") + `}`
}
示例#8
0
文件: funcs.go 项目: olragon/dna
// GetInsertIgnoreStatement returns insert ignore statement from a struct. If input value is not struct, it will panic.
//	* tbName : A name of table in database you want to insert
//	* structValue : A struct-typed value. The struct's fields has to be dna basic types (dna.Int, dna.String..) or time.Time
//	* primaryColumn : A name of primary column. If a row is duplicate, it will be discarded.
//	* primaryValue : A value of row needed to be compared.
//	* isPrintable: A param determines where to print the pretty result statement
//	* Return an insert statement
// Notice:  Insert statement uses Dollar-quoted String Constants with special tag "binhdna".
// So string or array is contained between $binhdna$ symbols.
// Therefore no need to escape values.
func GetInsertIgnoreStatement(tbName dna.String, structValue interface{}, primaryColumn dna.String, primaryValue interface{}, isPrintable dna.Bool) dna.String {
	var realKind string
	var columnNames, columnValues dna.StringArray
	tempintslice := []int{0}
	var ielements int
	var kind string = reflect.TypeOf(structValue).Kind().String()
	if kind == "ptr" {
		realKind = reflect.TypeOf(structValue).Elem().Kind().String()

	} else {
		realKind = reflect.TypeOf(structValue).Kind().String()

	}

	if realKind != "struct" {
		panic("Param has to be struct")
	}

	if kind == "ptr" {
		ielements = reflect.TypeOf(structValue).Elem().NumField()
	} else {
		ielements = reflect.TypeOf(structValue).NumField()
	}

	for i := 0; i < ielements; i++ {
		tempintslice[0] = i
		if kind == "ptr" {
			f := reflect.TypeOf(structValue).Elem().FieldByIndex(tempintslice)
			v := reflect.ValueOf(structValue).Elem().FieldByIndex(tempintslice)
			clName, clValue := getColumn(f, v.Interface())
			columnNames.Push(clName)
			columnValues.Push(clValue)
		} else {
			f := reflect.TypeOf(structValue).FieldByIndex(tempintslice)
			v := reflect.ValueOf(structValue).FieldByIndex(tempintslice)
			clName, clValue := getColumn(f, v.Interface())
			columnNames.Push(clName)
			columnValues.Push(clValue)
		}

	}
	condStr := dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v)", tbName, primaryValue)
	if isPrintable == true {
		return "INSERT INTO " + tbName + "\n(" + columnNames.Join(",") + ")\n" + " SELECT " + columnValues.Join(",\n") + " \n" + condStr + ";" + " \n"
	} else {
		return "INSERT INTO " + tbName + "(" + columnNames.Join(",") + ")" + " SELECT " + columnValues.Join(",") + " " + condStr + ";"
	}
}
示例#9
0
文件: funcs.go 项目: olragon/dna
// GetInsertStatement returns insert statement from a struct. If input value is not struct, it will panic.
//	* tbName : A name of table in database you want to insert
//	* structValue : A struct-typed value. The struct's fields has to be dna basic types (dna.Int, dna.String..) or time.Time
//	* isPrintable: A param determines where to print the pretty result statement
//	* Return an insert statement
// Notice:  Insert statement uses Dollar-quoted String Constants with special tag "binhdna".
// So string or array is contained between $binhdna$ symbols.
// Therefore no need to escape values.
func GetInsertStatement(tbName dna.String, structValue interface{}, isPrintable dna.Bool) dna.String {
	var realKind string
	var columnNames, columnValues dna.StringArray
	tempintslice := []int{0}
	var ielements int
	var kind string = reflect.TypeOf(structValue).Kind().String()
	if kind == "ptr" {
		realKind = reflect.TypeOf(structValue).Elem().Kind().String()

	} else {
		realKind = reflect.TypeOf(structValue).Kind().String()

	}

	if realKind != "struct" {
		panic("Param has to be struct")
	}

	if kind == "ptr" {
		ielements = reflect.TypeOf(structValue).Elem().NumField()
	} else {
		ielements = reflect.TypeOf(structValue).NumField()
	}

	for i := 0; i < ielements; i++ {
		tempintslice[0] = i
		if kind == "ptr" {
			f := reflect.TypeOf(structValue).Elem().FieldByIndex(tempintslice)
			v := reflect.ValueOf(structValue).Elem().FieldByIndex(tempintslice)
			clName, clValue := getColumn(f, v.Interface())
			columnNames.Push(clName)
			columnValues.Push(clValue)
		} else {
			f := reflect.TypeOf(structValue).FieldByIndex(tempintslice)
			v := reflect.ValueOf(structValue).FieldByIndex(tempintslice)
			clName, clValue := getColumn(f, v.Interface())
			columnNames.Push(clName)
			columnValues.Push(clValue)
		}

	}
	if isPrintable == true {
		return "INSERT INTO " + tbName + "\n(" + columnNames.Join(",") + ")\n" + "VALUES (\n" + columnValues.Join(",\n") + "\n);"
	} else {
		return "INSERT INTO " + tbName + "(" + columnNames.Join(",") + ")" + " VALUES (" + columnValues.Join(",") + ");"
	}
}
示例#10
0
文件: question.go 项目: olragon/dna
func (q Question) ToRecord() []string {

	if q.Cat.Subject == "non-verbal-reasoning/questions-and-answers" {
		imagesArr := q.QuestionContent.FindAllString(`<img.+?src=.+?>`, -1)
		images := dna.StringArray{}
		for _, image := range imagesArr {
			images.Push("http://indiabix.com" + image.GetTagAttributes("src"))
		}
		return []string{"Non Verbal Reasoning", q.Cat.Name.String(), q.Description.String(), images.Join(",").String(), q.QuestionContent.String(), q.OptionA.String(), q.OptionB.String(), q.OptionC.String(), q.OptionD.String(), q.OptionE.String(), q.Answer.String(), q.Explaination.String(), q.ExerciseNo.ToString().String()}
	} else {
		return []string{q.Cat.Subject.String(), q.Cat.Name.String(), q.Description.String(), q.QuestionContent.String(), q.OptionA.String(), q.OptionB.String(), q.OptionC.String(), q.OptionD.String(), q.OptionE.String(), q.Answer.String(), q.Explaination.String(), q.ExerciseNo.ToString().String()}
	}
}