// SelectMissingKeys accepts a table name as an input and a list of keys as a source.
// It returns a new list of keys that does not exist in the destination table
//
// * tblName : a table name
// * srcKeys : a source keys
// * db : a pointer to connected databased
// * Returns a new list of keys which are not from the specified table
//
// Notice: Only applied to a table having a column named "key".
// The column has to be indexed to ensure good performance
//
// The format of sql statement is:
// with dna (key) as (values ('43f3HhhU6DGV'),('uFfgQhKbwAfN'),('RvFDlckJB5QU'),('uIF7rwd5wo4p'),('Kveukbhre1ry'),('oJ1lzAlKwJX6'),('43f3HhhU6DGV'),('uFfgQhKbwAfN'),('hfhtyMdywMau'),('PpZuccjYqy1b'))
// select key from dna where key not in
// (select key from nctalbums where key in ('43f3HhhU6DGV','uFfgQhKbwAfN','RvFDlckJB5QU','uIF7rwd5wo4p','Kveukbhre1ry','oJ1lzAlKwJX6','43f3HhhU6DGV','uFfgQhKbwAfN','hfhtyMdywMau','PpZuccjYqy1b'))
func SelectMissingKeys(tblName dna.String, srcKeys *dna.StringArray, db *sqlpg.DB) (*dna.StringArray, error) {
if srcKeys.Length() > 0 {
val := dna.StringArray(srcKeys.Map(func(val dna.String, idx dna.Int) dna.String {
return `('` + val + `')`
}).([]dna.String))
val1 := dna.StringArray(srcKeys.Map(func(val dna.String, idx dna.Int) dna.String {
return `'` + val + `'`
}).([]dna.String))
selectStmt := "with dna (key) as (values " + val.Join(",") + ") \n"
selectStmt += "select key from dna where key not in \n(select key from " + tblName + " where key in (" + val1.Join(",") + "))"
keys := &[]dna.String{}
err := db.Select(keys, selectStmt)
switch {
case err != nil:
return nil, err
case err == nil && keys != nil:
slice := dna.StringArray(*keys)
return &slice, nil
case err == nil && keys == nil:
return &dna.StringArray{}, nil
default:
panic("Default case triggered. Case is not expected. Cannot select non existed keys")
}
} else {
return nil, errors.New("Empty input array")
}
}
func getTSGM(data *dna.String, kind dna.String) dna.String {
var itemArr dna.StringArray
switch kind {
case "genres":
itemArr = data.FindAllString(`(?mis)<h4>Genre</h4>(.+?)</div>`, 1)
case "styles":
itemArr = data.FindAllString(`(?mis)<h4>Styles</h4>(.+?)</div>`, 1)
case "moods":
itemArr = data.FindAllString(`(?mis)<h4>Album Moods</h4>(.+?)</div>`, 1)
case "themes":
itemArr = data.FindAllString(`(?mis)<h4>Themes</h4>(.+?)</div>`, 1)
default:
panic("Wrong kind!!!")
}
if itemArr.Length() > 0 {
catArr := itemArr[0].FindAllString(`<a href=.+?</a>`, -1)
categories := catArr.Map(func(val dna.String, idx dna.Int) Category {
var idArr []dna.StringArray
var id dna.Int = 0
name := val.RemoveHtmlTags("")
if kind == "moods" {
idArr = val.FindAllStringSubmatch(`xa([0-9]+)`, 1)
} else {
idArr = val.FindAllStringSubmatch(`ma([0-9]+)`, 1)
}
if len(idArr) > 0 {
id = idArr[0][1].ToInt()
}
return Category{id, name}
}).([]Category)
if len(categories) > 0 {
bCat, merr := json.Marshal(categories)
if merr != nil {
return "[]"
} else {
return dna.String(string(bCat))
}
} else {
return "[]"
}
} else {
return "[]"
}
}
func getGenresFromDesc(desc dna.String) dna.StringArray {
var ret dna.StringArray
genres := desc.FindAllString(`(?i)genres?(\s+)?:?.+`, 1)
// "Released:" found in album id: 836258
if genres.Length() > 0 {
ret = dna.StringArray(genres[0].ReplaceWithRegexp(`(?mis)genres?(\s+)?:?`, "").ReplaceWithRegexp(`\.?\s*Released:.+`, "").Trim().Split(",").Map(func(val dna.String, idx dna.Int) dna.String {
return val.ReplaceWithRegexp(":", "").Trim()
}).([]dna.String))
if ret.Length() == 1 {
arr := dna.StringArray{}
if ret[0].FindAllString(`(?mis)K-Pop`, 1).Length() > 0 {
arr.Push("Korean Pop")
arr.Push(ret[0].ReplaceWithRegexp(`(?mis)\(?K-Pop\)?`, "").Trim())
ret = arr
}
}
}
return ret.SplitWithRegexp(` > `).SplitWithRegexp(`/`)
}
func getQuestions(cat Category, isFirstPage dna.Bool, pageUrl dna.String, pageLinks *dna.StringArray) <-chan Questions {
channel := make(chan Questions)
questions := Questions{}
go func() {
var url dna.String
if isFirstPage == true {
url = cat.Url
} else {
url = pageUrl
}
link := "http://www.indiabix.com" + url
// dna.Log(link)
result, err := http.Get(link)
if err == nil {
data := &result.Data
if isFirstPage == true {
// getting page links if the first page found
pageLinkArr := data.FindAllString(`<p class="ib-pager">.+</p>`, 1)
if pageLinkArr.Length() > 0 {
links := pageLinkArr[0].FindAllString(`<a href.+?>`, -1)
for _, link := range links {
pageLinks.Push(link.GetTagAttributes("href"))
}
}
}
// Getting direction for the questions
descriptionArr := data.FindAllString(`(?mis)id="divDirectionText".+?<div class="bix-div-container">`, 1)
description := dna.String("")
if descriptionArr.Length() > 0 {
description = descriptionArr[0].ReplaceWithRegexp(`<div class="bix-div-container">$`, "").Trim().ReplaceWithRegexp(`<td style="padding-left:5px" valign="top">$`, "").ReplaceWithRegexp(`id="divDirectionText">?`, "").Trim().ReplaceWithRegexp(`<tr>$`, "").Trim().ReplaceWithRegexp(`</div></div></td></tr>$`, "").Trim().ReplaceWithRegexp(`^<p>`, "").Trim().ReplaceWithRegexp(`</p>$`, "").Trim()
}
questionTables := data.FindAllString(`(?mis)<table class="bix-tbl-container".+?<hr />`, -1)
// if questionTables.Length() > 0 {
// dna.Log("# of questions: ", questionTables.Length())
// }
for _, questionTable := range questionTables {
question := NewQuestion()
question.Cat = cat
question.Description = description
// Getting question content
questionContentArr := questionTable.FindAllString(`(?mis)<td class="bix-td-qtxt".+?<td class="bix-td-miscell"`, 1)
if questionContentArr.Length() > 0 {
question.QuestionContent = questionContentArr[0].ReplaceWithRegexp(`^<td class="bix-td-qtxt.+?>`, "").ReplaceWithRegexp(`<td class="bix-td-miscell"`, "").Trim().ReplaceWithRegexp(`^<p>`, "").Trim().ReplaceWithRegexp(`<tr>$`, "").Trim().ReplaceWithRegexp(`</tr>$`, "").Trim().ReplaceWithRegexp(`</p></td>$`, "").Trim()
}
// Getting question answers
// answersArr is table html contains a list of options for the question
optionArr := questionTable.FindAllString(`(?mis)<td class="bix-td-option.+tdOptionDt.+?<div class="bix-div-answer"`, -1)
getOptionsForQuestion(optionArr[0], question)
// Getting explaination for the question
explantionArr := questionTable.FindAllString(`(?mis)Explanation:</b></span></p>.+?<div class="bix-div-workspace"`, 1)
if explantionArr.Length() > 0 {
question.Explaination = explantionArr[0].ReplaceWithRegexp(`^Explanation:</b></span></p>`, "").Trim().ReplaceWithRegexp(`^<p>`, "").Trim().ReplaceWithRegexp(`<div class="bix-div-workspace"$`, "").Trim().ReplaceWithRegexp(`</div>$`, "").Trim().ReplaceWithRegexp(`</div>$`, "").Trim().ReplaceWithRegexp(`</p>$`, "").Trim()
}
questions = append(questions, *question)
}
// Getting answers for all question
// If length of answers != length of questions, raise an error
answersArr := data.FindAllString(`<input id="hdnAjaxImageCacheKey".+`, -1)
var answers = dna.StringArray{}
if answersArr.Length() > 0 {
answer := answersArr[0].GetTagAttributes("value")
// $('input' + '#' + 'hdn' + 'Ajax' + 'Image' + 'Cache' + 'Key').val().substr(18).split('').reverse().join('').substr(17).toUpperCase().split('');
tmp := answer.Substring(18, answer.Length()).Split("").Reverse().Join("")
answers = tmp.Substring(17, tmp.Length()).ToUpperCase().Split("")
}
if answers.Length().ToPrimitiveValue() != len(questions) {
panic("Answer length differs from question length - " + cat.ToString())
}
for idx, ans := range answers {
questions[idx].Answer = ans
}
}
channel <- questions
}()
return channel
}