// Recursively resolve pointers to other factoids
func recurse(fact *factoids.Factoid, keys map[string]bool) {
val := fact.Value
key, start, end := util.FactPointer(val)
if key == "" {
return
}
if _, ok := keys[key]; ok || len(keys) > 20 {
fact.Value = val[:start] + "[circular reference]" + val[end:]
return
}
keys[key] = true
if f2 := fc.GetPseudoRand(key); f2 != nil {
fact.Value = val[:start] + f2.Value + val[end:]
if start == 0 && fact.Type != f2.Type {
// Propagate change of factoid type when the pointer
// is at the beginning of the string.
fact.Type = f2.Type
}
recurse(fact, keys)
return
}
// if we get here, we found a pointer key but no matching factoid
// so recurse on the stuff after that key *only* to avoid loops.
fact.Value = val[end:]
recurse(fact, keys)
fact.Value = val[:end] + fact.Value
}
// Parse a single factoid value, stripping <me>/<reply>
func parseValue(k, r, v string) (ft factoids.FactoidType, fv string) {
v = strings.TrimSpace(v)
ft, fv = factoids.ParseValue(v)
if ft == factoids.F_FACT && fv == v {
// If fv == v, ParseValue hasn't stripped off a <reply>, so this is
// just a normal factoid whose value is actually "key relation value"
// as that's how they're stored in the old SQLite database...
key, _, _ := util.FactPointer(fv)
if _, ok := ptrkeys[k]; !(ok || fv == "*"+key) {
// ... (HACK1) except if they have been used as a pointer key...
// ... (HACK2) or if it's *just* a pointer to another factoid.
fv = strings.Join([]string{k, r, v}, " ")
}
}
return
}
// Recursively resolve pointers to other factoids
func recurse(val string, keys map[string]bool) string {
key, start, end := util.FactPointer(val)
if key == "" {
return val
}
if _, ok := keys[key]; ok || len(keys) > 20 {
val = val[:start] + "[circular reference]" + val[end:]
return val
}
keys[key] = true
if fact := fc.GetPseudoRand(key); fact != nil {
val = val[:start] + fact.Value + val[end:]
return recurse(val, keys)
}
// if we get here, we found a pointer key but no matching factoid
// so recurse on the stuff after that key *only* to avoid loops.
return val[:end] + recurse(val[end:], keys)
}
func main() {
flag.Parse()
logging.InitFromFlags()
// Let's go find some mongo.
db.Init()
defer db.Close()
fc := factoids.Init()
// A communication channel of Factoids.
facts := make(chan *factoids.Factoid)
ptrs := make(chan []interface{})
rows := make(chan []interface{})
// Function to execute some queries on the SQLite db and shove the results
// into the ptrs and rows channels created above.
db_query := func(dbh *sqlite3.Database) {
_, err := dbh.Execute("SELECT * FROM Factoids WHERE Value LIKE '%*%';", feeder(ptrs))
close(ptrs)
if err != nil {
logging.Error("DB error: %s", err)
}
n, err := dbh.Execute("SELECT * FROM Factoids;", feeder(rows))
close(rows)
if err == nil {
logging.Info("Read %d rows from database.", n)
} else {
logging.Error("DB error: %s", err)
}
}
go func() {
sqlite3.Session(*file, db_query)
}()
// First, synchronously read all the stuff from the ptrs channel
// and build a set of all the factoid keys that are used as pointers
for row := range ptrs {
for _, val := range parseMultipleValues(toString(row[cValue])) {
if key, _, _ := util.FactPointer(val); key != "" {
ptrkeys[key] = true
}
}
}
// Now run another goroutine to munge the rows into factoids.
// This was originally done inside the SQLite callbacks, but
// cgo or sqlite3 obscures runtime panics and makes fail happen.
go func() {
for row := range rows {
parseFactoid(row, facts)
}
close(facts)
}()
// And finally...
count := 0
var err error
for fact := range facts {
// ... push each fact into mongo
err = fc.Insert(fact)
if err != nil {
logging.Error("Awww: %v\n", err)
} else {
if count%1000 == 0 {
fmt.Printf("%d...", count)
}
count++
}
}
fmt.Println("done.")
logging.Info("Inserted %d factoids.\n", count)
}