// -----------------------------------------------------------------------------------------------------------------------------------------------
func (dfa *DFA_PoolType) DumpPoolOneState(ii int) string {
pLnNo := com.DbOn("db_DFA_LnNo")
IfLnNo := func(s string) string {
if pLnNo {
t := fmt.Sprintf("[%3s]", s)
return t
}
return ""
}
vv := dfa.Pool[ii]
s := ""
s += fmt.Sprintf("%3d%s: ", ii, IfLnNo(vv.LineNo))
s += fmt.Sprintf(" %12s %12s %s :", vv.StateName, com.SVar(vv.StateSet), com.ChkOrBlank(vv.Visited))
if vv.Rv > 0 {
s += fmt.Sprintf(" T:%04d ", vv.Rv)
} else {
s += fmt.Sprintf(" ")
}
s += fmt.Sprintf("\t E:")
for _, ww := range vv.Next2 {
if ww.IsLambda {
s += fmt.Sprintf("{ ERROR!! \u03bb %2d -> %2d %s} ", ww.From, ww.To, ww.LineNo)
} else {
s += fmt.Sprintf("{ \"%s\" %2d -> %2d %s} ", ww.On, ww.From, ww.To, IfLnNo(ww.LineNo))
}
}
s += "\n"
if vv.Info.Action != 0 || vv.Info.MatchLength != 0 {
s += fmt.Sprintf("\t\t\tDFA.Info: %s", nfa.DumpInfo(vv.Info))
}
s += "\n"
return s
}
// Output the Im structure
func (Im *ImType) OutputImType() {
dpt := []string{"???", "Pat", "EOF", "???"}
if com.DbOn("in-echo-machine") {
Im.OutputDef()
Im.OutputDefAsGoCode(os.Stdout)
for ii, vv := range Im.Machine {
fmt.Printf("Machine[%d] Name[%s]-----------------------------------------------------------------\n", ii, vv.Name)
fmt.Printf(" Mixins: %v\n", vv.Mixins)
// Rules []*ImRuleType //
for jj, ww := range vv.Rules {
s := fmt.Sprintf("%q", ww.Pattern)
s = s[1:]
s = s[0 : len(s)-1]
fmt.Printf(" %3d: %3s %-30s ", jj, dpt[ww.PatternType], s)
if len(ww.RvName) > 0 {
fmt.Printf("%-20s", fmt.Sprintf(" Rv:%d=%s ", ww.Rv, ww.RvName))
} else {
fmt.Printf("%-20s", "")
}
if len(ww.CallName) > 0 {
fmt.Printf("%-20s", fmt.Sprintf(" Call:%d=%s ", ww.Call, ww.CallName))
} else {
fmt.Printf("%-20s", "")
}
if ww.Return {
fmt.Printf(" Return ")
}
if ww.Repl {
fmt.Printf(" Repl:%s ", ww.ReplString)
}
if ww.Ignore {
fmt.Printf(" [Ignore] ")
}
if ww.ReservedWord {
fmt.Printf(" [ReservedWord] ")
}
if ww.Err {
fmt.Printf(" [Err=%s] ", ww.WEString)
}
if ww.Warn {
fmt.Printf(" [Warn=%s] ", ww.WEString)
}
fmt.Printf("\n")
}
}
}
}
// -----------------------------------------------------------------------------------------------------------------------------------------------
func (dfa *DFA_PoolType) OutputInFormat(fo io.Writer, format string) {
// dfa.FinializeDFA()
dt := dfa.MTab
//dt := dfa.ConvertToTable()
//dfa.MTab = &dt
if com.DbOn("output-machine") {
if format == "text" {
fmt.Fprintf(fo, `
Sigma = %q
InitState = %d
N_States = %d
Width = %d
`, dfa.Sigma, dt.InitState, dt.N_States, dt.Width)
SigmaArray := make([]rune, dt.Width, dt.Width)
pp := 0
for jj := 0; jj < dt.Width; jj++ {
if pp < len(dfa.Sigma) {
rn, sz := utf8.DecodeRune([]byte(dfa.Sigma[pp:]))
SigmaArray[jj] = rn
pp += sz
} else {
// SigmaArray[jj] = rune(0xFBAD)
SigmaArray[jj] = dt.SMap.NoMap
}
}
fmt.Fprintf(fo, "SMap = %+v\n", dt.SMap)
fmt.Fprintf(fo, "%-6s : %-4s %-5s %-5s %-4s %-4s | ", "State", "Rv", "Actn", "Hard", "Next", "Leng")
for jj := 0; jj < dt.Width; jj++ {
fmt.Fprintf(fo, " %2d", jj)
}
fmt.Fprintf(fo, "\n")
fmt.Fprintf(fo, "%-6s : %-4s %-5s %-5s %-4s %-4s | ", "======", "====", "-----", "-----", "----", "----")
for jj := 0; jj < dt.Width; jj++ {
fmt.Fprintf(fo, " %2s", "--")
}
fmt.Fprintf(fo, "\n")
fmt.Fprintf(fo, "%-6s : %-4s %-5s %-5s %-4s %-4s | ", " ", " ", " ", " ", " ", " ")
for jj := 0; jj < dt.Width; jj++ {
if SigmaArray[jj] < ' ' {
s := fmt.Sprintf("%q", string(SigmaArray[jj]))
s = s[1:]
fmt.Fprintf(fo, " %2s", s[0:len(s)-1])
} else {
fmt.Fprintf(fo, " %2s", string(SigmaArray[jj]))
}
}
fmt.Fprintf(fo, "\n")
fmt.Fprintf(fo, "%-6s : %-4s %-5s %-5s %-4s %-4s | ", "======", "====", "-----", "-----", "----", "----")
for jj := 0; jj < dt.Width; jj++ {
fmt.Fprintf(fo, " %2s", "--")
}
fmt.Fprintf(fo, "\n")
for ii, vv := range dt.Machine {
tau := " "
if vv.Tau {
tau = "\u03c4"
}
if vv.Info.Action == 0 {
fmt.Fprintf(fo, "m[%3d] : %4d%s %5s %5v %4d %4d | ", ii, vv.Rv, tau, "", vv.Info.HardMatch, vv.Info.NextState, vv.Info.MatchLength)
} else {
fmt.Fprintf(fo, "m[%3d] : %4d%s %5x %5v %4d %4d | ", ii, vv.Rv, tau, vv.Info.Action, vv.Info.HardMatch, vv.Info.NextState, vv.Info.MatchLength)
}
for _, ww := range vv.To {
if ww == -1 {
// fmt.Fprintf(fo, " %2s", "\u26d4") // No Entry
fmt.Fprintf(fo, " %2s", "\u2629")
} else {
fmt.Fprintf(fo, " %2d", ww)
}
}
fmt.Fprintf(fo, "\n")
}
fmt.Fprintf(fo, "\n")
} else {
fmt.Fprintf(os.Stderr, "Invalid Output Format for dfa.OutputInFomrat, %s\n", format)
}
}
}
func (dfa *DFA_PoolType) ConvNDA_to_DFA(nn *nfa.NFA_PoolType) {
StartState := nn.InitState
dfa.NoneVisited()
nn.Sigma = nn.GenerateSigma()
dfa.Sigma = nn.Sigma
com.DbPrintf("dfa2", "Sigma at top ->%s<-\n", dfa.Sigma)
// Build initial state
dfa_set := nn.LambdaClosure([]int{StartState}) // Find all the lambda closures from specified state
dfa_set = append(dfa_set, StartState) // Add in initial state
dfa_set = com.USortIntSlice(dfa_set) // Make set unique
com.DbPrintf("db_DFAGen", "\nStart: %s, \u03a3 =->%s<-, %s\n", com.SVar(dfa_set), dfa.Sigma, com.LF())
A := dfa.GetDFAName(dfa_set)
if r, is, info, Is0Ch := nn.IsTerminalState(dfa_set); is {
dfa.Pool[A].Rv = r
dfa.Pool[A].Is0Ch = Is0Ch
dfa.Pool[A].Info = info
} else {
dfa.Pool[A].Info = info
}
if com.DbOn("db_DFAGen") {
dfa.DumpPool(false)
}
// Look at all the locaitons we can get to from this "state"
for _, S := range dfa.Sigma {
StateSet := nn.LambdaClosureSet(dfa_set, string(S))
com.DbPrintf("db_DFAGen", "FOR INITIAL state ->%s<- StateSet=%s, %s\n", string(S), com.SVar(StateSet), com.LF())
if len(StateSet) > 0 {
com.DbPrintf("db_DFAGen", "Have a non-empty result, %s\n", com.LF())
com.DbPrintf("db_DFAGen", "<><><> this is the point where we should check to see if 'S' is DOT or NCCL, %s\n", com.LF())
StateSetT := nn.LambdaClosure(StateSet) // need to lambda complete the state set
StateSet = append(StateSet, StateSetT...)
StateSet = com.USortIntSlice(StateSet) // Make set unique
com.DbPrintf("db_DFAGen", " Output Is %s, %s\n", com.SVar(StateSet), com.LF())
B := 0
if t := dfa.HaveStateAlready(StateSet); t != -1 { // Have Already
B = t
com.DbPrintf("db_DFAGen", " Already have this state at location %d, %s\n", t, com.LF())
} else {
B = dfa.GetDFAName(StateSet)
com.DbPrintf("db_DFAGen", " *** New state %d, %s\n", B, com.LF())
}
dfa.AddEdge(A, B, string(S))
com.DbPrintf("db_DFAGen", " *** Before (top) %s\n", com.LF())
if r, is, info, Is0Ch := nn.IsTerminalState(StateSet); is {
dfa.Pool[B].Rv = r
dfa.Pool[B].Is0Ch = Is0Ch
dfa.Pool[B].Info = info
com.DbPrintf("db_DFAGen", " *** New state %d, %s\n", B, com.LF())
} else if _, is, info, Is0Ch := nn.IsNonTerminalPushPopState(StateSet); is {
dfa.Pool[B].Is0Ch = Is0Ch
dfa.Pool[B].Info = info
com.DbPrintf("db_DFAGen", " *** New info for state %d, %s\n", B, com.LF())
} else {
dfa.Pool[B].Info = info
com.DbPrintf("db_DFAGen", " *** NO State Info for state %d, %s\n", B, com.LF())
}
com.DbPrintf("db_DFAGen", " *** After (top) %s\n", com.LF())
if com.DbOn("db_DFAGen") {
fmt.Printf("for %s StateSet=%s, A=%d, B=%s %s\n", string(S), com.SVar(StateSet), A, com.SVar(B), com.LF())
dfa.DumpPool(false)
}
}
}
dfa.Pool[A].Visited = true
com.DbPrintf("db_DFAGen", "\nBefore Main Loop, %s\n", com.LF())
limit := 0
for stateToDo := dfa.FindNotVisited(); stateToDo != -1; stateToDo = dfa.FindNotVisited() {
com.DbPrintf("db_DFAGen", "\nMain Loop: !!TOP!! State:%d\n", stateToDo)
// -----------------------------------------------------------------------------------------------------------
if !dfa.Pool[stateToDo].Visited {
dfa_set := nn.LambdaClosure(dfa.Pool[stateToDo].StateSet) // Find all the lambda closures from specified state
dfa_set = append(dfa_set, dfa.Pool[stateToDo].StateSet...) // Add in initial state
dfa_set = com.USortIntSlice(dfa_set) // Make set unique
for _, S := range dfa.Sigma {
StateSet := nn.LambdaClosureSet(dfa_set, string(S))
com.DbPrintf("db_DFAGen", " for initial state %s StateSet=%s, %s\n", string(S), com.SVar(StateSet), com.LF())
com.DbPrintf("db_DFAGen", "<><><> this is the point where we should check to see if 'S' is DOT or NCCL, %s\n", com.LF())
if len(StateSet) > 0 {
com.DbPrintf("db_DFAGen", " >>> Have a non-empty result, Input Is %s, %s\n", com.SVar(StateSet), com.LF())
StateSetT := nn.LambdaClosure(StateSet) // need to lambda complete the state set
StateSet = append(StateSet, StateSetT...)
StateSet = com.USortIntSlice(StateSet) // Make set unique
com.DbPrintf("db_DFAGen", " >>> Output Is %s, %s\n", com.SVar(StateSet), com.LF())
B := 0
if t := dfa.HaveStateAlready(StateSet); t != -1 { // Have Already
B = t
com.DbPrintf("db_DFAGen", " Already have this state at location %d, %s\n", t, com.LF())
} else {
B = dfa.GetDFAName(StateSet)
}
dfa.AddEdge(stateToDo, B, string(S))
com.DbPrintf("db_DFAGen", " *** Before %s\n", com.LF())
if r, is, info, Is0Ch := nn.IsTerminalState(StateSet); is {
dfa.Pool[B].Rv = r
dfa.Pool[B].Is0Ch = Is0Ch
dfa.Pool[B].Info = info
com.DbPrintf("db_DFAGen", " *** New state %d, %s\n", B, com.LF())
} else if _, is, info, Is0Ch := nn.IsNonTerminalPushPopState(StateSet); is {
dfa.Pool[B].Is0Ch = Is0Ch
dfa.Pool[B].Info = info
com.DbPrintf("db_DFAGen", " *** New info for state %d, %s\n", B, com.LF())
} else {
com.DbPrintf("db_DFAGen", " *** NO State Info for state %d, %s\n", B, com.LF())
dfa.Pool[B].Info = info
}
com.DbPrintf("db_DFAGen", " *** After %s\n", com.LF())
if com.DbOn("db_DFAGen") {
fmt.Printf(" Add New Edge on %s fr %d to %d, %s\n", string(S), stateToDo, B, com.LF())
fmt.Printf(" for %s StateSet=%s, A(stateToDo)=%d, %s\n", string(S), com.SVar(StateSet), stateToDo, com.LF())
dfa.DumpPool(false)
}
}
}
}
// -----------------------------------------------------------------------------------------------------------
dfa.Pool[stateToDo].Visited = true
limit++
if limit > 50000 {
break
}
}
dfa.VerifyMachine()
}
func (dfa *DFA_PoolType) DumpPool(all bool) {
if all {
com.DbPrintf("db_DumpDFAPool", "Cur: %d Top: %d NextFree %d\n", dfa.Cur, dfa.Top, dfa.NextFree)
}
com.DbPrintf("db_DumpDFAPool", "\n---------------------------- DFA Output -----------------------------------------------\n")
com.DbPrintf("db_DumpDFAPool", "\nDFA InitState: %d, Sigma ->%s<-\n\n", dfa.InitState, dfa.Sigma)
pLnNo := com.DbOn("db_DFA_LnNo")
IfLnNo := func(s string) string {
if pLnNo {
t := fmt.Sprintf("[%3s]", s)
return t
}
return ""
}
com.DbPrintf("db_DumpDFAPool", "%3s%s: ", "St", IfLnNo("/Ln"))
com.DbPrintf("db_DumpDFAPool", " %12s %12s \u2714 \tEdges", "StateName", "StateSet")
com.DbPrintf("db_DumpDFAPool", "\n\n")
for ii, vv := range dfa.Pool {
if all || vv.IsUsed {
com.DbPrintf("db_DumpDFAPool", "%3d%s: ", ii, IfLnNo(vv.LineNo))
com.DbPrintf("db_DumpDFAPool", " %12s %12s %s :", vv.StateName, com.SVar(vv.StateSet), com.ChkOrBlank(vv.Visited))
if vv.Rv > 0 {
if vv.Is0Ch {
com.DbPrintf("db_DumpDFAPool", " \u03c4:Tau:%04d ", vv.Rv)
} else {
com.DbPrintf("db_DumpDFAPool", " T:%04d ", vv.Rv)
}
} else {
com.DbPrintf("db_DumpDFAPool", " ")
}
if com.DbOn("db_DumpDFAPool") {
fmt.Printf("\t E:")
for _, ww := range vv.Next2 {
if ww.Is0ChMatch {
fmt.Printf("//Found \u03c4 (%s) //", com.LF()) // Show a Tau(t) for a lambda that matchiens on else conditions.
}
if ww.IsLambda {
fmt.Printf("{ ERROR!! \u03bb %2d -> %2d %s} ", ww.From, ww.To, ww.LineNo)
} else {
// fmt.Printf("{ \"%s\" %2d -> %2d %s} ", ww.On, ww.From, ww.To, IfLnNo(ww.LineNo))
on, _ := utf8.DecodeRune([]byte(ww.On))
son := fmt.Sprintf("%q", ww.On)
switch on {
case re.R_DOT: // = '\uF8FA' // Any char in Sigma
son = "DOT/uF8FA"
case re.R_BOL: // = '\uF8F3' // Beginning of line
son = "BOL/uF8F3"
case re.R_EOL: // = '\uF8F4' // End of line
son = "EOL/uF8F4"
case re.R_NUMERIC: // = '\uF8F5'
son = "NUMERIC/uF8F5"
case re.R_LOWER: // = '\uF8F6'
son = "LOWER/uF8F6"
case re.R_UPPER: // = '\uF8F7'
son = "UPPER/uF8F7"
case re.R_ALPHA: // = '\uF8F8'
son = "ALPHA/uF8F8"
case re.R_ALPHNUM: // = '\uF8F9'
son = "ALPHANUM/uF8F9"
case re.R_EOF: // = '\uF8FB'
son = "EOF/uF8FB"
case re.R_not_CH: // = '\uF8FC' // On input lookup if the char is NOT in Signa then it is returned as this.
son = "else_CH/uF8Fc"
case re.R_N_CCL: // = '\uF8FD' // If char is not matched in this state then take this path
son = "N_CCL/uF8Fd"
case re.R_LAMBDA_MATCH: // = '\uF8FE'
son = "LambdaM/uF8FE"
}
fmt.Printf("{ %s %2d -> %2d %s} ", son, ww.From, ww.To, IfLnNo(ww.LineNo))
}
}
fmt.Printf("\n")
if vv.Info.Action != 0 || vv.Info.MatchLength != 0 {
// fmt.Printf("\t\t\tInfo: %s\n", com.SVar(vv.Info)) // xyzzy - output Info
// xyzzy - NextState info
fmt.Printf("\t\t\tDFA.Info: %s", nfa.DumpInfo(vv.Info))
// if ((vv.Info.Action&com.A_Pop) != 0 || (vv.Info.Action&com.A_Push) != 0 || (vv.Info.Action&com.A_Reset) != 0) && !vv.Info.HardMatch { // xyzzy8
fmt.Printf(" IsHard=%v (((false imples else case Rv!)))\n", vv.Info.HardMatch)
}
fmt.Printf("\n")
}
}
}
}
func Test_PbBufer01(t *testing.T) {
SymbolTable := make([]*MacroDefTestType, 0, 100)
Define := func(name rune, body string) {
for ii := 0; ii < len(SymbolTable); ii++ {
// fmt.Printf("Search at %d, for %s\n", ii, string(name))
if SymbolTable[ii] != nil && SymbolTable[ii].Rn == name {
SymbolTable[ii].Body = body
return
}
}
// fmt.Printf("Append\n")
SymbolTable = append(SymbolTable, &MacroDefTestType{Rn: name, Body: body})
}
ResetST := func() {
// SymbolTable = make([]*MacroDefTestType, 0, 100)
SymbolTable = SymbolTable[:1]
}
HaveMacro := func(name rune) (body string, found bool) {
body = ""
found = false
for ii := 0; ii < len(SymbolTable); ii++ {
if SymbolTable[ii] != nil && SymbolTable[ii].Rn == name {
body, found = SymbolTable[ii].Body, true
return
}
}
return
}
for ii, vv := range Pb01Test {
if !vv.SkipTest {
// Implement a quick - fetch execute macine to test - the PbBuffer - commands/opcodes are the Cmd* constants above.
ss := ""
pb := NewPbRead()
ResetST()
for pc, ww := range vv.Actions {
switch ww.OpCode {
case CmdOpenFile: // Open a file , at the tail end of list of input
pb.OpenFile(ww.Fn)
com.DbPrintf("testCode", "Open file %s At: %s\n", ww.Fn, com.LF())
if com.DbOn("testDump") {
pb.Dump01(os.Stdout)
}
case CmdPbString: // Push back a string
pb.PbString(ww.Data)
if com.DbOn("testDump") {
pb.Dump01(os.Stdout)
}
case CmdPbRune: // Push back a rune
pb.PbRune(ww.Rn)
if com.DbOn("testDump") {
pb.Dump01(os.Stdout)
}
case CmdPbRuneArray: // Push back a rune array
pb.PbRuneArray(ww.RnS)
if com.DbOn("testDump") {
pb.Dump01(os.Stdout)
}
case CmdNextNChar:
for ll := 0; ll < ww.X; ll++ {
rn, done := pb.NextRune()
if !done {
ss = ss + string(rn)
}
com.DbPrintf("testCode", "Case 5: At: ->%s<- ll=%d ss >>>%s<<< %s\n", string(rn), ll, ss, com.LF())
}
if com.DbOn("testDump") {
pb.Dump01(os.Stdout)
}
case CmdPeek:
rn, done := pb.PeekRune()
if done || rn != ww.Rn {
t.Errorf("%04s: Peek at [pc=%d] in test [%s] did not work, got %s expected %s, done=%v\n", pc, ii, string(rn), string(ww.Rn), done)
}
case CmdOutputToEof:
com.DbPrintf("testCode", "All Done: ss >>>%s<<< before At: %s\n", ss, com.LF())
if com.DbOn("testDump") {
pb.Dump01(os.Stdout)
}
for rn, done := pb.NextRune(); !done; rn, done = pb.NextRune() {
ss = ss + string(rn)
}
com.DbPrintf("testCode", "All Done: ss >>>%s<<< after At: %s\n", ss, com.LF())
case CmdPbByteArray: // Push back a byte array
pb.PbByteArray([]byte(ww.Data))
if com.DbOn("testDump") {
pb.Dump01(os.Stdout)
}
case CmdPbFile: // Open file and push contents back onto input at head of list. (Macro file, Include, Require)
pb.PbFile(ww.Fn)
com.DbPrintf("testCode", "Pb file %s At: %s\n", ww.Fn, com.LF())
if com.DbOn("testDump") {
pb.Dump01(os.Stdout)
}
case CmdFileSeen:
fs := pb.FileSeen(ww.Fn)
if fs != ww.FileSeenFlag {
t.Errorf("%04s: Peek at [pc=%d] in test [%s] did not work, got %v expected %s for file seen flagv\n", pc, ii, fs, ww.FileSeenFlag)
}
case CmdGetPos: // Check get file name
ln, cn, fn := pb.GetPos()
com.DbPrintf("testCode", "fn=%s ln=%d cn=%d\n", fn, ln, cn)
if ln != ww.LineNo {
t.Errorf("%04s: %d: did not match line no Expected ->%d<-, Got ->%d<-\n", vv.Test, pc, ww.LineNo, ln)
}
if cn != ww.ColNo {
t.Errorf("%04s: %d: did not match col no Expected ->%d<-, Got ->%d<-\n", vv.Test, pc, ww.ColNo, cn)
}
if fn != ww.Fn {
t.Errorf("%04s: %d: did not match file name Expected ->%s<-, Got ->%s<-\n", vv.Test, pc, ww.Fn, fn)
}
case CmdSetPos: // Check get file name
pb.SetPos(ww.LineNo, ww.ColNo, ww.Fn)
case CmdResetST: // Reset symbol table
ResetST()
case CmdMacroProc: // Apply 1 char macros to input and process
for rn, done := pb.NextRune(); !done; rn, done = pb.NextRune() {
if m_body, m_found := HaveMacro(rn); m_found {
pb.PbString(m_body)
} else {
ss = ss + string(rn)
}
}
case CmdDefineMacro: // Define
Define(ww.Rn, ww.Data)
case CmdDumpBuffer: // Dump the buffer - debuging
if ww.Fn == "" {
pb.Dump01(os.Stdout)
} else {
fp, err := com.Fopen(ww.Fn, "w")
if err == nil {
pb.Dump01(fp)
fp.Close()
} else {
pb.Dump01(os.Stdout)
t.Errorf("%04s: Unable to open file for output ->%s<-, error: %s\n", vv.Test, ww.Fn, err)
}
}
case CmdResetOutput: // Reset output
ss = ""
case CmdPushBackXCopies: // Special test to pub back more than buffer of 'x'
x := strings.Repeat(ww.Data, ww.X)
pb.PbString(x)
}
}
if ss != vv.Results {
t.Errorf("%04s: did not match Expected ->%s<-, Got ->%s<-\n", vv.Test, vv.Results, ss)
} else {
com.DbPrintf("testCode", "%04s: Passed ------------------------------------------------------------------------------------------------\n\n", vv.Test)
}
}
}
}
func (lr *LexReType) Err(s string) {
if com.DbOn("OutputErrors") {
fmt.Printf("Error: %s\n", s)
}
lr.Error = append(lr.Error, errors.New(fmt.Sprintf("Error: %s, %s", s, com.LF(2))))
}
func (tl *TokenList) FlushTokenBuffer(TokStart int, isHard bool, atEof bool) {
ok := true
if !isHard {
n := len(tl.TL) - 1
if tl.TL[n].AToken.TokNo != tl.IgnoreToken {
tl.TokenData = append(tl.TokenData, tl.TL[n].AToken)
}
tl.TL = tl.TL[:0]
tl.EndIdx = make(map[int]int)
return
}
if com.DbOn("db_FlushTokenBeforeBefore") {
fmt.Printf("Before Flush, TokStart=%d, eof = %v\n", TokStart, atEof)
tl.DumpTokenBuffer()
}
keepTok := make([]int, 0, len(tl.TL)) // Set of tokens to keepTokerve
//com.DbPrintf("db_tok01","tl.EndIdx = %+v\n", tl.EndIdx)
// Walk Backwards Creating List
limit := 15
for ii := len(tl.TL) - 1; ii >= 0 && limit > 0; limit-- {
top_ii := ii
com.DbPrintf("db_tok01", "At top of loop, ii = %d, tl.TL[ii]=%+v\n", ii, tl.TL[ii])
keepTok = append(keepTok, ii)
if ii <= 0 {
break
}
end := tl.TL[ii].Start
ii, ok = tl.EndIdx[end]
if atEof {
ii--
}
com.DbPrintf("db_tok01", "CCC ii=%d ok=%v end=%d atEof=%v\n", ii, ok, end, atEof)
if !ok {
if com.DbOn("OutputErrors") {
fmt.Printf("Note: Failed to flush - invalid token set, ii=%v, end=%v, %s\n", ii, end, com.LF())
}
break
}
if end == 0 {
fmt.Printf("Reached and end of 0\n")
break
}
l := tl.TL[top_ii].End - tl.TL[top_ii].Start
d := len(tl.TL[top_ii].AToken.Match)
//com.DbPrintf("db_tok01","DDD top_ii=%d l=%d d=%d before ->%s<-\n", top_ii, l, d, tl.TL[top_ii].AToken.Match)
if d-l >= 0 && d-l < d && top_ii >= 0 && top_ii < len(tl.TL) {
tl.TL[top_ii].AToken.Match = tl.TL[top_ii].AToken.Match[d-l:]
}
//com.DbPrintf("db_tok01","EEE top_ii=%d l=%d d=%d after ->%s<-\n", top_ii, l, d, tl.TL[top_ii].AToken.Match)
if tl.TL[top_ii].AToken.IsRepl {
tl.TL[top_ii].AToken.Val = tl.TL[top_ii].AToken.ReplStr
} else {
tl.TL[top_ii].AToken.Val = tl.TL[top_ii].AToken.Val[d-l:]
}
}
last := len(tl.TokenData) - 1
com.DbPrintf("db_tok01", "keepTok = %s, last=%d\n", com.SVar(keepTok), last)
for ii := len(keepTok) - 1; ii >= 0; ii-- {
vv := keepTok[ii]
// xyzzy - if not an Ignore token then
if tl.TL[vv].AToken.TokNo != tl.IgnoreToken {
tl.TokenData = append(tl.TokenData, tl.TL[vv].AToken)
}
}
// send EOF token -- Handeled in matcher better
//if atEof {
// tl.TokenData = append(tl.TokenData, Token{TokNo: 37}) // tl.TokenData = append(tl.TokenData, Token{TokNo: Tok_EOF})
//}
tl.TL = tl.TL[:0]
tl.EndIdx = make(map[int]int)
if com.DbOn("db_FlushTokenBeforeAfter") {
fmt.Printf("After Flush\n")
fmt.Printf("--------------------------------------------------------\n")
tl.DumpTokenBuffer()
fmt.Printf("--------------------------------------------------------\n")
}
}
func (lex *Lexie) NewReadFile(path string) {
lex.Im = in.ImReadFile(path)
lex.NFA_Machine = make([]*nfa.NFA_PoolType, 0, 100)
lex.DFA_Machine = make([]*DFA_PoolType, 0, 100)
// vv=in.ImDefinedValueType {Seq:1 WhoAmI:ReservedWords NameValueStr:map[and:Tok_L_AND not:Tok_not as:Tok_as in:Tok_in bor:Tok_B_OR band:Tok_B_AND xor:Tok_XOR or:Tok_L_OR true:Tok_true false:Tok_false export:Tok_export] NameValue:map[and:4 true:32 as:34 bor:42 band:41 xor:64 or:5 false:33 not:31 export:35 in:28] Reverse:map[5:or 32:true 42:bor 31:not 41:band 35:export 33:false 28:in 64:xor 4:and 34:as] SeenAt:map[bor:{LineNo:[39] FileName:[unk-file]} band:{LineNo:[39] FileName:[unk-file]} and:{LineNo:[39] FileName:[unk-file]} true:{LineNo:[39] FileName:[unk-file]} export:{LineNo:[39] FileName:[unk-file]} in:{LineNo:[39] FileName:[unk-file]} as:{LineNo:[39] FileName:[unk-file]} or:{LineNo:[39] FileName:[unk-file]} false:{LineNo:[39] FileName:[unk-file]} not:{LineNo:[39 39] FileName:[unk-file unk-file]} xor:{LineNo:[39] FileName:[unk-file]}]}, File: /Users/corwin/Projects/pongo2/lexie/dfa/match.go LineNo:260
for ii, vv := range lex.Im.Def.DefsAre {
// ["ReservedWords"] {
// func (st *SymbolTable) DefineReservedWord(name string, fxid int) (ss *SymbolType) {
_ = ii
_ = vv
com.DbPrintf("dfa5", "vv=%T %+v, %s\n", vv, vv, com.LF())
}
for ii, vv := range lex.Im.Machine {
nm := vv.Name
Nfa := nfa.NewNFA_Pool()
Cur := Nfa.GetNFA()
Nfa.InitState = Cur
for jj, ww := range vv.Rules {
rVx := ww.Rv
if ww.ReservedWord {
com.DbPrintf("dfa5", "This rule rv=%d is a reserved word rule, AAbbCC\n", rVx)
}
ww_A := convRuleToActionFlag(ww)
if ww.Repl {
rVx = 9900 // 9900 is replace
com.DbPrintf("match", "###################################### ww.Replace: ii=%d jj=%d ->%s<-, %s\n", ii, jj, ww.ReplString, com.LF())
}
cur := -1
if ww.PatternType == 2 {
com.DbPrintf("db_Matcher_02", "ADDING AT %2d RE: %-30s (Rv:%2d, final=%4d), %s\n", jj, "<M_EOF>", ww.Rv, rVx, com.LF())
cur = Nfa.AddReInfo(re.X_EOF, "", jj+1, rVx, nfa.InfoType{Action: ww_A, NextState: ww.Call})
} else {
com.DbPrintf("db_Matcher_02", "M= ->%s<- Adding at %2d RE: %-30s (Rv:%2d, final=%4d), %s\n", ww.Pattern, jj, ww.Pattern, ww.Rv, rVx, com.LF())
cur = Nfa.AddReInfo(ww.Pattern, "", jj+1, rVx, nfa.InfoType{Action: ww_A, NextState: ww.Call, ReplStr: ww.ReplString})
}
if ww.ReservedWord {
Nfa.SetReservedWord(cur)
}
}
com.DbPrintf("match", "BuildDFA_2: Nfa.Sigma Before Finialize->%s<-\n", Nfa.Sigma)
if com.DbOn("db_Matcher_02") {
com.DbPrintf("match", "NFA for (Before Finialize) ->%s<-\n", nm)
Nfa.DumpPool(false)
}
Nfa.FinializeNFA()
com.DbPrintf("match", "BuildDFA_2: Nfa.Sigma ->%s<-\n", Nfa.Sigma)
if com.DbOn("db_Matcher_02") {
com.DbPrintf("match", "Final NFA for ->%s<-\n", nm)
Nfa.DumpPool(false)
}
lex.NFA_Machine = append(lex.NFA_Machine, Nfa)
Dfa := NewDFA_Pool()
Dfa.ConvNDA_to_DFA(Nfa)
if com.DbOn("db_Matcher_02") {
com.DbPrintf("match", "Final DFA for ->%s<-\n", nm)
Dfa.DumpPool(false)
}
lex.DFA_Machine = append(lex.DFA_Machine, Dfa)
if com.DbOn("db_Matcher_02") {
last := len(lex.DFA_Machine) - 1
newFile := fmt.Sprintf("../ref/mmm_%s_%d.tst", "machine", last)
gvFile := fmt.Sprintf("../ref/mmm_%s_%d.gv", "machine", last)
svgFile := fmt.Sprintf("../ref/mmm_%s_%d.svg", "machine", last)
fp, _ := com.Fopen(newFile, "w")
lex.DFA_Machine[last].DumpPoolJSON(fp, fmt.Sprintf("Lex-Machine-%d", last), 1)
fp.Close()
gv, _ := com.Fopen(gvFile, "w")
lex.DFA_Machine[last].GenerateGVFile(gv, fmt.Sprintf("Lex-Machine-%d", last), 1)
gv.Close()
out, err := exec.Command("/usr/local/bin/dot", "-Tsvg", "-o"+svgFile, gvFile).Output()
if err != nil {
com.DbPrintf("match", "%sError%s from dot, %s, %s\n", com.Red, com.Reset, err, com.LF())
com.DbPrintf("match", "Output: %s\n", out)
}
}
}
}
