func (lr *LexReType) ParseRe(ss string) {
com.DbPrintf("db2", "at %s\n", com.LF())
lr.SetBuf(ss)
com.DbPrintf("db2", "at %s\n", com.LF())
lr.parseExpression(0, 0, nil)
com.DbPrintf("db2", "at %s\n", com.LF())
}
//
// What can I see at the top of a RE
//
// LR_Text //
// LR_EOF //
// LR_DOT // . -- Match any char
// LR_STAR // * -- Error if 1st char
// LR_PLUS // + -- Error if 1st char
// LR_QUEST // ? -- Error if 1st char
// LR_B_CCL // [ -- Start of CCL Node
// LR_E_CCL // ]
// LR_OP_PAR // ( -- Start of Sub_Re
// LR_CL_PAR // )
// LR_CCL // [...] -- CCL Node (Above)
// LR_N_CCL // [^...] -- N_CCL Node
// LR_CARROT // ^ -- BOL
// LR_MINUS // - -- Text if not in CCL and not 1st char in CCL
//
// Item string
// LR_Tok LR_TokType
// Children []*ReTreeNodeType
// Next *ReTreeNodeType
//
func (lr *LexReType) DumpParseNodesChild(ch []ReTreeNodeType, d int) {
com.DbPrintf("DumpParseNodes", "\n%sDumpParseNodesChild: At %s\n", N4Blanks(d), com.LF())
for ii, vv := range ch {
com.DbPrintf("DumpParseNodes", "%sat %s [step %d] ", N4Blanks(d), com.LF(), ii)
com.DbPrintf("DumpParseNodes", "Item: [%s] %d=%s, N-Children=%d\n", vv.Item, vv.LR_Tok, NameOfLR_TokType(vv.LR_Tok), len(vv.Children))
if len(vv.Children) > 0 {
lr.DumpParseNodesChild(vv.Children, d+1)
}
}
com.DbPrintf("DumpParseNodes", "%sDumpParseNodesChild: Done %s\n\n", N4Blanks(d), com.LF())
}
// dfa.TokList.ReplaceToken ( dfa.MTab.Machine[ctx.St].Info.MatchLength, dfa.MTab.Machine[ctx.St].Info.ReplStr )
func (tl *TokenList) ReplaceToken(l int, s string) {
ii := len(tl.TL) - 1
lv := len(tl.TL[ii].AToken.Val)
tl.TL[ii].AToken.IsRepl = true
tl.TL[ii].AToken.ReplStr = s
if lv-l >= 1 {
tl.TL[ii].AToken.Val = tl.TL[ii].AToken.Val[0:lv-l] + s
} else {
com.DbPrintf("db_tok01", "Error: ReplaceToken has invalid data, %s\n", com.LF())
}
com.DbPrintf("db_tok01", "ReplaceToken: Match: ->%s<- Val: ->%s<-\n", tl.TL[ii].AToken.Match, tl.TL[ii].AToken.Val)
}
func (lr *LexReType) DumpParseNodes() {
com.DbPrintf("DumpParseNodes", "\nDumpParseNodes: At %s\n", com.LF())
for ii, vv := range lr.Tree.Children {
com.DbPrintf("DumpParseNodes", "at %s [step %d] ", com.LF(), ii)
com.DbPrintf("DumpParseNodes", "Item: [%s] %d=%s, N-Children=%d\n", vv.Item, vv.LR_Tok, NameOfLR_TokType(vv.LR_Tok), len(vv.Children))
if len(vv.Children) > 0 {
lr.DumpParseNodesChild(vv.Children, 1)
}
}
com.DbPrintf("DumpParseNodes", "DumpParseNodes: Done %s\n\n", com.LF())
com.DbPrintf("DumpParseNodesX", "DumpParseNodes: %s\n\n", com.SVarI(lr.Tree))
}
func (lex *Lexie) OutputActionFlags(dfa *DFA_PoolType) {
com.DbPrintf("match", "Action Flags Are:\n")
// com. ConvertActionFlagToString(kk int) (rv string) {
dn := make(map[int]bool)
for _, vv := range dfa.Pool {
if vv.IsUsed {
if vv.Info.Action != 0 {
if _, ok := dn[vv.Info.Action]; !ok {
com.DbPrintf("match", " %2x: %s\n", vv.Info.Action, com.ConvertActionFlagToString(vv.Info.Action))
dn[vv.Info.Action] = true
}
}
}
}
}
// Get the current line/col no and file name
func (pb *PBReadType) GetPos() (LineNo int, ColNo int, FileName string) {
com.DbPrintf("pbbuf02", "At: %s\n", com.LF())
if len(pb.PbBuffer) > 0 {
com.DbPrintf("pbbuf02", "From Buffer At: %s\n", com.LF())
LineNo = pb.PbBuffer[0].LineNo
ColNo = pb.PbBuffer[0].ColNo
FileName = pb.PbBuffer[0].FileName
} else {
com.DbPrintf("pbbuf02", "Not set At: %s\n", com.LF())
LineNo = 1
ColNo = 1
FileName = ""
}
return
}
// Open a file - this puts the file at the end of the input. This is used on the command line for a list of files
// in order. Each opened and added to the end of the list.
func (pb *PBReadType) OpenFile(fn string) (err error) {
com.DbPrintf("pbbuf01", "At: %s\n", com.LF())
pb.FileName = fn
pb.AbsFileName, _ = filepath.Abs(fn)
pb.FilesOpened[pb.AbsFileName] = true
// read file -> PbBuffer
b := &ABuffer{
FileName: fn,
AbsFileName: pb.AbsFileName,
LineNo: 1,
ColNo: 1,
}
pb.PbBuffer = append(pb.PbBuffer, b)
bb, err := ioutil.ReadFile(fn)
if err != nil {
return
}
b.EofOnFile = true
b.Pos = 0
var rn rune
var sz int
b.Buffer = make([]rune, 0, len(bb))
for ii := 0; ii < len(bb); ii += sz {
rn, sz = utf8.DecodeRune(bb[ii:])
b.Buffer = append(b.Buffer, rn)
}
return nil
}
// Return the next rune. If runes have been pushed back then use those first.
func (pb *PBReadType) NextRune() (rn rune, done bool) {
com.DbPrintf("pbbuf01", "At: %s\n", com.LF())
done = false
if pb.PbTop > 0 {
pb.PbTop--
rn = pb.PbAFew[pb.PbTop]
} else if len(pb.PbBuffer) <= 0 {
done = true
// } else if len(pb.PbBuffer) == 1 && pb.PbBuffer[0].Pos >= len(pb.PbBuffer[0].Buffer) && !pb.PbBuffer[0].EofOnFile {
// Xyzzy - read in more form file - append
// so far case never happens because EofOnFile is constant true at init time.
} else if len(pb.PbBuffer) == 1 && pb.PbBuffer[0].Pos >= len(pb.PbBuffer[0].Buffer) && pb.PbBuffer[0].EofOnFile {
done = true
} else if len(pb.PbBuffer) > 1 && pb.PbBuffer[0].Pos >= len(pb.PbBuffer[0].Buffer) && pb.PbBuffer[0].EofOnFile {
pb.PbBuffer = pb.PbBuffer[1:]
return pb.NextRune()
} else {
//fmt.Printf("Just before core, Pos=%d\n", pb.PbBuffer[0].Pos)
//fmt.Printf("Just before core, Len 1=%d\n", len(pb.PbBuffer[0].Buffer))
if pb.PbBuffer[0].Pos >= len(pb.PbBuffer[0].Buffer) { // xyzzy --------------------- pjs - new code - not certain if correct ---------------------------------
done = true
} else {
rn = pb.PbBuffer[0].Buffer[pb.PbBuffer[0].Pos]
pb.PbBuffer[0].Pos++
if rn == '\n' {
pb.PbBuffer[0].LineNo++
pb.PbBuffer[0].ColNo = 1
} else {
pb.PbBuffer[0].ColNo++
}
}
}
return
}
// Have we already seen the specified file. Useful for require(fn)
func (pb *PBReadType) FileSeen(fn string) bool {
com.DbPrintf("pbbuf01", "At: %s\n", com.LF())
a, _ := filepath.Abs(fn)
if t, ok := pb.FilesOpened[a]; ok && t {
return true
}
return false
}
// Set the line/col/file-name for the current buffer - Useful for constructing something like C/Pre processor's #line
func (pb *PBReadType) SetPos(LineNo int, ColNo int, FileName string) {
com.DbPrintf("pbbuf01", "At: %s\n", com.LF())
pb.pushbackIntoBuffer()
if len(pb.PbBuffer) > 0 {
pb.PbBuffer[0].LineNo = LineNo
pb.PbBuffer[0].ColNo = ColNo
pb.PbBuffer[0].FileName = FileName
}
return
}
// Push back a single rune onto input. You can call this more than one time.
func (pb *PBReadType) PbRune(rn rune) {
com.DbPrintf("pbbuf01", "At: %s\n", com.LF())
if pb.PbTop >= MaxAFew { // Buffer is full
pb.pushbackIntoBuffer()
}
pb.PbAFew[pb.PbTop] = rn
pb.PbTop++
}
// Push back a string. Will be converted from an array of byte to an array of runes.
func (pb *PBReadType) PbByteArray(s []byte) {
com.DbPrintf("pbbuf01", "At: %s\n", com.LF())
rns := make([]rune, 0, len(s))
var rn rune
var sz int
for ii := 0; ii < len(s); ii += sz {
rn, sz = utf8.DecodeRune(s[ii:])
rns = append(rns, rn)
}
pb.PbRuneArray(rns)
}
func ParsePlist(pl string) (aa []string) {
t1 := pl_re.FindAllStringSubmatch(pl, -1)
if t1 != nil {
com.DbPrintf("in", "t1=%s\n", com.SVarI(t1))
for _, vv := range t1 {
if len(vv) > 3 && vv[2] != "" {
aa = append(aa, vv[2])
}
}
}
return
}
// Map an input rune to one of the possible output subscripts
func (smap *SMapType) MapRune(rn rune) int {
x := int(rn) - smap.MinV
// fmt.Printf("x=%d, rn=%04x ( %s ), %s\n", x, rn, string(rn), com.LF())
if x > smap.MaxV {
// fmt.Printf("At %s\n", com.LF())
if y, ok := smap.M1[rn]; ok {
// fmt.Printf("At %s\n", com.LF())
return y
}
// fmt.Printf("At %s\n", com.LF())
return smap.NoMapTo
} else if x >= 0 {
com.DbPrintf("smap", "********************************** this one **************************, %s\n", string(rn))
v := smap.M0[x]
if v == smap.NoMapTo {
com.DbPrintf("smap", "********************************** No MAP - 28 case \n")
if unicode.IsDigit(rn) {
if y, ok := smap.M1[re.R_NUMERIC]; ok {
com.DbPrintf("smap", "********************************** case - numeric \n")
return y
}
} else if unicode.IsUpper(rn) {
if y, ok := smap.M1[re.R_UPPER]; ok {
com.DbPrintf("smap", "********************************** case - upper\n")
return y
}
} else if unicode.IsLower(rn) {
if y, ok := smap.M1[re.R_LOWER]; ok {
com.DbPrintf("smap", "********************************** case - lower\n")
return y
}
}
}
// fmt.Printf("At %s\n", com.LF())
return v
} else {
// fmt.Printf("At %s\n", com.LF())
return smap.NoMapTo
}
}
// Tok_Name=1 Tok_Name "T O K"
func ParseNameValue(nv string) (name string, value string) {
name, value = "", ""
t1 := pnv_re.FindAllStringSubmatch(nv, -1)
com.DbPrintf("in", "t1=%s\n", com.SVarI(t1))
if t1 != nil && len(t1[0]) > 0 {
name = t1[0][1]
if len(t1[0]) > 3 {
value = t1[0][3]
}
} else {
name = nv
}
return
}
// Rv(Name) Ignore(Xxx)
func ParseActionItem(act string) (aa string, pp string) {
aa, pp = "", ""
t1 := fx_re.FindAllStringSubmatch(act, -1)
if t1 != nil {
com.DbPrintf("in", "t1=%s\n", com.SVarI(t1))
aa = t1[0][1]
if len(t1[0]) > 1 {
pp = t1[0][2]
}
} else {
aa = act
}
return
}
// -----------------------------------------------------------------------------------------------------------------------------
// -----------------------------------------------------------------------------------------------------------------------------
//type ImRuleType struct {
// PatternType int // Pattern, Str0,1,2, $eof etc. // Pattern Stuff --------------------------------------------------------------------------------
// Pattern string //
func (Im *ImType) LocatePattern(ff *ImRuleType, in []*ImRuleType) (rv int) {
rv = -1
// fmt.Printf("LocatePattern for %s %d\n", ff.Pattern, ff.PatternType)
for kk, tt := range in {
// fmt.Printf(" Compare to %s %d\n", tt.Pattern, tt.PatternType)
if tt.PatternType == ff.PatternType && tt.Pattern == ff.Pattern {
com.DbPrintf("in", " Found\n")
return kk
}
}
// fmt.Printf(" NOT NOT NOT Found\n")
// Xyzzy - should add an error at this point?
// 2. Errors about missing machines in mixin not reported at all - see xyzzyMixin01
return
}
// -----------------------------------------------------------------------------------------------------------------------------
// -----------------------------------------------------------------------------------------------------------------------------
func (Im *ImType) SaveDef(DefType string, Defs []string, line_no int, file_name string) {
if validateDefType(DefType) {
for _, nm := range Defs {
dd, ok := Im.Def.DefsAre[DefType]
if !ok {
dd = ImDefinedValueType{
Seq: 1,
WhoAmI: DefType,
NameValue: make(map[string]int),
NameValueStr: make(map[string]string),
Reverse: make(map[int]string),
SeenAt: make(map[string]ImSeenAtType),
}
}
// seq := dd.Seq
n, v := ParseNameValue(nm)
if n == "" && v == "" { // xyzzy100
return
}
com.DbPrintf("in", "Input: ->%s<- n >%s< v >%s<\n", nm, n, v)
if v != "" {
dd.NameValue[n] = -2 // //
if vv, ok1 := dd.NameValueStr[n]; !ok1 {
dd.NameValueStr[n] = v // //
} else {
if vv != v {
fmt.Printf("Error: Attempt to redfine %s from %s to %s - Probably an error\n", n, vv, v)
}
}
} else {
dd.NameValue[n] = -1 // //
if _, ok1 := dd.NameValueStr[n]; !ok1 {
dd.NameValueStr[n] = "" // //
}
}
// dd.Seq = seq + 1
sa := dd.SeenAt[n]
sa.LineNo = append(sa.LineNo, line_no)
sa.FileName = append(sa.FileName, file_name)
dd.SeenAt[n] = sa
Im.Def.DefsAre[DefType] = dd
}
}
// fmt.Printf("It Is:%+v\n", Im)
}
// Place the contents of a file in buffers at the head so NextRune will pull from this next.
func (pb *PBReadType) PbFile(fn string) (err error) {
com.DbPrintf("pbbuf01", "At: %s\n", com.LF())
err = nil
pb.pushbackIntoBuffer()
pb.FileName = fn
pb.AbsFileName, _ = filepath.Abs(fn)
pb.FilesOpened[pb.AbsFileName] = true
// read file -> PbBuffer
b := &ABuffer{
FileName: fn,
AbsFileName: pb.AbsFileName,
LineNo: 1,
ColNo: 1,
}
// pb.PbBuffer = append(pb.PbBuffer, b)
// data = append([]string{"Prepend Item"}, data...)
pb.PbBuffer = append([]*ABuffer{b}, pb.PbBuffer...) // prepend
bb, err := ioutil.ReadFile(fn)
if err != nil {
return
}
b.EofOnFile = true
b.Pos = 0
var rn rune
var sz int
b.Buffer = make([]rune, 0, len(bb))
for ii := 0; ii < len(bb); ii += sz {
rn, sz = utf8.DecodeRune(bb[ii:])
b.Buffer = append(b.Buffer, rn)
}
return
}
func (dfa *DFA_PoolType) DumpPool(all bool) {
if all {
com.DbPrintf("db_DumpDFAPool", "Cur: %d Top: %d NextFree %d\n", dfa.Cur, dfa.Top, dfa.NextFree)
}
com.DbPrintf("db_DumpDFAPool", "\n---------------------------- DFA Output -----------------------------------------------\n")
com.DbPrintf("db_DumpDFAPool", "\nDFA InitState: %d, Sigma ->%s<-\n\n", dfa.InitState, dfa.Sigma)
pLnNo := com.DbOn("db_DFA_LnNo")
IfLnNo := func(s string) string {
if pLnNo {
t := fmt.Sprintf("[%3s]", s)
return t
}
return ""
}
com.DbPrintf("db_DumpDFAPool", "%3s%s: ", "St", IfLnNo("/Ln"))
com.DbPrintf("db_DumpDFAPool", " %12s %12s \u2714 \tEdges", "StateName", "StateSet")
com.DbPrintf("db_DumpDFAPool", "\n\n")
for ii, vv := range dfa.Pool {
if all || vv.IsUsed {
com.DbPrintf("db_DumpDFAPool", "%3d%s: ", ii, IfLnNo(vv.LineNo))
com.DbPrintf("db_DumpDFAPool", " %12s %12s %s :", vv.StateName, com.SVar(vv.StateSet), com.ChkOrBlank(vv.Visited))
if vv.Rv > 0 {
if vv.Is0Ch {
com.DbPrintf("db_DumpDFAPool", " \u03c4:Tau:%04d ", vv.Rv)
} else {
com.DbPrintf("db_DumpDFAPool", " T:%04d ", vv.Rv)
}
} else {
com.DbPrintf("db_DumpDFAPool", " ")
}
if com.DbOn("db_DumpDFAPool") {
fmt.Printf("\t E:")
for _, ww := range vv.Next2 {
if ww.Is0ChMatch {
fmt.Printf("//Found \u03c4 (%s) //", com.LF()) // Show a Tau(t) for a lambda that matchiens on else conditions.
}
if ww.IsLambda {
fmt.Printf("{ ERROR!! \u03bb %2d -> %2d %s} ", ww.From, ww.To, ww.LineNo)
} else {
// fmt.Printf("{ \"%s\" %2d -> %2d %s} ", ww.On, ww.From, ww.To, IfLnNo(ww.LineNo))
on, _ := utf8.DecodeRune([]byte(ww.On))
son := fmt.Sprintf("%q", ww.On)
switch on {
case re.R_DOT: // = '\uF8FA' // Any char in Sigma
son = "DOT/uF8FA"
case re.R_BOL: // = '\uF8F3' // Beginning of line
son = "BOL/uF8F3"
case re.R_EOL: // = '\uF8F4' // End of line
son = "EOL/uF8F4"
case re.R_NUMERIC: // = '\uF8F5'
son = "NUMERIC/uF8F5"
case re.R_LOWER: // = '\uF8F6'
son = "LOWER/uF8F6"
case re.R_UPPER: // = '\uF8F7'
son = "UPPER/uF8F7"
case re.R_ALPHA: // = '\uF8F8'
son = "ALPHA/uF8F8"
case re.R_ALPHNUM: // = '\uF8F9'
son = "ALPHANUM/uF8F9"
case re.R_EOF: // = '\uF8FB'
son = "EOF/uF8FB"
case re.R_not_CH: // = '\uF8FC' // On input lookup if the char is NOT in Signa then it is returned as this.
son = "else_CH/uF8Fc"
case re.R_N_CCL: // = '\uF8FD' // If char is not matched in this state then take this path
son = "N_CCL/uF8Fd"
case re.R_LAMBDA_MATCH: // = '\uF8FE'
son = "LambdaM/uF8FE"
}
fmt.Printf("{ %s %2d -> %2d %s} ", son, ww.From, ww.To, IfLnNo(ww.LineNo))
}
}
fmt.Printf("\n")
if vv.Info.Action != 0 || vv.Info.MatchLength != 0 {
// fmt.Printf("\t\t\tInfo: %s\n", com.SVar(vv.Info)) // xyzzy - output Info
// xyzzy - NextState info
fmt.Printf("\t\t\tDFA.Info: %s", nfa.DumpInfo(vv.Info))
// if ((vv.Info.Action&com.A_Pop) != 0 || (vv.Info.Action&com.A_Push) != 0 || (vv.Info.Action&com.A_Reset) != 0) && !vv.Info.HardMatch { // xyzzy8
fmt.Printf(" IsHard=%v (((false imples else case Rv!)))\n", vv.Info.HardMatch)
}
fmt.Printf("\n")
}
}
}
}
func Test_PbBufer01(t *testing.T) {
SymbolTable := make([]*MacroDefTestType, 0, 100)
Define := func(name rune, body string) {
for ii := 0; ii < len(SymbolTable); ii++ {
// fmt.Printf("Search at %d, for %s\n", ii, string(name))
if SymbolTable[ii] != nil && SymbolTable[ii].Rn == name {
SymbolTable[ii].Body = body
return
}
}
// fmt.Printf("Append\n")
SymbolTable = append(SymbolTable, &MacroDefTestType{Rn: name, Body: body})
}
ResetST := func() {
// SymbolTable = make([]*MacroDefTestType, 0, 100)
SymbolTable = SymbolTable[:1]
}
HaveMacro := func(name rune) (body string, found bool) {
body = ""
found = false
for ii := 0; ii < len(SymbolTable); ii++ {
if SymbolTable[ii] != nil && SymbolTable[ii].Rn == name {
body, found = SymbolTable[ii].Body, true
return
}
}
return
}
for ii, vv := range Pb01Test {
if !vv.SkipTest {
// Implement a quick - fetch execute macine to test - the PbBuffer - commands/opcodes are the Cmd* constants above.
ss := ""
pb := NewPbRead()
ResetST()
for pc, ww := range vv.Actions {
switch ww.OpCode {
case CmdOpenFile: // Open a file , at the tail end of list of input
pb.OpenFile(ww.Fn)
com.DbPrintf("testCode", "Open file %s At: %s\n", ww.Fn, com.LF())
if com.DbOn("testDump") {
pb.Dump01(os.Stdout)
}
case CmdPbString: // Push back a string
pb.PbString(ww.Data)
if com.DbOn("testDump") {
pb.Dump01(os.Stdout)
}
case CmdPbRune: // Push back a rune
pb.PbRune(ww.Rn)
if com.DbOn("testDump") {
pb.Dump01(os.Stdout)
}
case CmdPbRuneArray: // Push back a rune array
pb.PbRuneArray(ww.RnS)
if com.DbOn("testDump") {
pb.Dump01(os.Stdout)
}
case CmdNextNChar:
for ll := 0; ll < ww.X; ll++ {
rn, done := pb.NextRune()
if !done {
ss = ss + string(rn)
}
com.DbPrintf("testCode", "Case 5: At: ->%s<- ll=%d ss >>>%s<<< %s\n", string(rn), ll, ss, com.LF())
}
if com.DbOn("testDump") {
pb.Dump01(os.Stdout)
}
case CmdPeek:
rn, done := pb.PeekRune()
if done || rn != ww.Rn {
t.Errorf("%04s: Peek at [pc=%d] in test [%s] did not work, got %s expected %s, done=%v\n", pc, ii, string(rn), string(ww.Rn), done)
}
case CmdOutputToEof:
com.DbPrintf("testCode", "All Done: ss >>>%s<<< before At: %s\n", ss, com.LF())
if com.DbOn("testDump") {
pb.Dump01(os.Stdout)
}
for rn, done := pb.NextRune(); !done; rn, done = pb.NextRune() {
ss = ss + string(rn)
}
com.DbPrintf("testCode", "All Done: ss >>>%s<<< after At: %s\n", ss, com.LF())
case CmdPbByteArray: // Push back a byte array
pb.PbByteArray([]byte(ww.Data))
if com.DbOn("testDump") {
pb.Dump01(os.Stdout)
}
case CmdPbFile: // Open file and push contents back onto input at head of list. (Macro file, Include, Require)
pb.PbFile(ww.Fn)
com.DbPrintf("testCode", "Pb file %s At: %s\n", ww.Fn, com.LF())
if com.DbOn("testDump") {
pb.Dump01(os.Stdout)
}
case CmdFileSeen:
fs := pb.FileSeen(ww.Fn)
if fs != ww.FileSeenFlag {
t.Errorf("%04s: Peek at [pc=%d] in test [%s] did not work, got %v expected %s for file seen flagv\n", pc, ii, fs, ww.FileSeenFlag)
}
case CmdGetPos: // Check get file name
ln, cn, fn := pb.GetPos()
com.DbPrintf("testCode", "fn=%s ln=%d cn=%d\n", fn, ln, cn)
if ln != ww.LineNo {
t.Errorf("%04s: %d: did not match line no Expected ->%d<-, Got ->%d<-\n", vv.Test, pc, ww.LineNo, ln)
}
if cn != ww.ColNo {
t.Errorf("%04s: %d: did not match col no Expected ->%d<-, Got ->%d<-\n", vv.Test, pc, ww.ColNo, cn)
}
if fn != ww.Fn {
t.Errorf("%04s: %d: did not match file name Expected ->%s<-, Got ->%s<-\n", vv.Test, pc, ww.Fn, fn)
}
case CmdSetPos: // Check get file name
pb.SetPos(ww.LineNo, ww.ColNo, ww.Fn)
case CmdResetST: // Reset symbol table
ResetST()
case CmdMacroProc: // Apply 1 char macros to input and process
for rn, done := pb.NextRune(); !done; rn, done = pb.NextRune() {
if m_body, m_found := HaveMacro(rn); m_found {
pb.PbString(m_body)
} else {
ss = ss + string(rn)
}
}
case CmdDefineMacro: // Define
Define(ww.Rn, ww.Data)
case CmdDumpBuffer: // Dump the buffer - debuging
if ww.Fn == "" {
pb.Dump01(os.Stdout)
} else {
fp, err := com.Fopen(ww.Fn, "w")
if err == nil {
pb.Dump01(fp)
fp.Close()
} else {
pb.Dump01(os.Stdout)
t.Errorf("%04s: Unable to open file for output ->%s<-, error: %s\n", vv.Test, ww.Fn, err)
}
}
case CmdResetOutput: // Reset output
ss = ""
case CmdPushBackXCopies: // Special test to pub back more than buffer of 'x'
x := strings.Repeat(ww.Data, ww.X)
pb.PbString(x)
}
}
if ss != vv.Results {
t.Errorf("%04s: did not match Expected ->%s<-, Got ->%s<-\n", vv.Test, vv.Results, ss)
} else {
com.DbPrintf("testCode", "%04s: Passed ------------------------------------------------------------------------------------------------\n\n", vv.Test)
}
}
}
}
func (lr *LexReType) parseCCL(depth int, ww LR_TokType) (tree ReTreeNodeType) {
pos := 0
com.DbPrintf("db2", "parseCCL Top: depth=%d, %d=%s\n", depth, ww, NameOfLR_TokType(ww))
s := ""
dx := 0
marked := false
c, w := lr.Next()
for w != LR_EOF {
com.DbPrintf("re2", "Top of parseCCL dx=%d ->%s<-\n", dx, c)
switch w {
case LR_MINUS: // - -- Text if not in CCL and not 1st char in CCL
fallthrough
case LR_Text: // -- Add a node to list, move right
fallthrough
case LR_CARROT: // ^ -- BOL
fallthrough
case LR_DOT: // . -- Match any char
fallthrough
case LR_STAR: // * -- Error if 1st char, else take prev item from list, star and replace it.
fallthrough
case LR_PLUS: // + -- Error if 1st char
fallthrough
case LR_QUEST: // ? -- Error if 1st char
fallthrough
case LR_OP_PAR: // ( -- Start of Sub_Re
fallthrough
case LR_CL_PAR: // )
fallthrough
case LR_OR: // |
fallthrough
case LR_OP_BR: // {
fallthrough
case LR_CL_BR: // }
fallthrough
case LR_COMMA: // ,
fallthrough
case LR_DOLLAR: // $
s += c
case LR_N_CCL: // [^...] -- N_CCL Node
marked = true
com.DbPrintf("re2", " incr to %d, %s\n", dx, com.LF())
s += c // Add To CCL
case LR_CCL: // [...] -- CCL Node (Above)
marked = true
com.DbPrintf("re2", " incr to %d, %s\n", dx, com.LF())
s += c // Add To CCL
case LR_E_CCL:
dx--
com.DbPrintf("re2", " decr to %d, %s\n", dx, com.LF())
if dx < 0 {
tree.Item = expandCCL(s) // Do Something, Return
tree.LR_Tok = ww
return
} else {
s += c //
}
default:
lr.Error = append(lr.Error, errors.New(fmt.Sprintf("Unreacable Code, invalid token in parseCCL = %d, %s", w, com.LF())))
tree.Item = expandCCL(s) // Do Something, Return
tree.LR_Tok = ww
return
}
pos++
c, w = lr.Next()
if c == ":" && marked {
marked = false
dx++
}
}
if w == LR_EOF {
lr.Err("EOF found in Character Class [...] or [^...].")
}
tree.Item = expandCCL(s)
tree.LR_Tok = ww
return
}
func (s *ReTesteSuite) TestLexie(c *C) {
// Test of Parseing REs into RE-TParseTrees
// return
fmt.Fprintf(os.Stderr, "Test Parsing of REs, %s\n", com.LF())
com.DbOnFlags["db_DumpPool"] = true
com.DbOnFlags["parseExpression"] = true
com.DbOnFlags["CalcLength"] = true
com.DbOnFlags["DumpParseNodes"] = true
com.DbOnFlags["DumpParseNodesX"] = true
fmt.Printf("**** In Test RE\n")
n_err := 0
n_skip := 0
for ii, vv := range Lexie00Data {
if !vv.SkipTest {
fmt.Printf("\n\n--- %d Test: %s -----------------------------------------------------------------------------\n\n", ii, vv.Test)
lr := NewLexReType()
lr.SetBuf(vv.Re)
tn := 0
cc, ww := lr.Next()
for ww != LR_EOF {
fmt.Printf(" %s % x = %d %s\n", string(cc), string(cc), ww, LR_TokTypeLookup[ww])
if len(vv.SM) > 0 {
// Check correct token
if vv.SM[tn].Tok != ww {
fmt.Printf(" Failed to return the correct token, expecting %d/%s, got %d/%s\n", vv.SM[tn].Tok, LR_TokTypeLookup[vv.SM[tn].Tok], ww,
LR_TokTypeLookup[ww])
c.Check(int(vv.SM[tn].Tok), Equals, int(ww))
n_err++
}
// Check correct string/rune returned
if !CmpByteArr(vv.SM[tn].Dat, []byte(cc)) {
fmt.Printf(" The returned runes did not match\n")
c.Check(string(vv.SM[tn].Dat), Equals, cc)
n_err++
}
}
tn++
cc, ww = lr.Next()
}
fmt.Printf("\n--- %d End : %s -----------------------------------------------------------------------------\n\n", ii, vv.Test)
}
}
for ii, vv := range Lexie01Data {
if !vv.SkipTest {
fmt.Printf("\n\n--- %d Test: %s -----------------------------------------------------------------------------\n\n", ii, vv.Test)
lr := NewLexReType()
lr.ParseRe(vv.Re)
lr.Sigma = lr.GenerateSigma()
fmt.Printf("Sigma: ->%s<-\n", lr.Sigma)
if vv.Sigma != "" {
if vv.Sigma != lr.Sigma {
fmt.Printf(" The calculated and reference Sigma did not match\n")
c.Check(vv.Sigma, Equals, lr.Sigma)
n_err++
}
}
fmt.Printf("\n--- %d End : %s -----------------------------------------------------------------------------\n\n", ii, vv.Test)
}
}
// -------------------------------------------------------------------------------------------------------------------------------------------------
lr := NewLexReType()
com.DbOnFlags["DumpParseNodes"] = true
com.DbOnFlags["DumpParseNodesX"] = true
com.DbOnFlags["db_DumpPool"] = true
com.DbOnFlags["parseExpression"] = true
for i, v := range Test6Data {
com.DbPrintf("debug", "\nTest[%03d]: `%s` %s\n\n", i, v.Re, strings.Repeat("-", 120-len(v.Re)))
// fmt.Printf("%s *** com.Red *** %s\n", ansi.ColorCode("red"), ansi.ColorCode("reset"))
lr.ParseRe(v.Re)
lr.DumpParseNodes()
if len(v.TopTok) > 0 {
if len(v.TopTok) != len(lr.Tree.Children) {
com.DbPrintf("debug", "%sError%s: wrong number of tokens prsed, Expected: %d Got %d\n", com.Red, com.Reset, len(v.TopTok), len(lr.Tree.Children))
n_err++
} else {
for i := 0; i < len(v.TopTok); i++ {
if v.TopTok[i] != lr.Tree.Children[i].LR_Tok {
com.DbPrintf("debug", "%sError%s: invalid token returnd at postion %d\n", com.Red, com.Reset, i)
c.Check(v.TopTok[i], Equals, lr.Tree.Children[i].LR_Tok)
n_err++
}
}
}
}
if len(v.TopVal) > 0 {
if len(v.TopVal) != len(lr.Tree.Children) {
com.DbPrintf("debug", "%sError%s: wrong number of tokens prsed, Expected: %d Got %d - Based on number of values, TopVal\n", com.Red, com.Reset, len(v.TopVal), len(lr.Tree.Children))
n_err++
} else {
for i := 0; i < len(v.TopVal); i++ {
if v.TopVal[i] != lr.Tree.Children[i].Item {
com.DbPrintf("debug", "%sError%s: invalid value at postion %d, %s\n", com.Red, com.Reset, i, com.LF())
n_err++
}
}
}
}
if len(lr.Error) > v.NExpectedErr {
com.DbPrintf("debug", "%sError%s: Errors reported in R.E. parsing %d\n", com.Red, com.Reset, len(lr.Error))
n_err++
} else if len(lr.Error) > 0 {
com.DbPrintf("debug", "%sNote%s: Errors reported in R.E. parsing %d\n", com.Green, com.Reset, len(lr.Error))
}
lr.Error = lr.Error[:0]
com.DbPrintf("debug", "\nDone[%03d]: `%s` %s\n\n", i, v.Re, strings.Repeat("-", 120-len(v.Re)))
}
if n_err > 0 {
fmt.Fprintf(os.Stderr, "%sFailed, # of errors = %d%s\n", com.Red, n_err, com.Reset)
com.DbPrintf("debug", "\n\n%sFailed, # of errors = %d%s\n", com.Red, n_err, com.Reset)
} else {
fmt.Fprintf(os.Stderr, "%sPASS%s\n", com.Green, com.Reset)
com.DbPrintf("debug", "\n\n%sPASS%s\n", com.Green, com.Reset)
}
if n_skip > 0 {
fmt.Fprintf(os.Stderr, "%sSkipped, # of files without automated checks = %d%s\n", com.Yellow, n_skip, com.Reset)
com.DbPrintf("debug", "\n\n%sSkipped, # of files without automated checks = %d%s\n", com.Yellow, n_skip, com.Reset)
}
if n_err > 0 {
c.Check(n_err, Equals, 0)
fmt.Fprintf(os.Stderr, "%sFailed, # of errors = %d%s\n", com.Red, n_err, com.Reset)
com.DbPrintf("debug", "\n\n%sFailed, # of errors = %d%s\n", com.Red, n_err, com.Reset)
} else {
fmt.Fprintf(os.Stderr, "%sPASS%s\n", com.Green, com.Reset)
com.DbPrintf("debug", "\n\n%sPASS%s\n", com.Green, com.Reset)
}
}
func expandCCL(s string) (ccl string) {
ccl = ""
com.DbPrintf("db2", "at %s\n", com.LF())
pos := 0
if len(s) > 0 && s[0:1] == "-" { // Check for leading '-' include in CCL
ccl += "-"
pos = 1
}
for ii := pos; ii < len(s); ii++ {
com.DbPrintf("re2", "ii=%d remaining ->%s<-, %s\n", ii, s[ii:], com.LF())
if strings.HasPrefix(s[ii:], "[:alphnum:]") {
com.DbPrintf("re2", " Matched: %s\n", com.LF())
// ccl += X_ALPHA
ccl += X_LOWER
ccl += X_UPPER
ccl += X_NUMERIC
ii += len("[:alphnum:]") - 1
} else if strings.HasPrefix(s[ii:], "[:alpha:]") {
com.DbPrintf("re2", " Matched: %s\n", com.LF())
// ccl += X_ALPHA
ccl += X_LOWER
ccl += X_UPPER
ii += len("[:alpha:]") - 1
} else if strings.HasPrefix(s[ii:], "[:lower:]") {
com.DbPrintf("re2", " Matched: %s\n", com.LF())
ccl += X_LOWER
ii += len("[:lower:]") - 1
} else if strings.HasPrefix(s[ii:], "[:upper:]") {
com.DbPrintf("re2", " Matched: %s\n", com.LF())
ccl += X_UPPER
ii += len("[:upper:]") - 1
} else if strings.HasPrefix(s[ii:], "[:numeric:]") {
com.DbPrintf("re2", " Matched: %s\n", com.LF())
ccl += X_NUMERIC
ii += len("[:numeric:]") - 1
} else if ii+9 <= len(s) && s[ii:ii+9] == "a-zA-Z0-9" {
com.DbPrintf("re2", " Matched: %s\n", com.LF())
// ccl += X_ALPHA
ccl += X_LOWER
ccl += X_UPPER
ccl += X_NUMERIC
ii += 8
} else if ii+6 <= len(s) && s[ii:ii+6] == "a-zA-Z" {
com.DbPrintf("re2", " Matched: %s\n", com.LF())
// ccl += X_ALPHA
ccl += X_LOWER
ccl += X_UPPER
ii += 5
} else if ii+3 <= len(s) && s[ii:ii+3] == "0-9" {
com.DbPrintf("re2", " Matched: %s\n", com.LF())
// fmt.Printf("matched 0-9 pattern\n")
ccl += X_NUMERIC
ii += 2
} else if ii+3 <= len(s) && s[ii:ii+3] == "a-z" {
com.DbPrintf("re2", " Matched: %s\n", com.LF())
ccl += X_LOWER
ii += 2
} else if ii+3 <= len(s) && s[ii:ii+3] == "A-Z" {
com.DbPrintf("re2", " Matched: %s\n", com.LF())
ccl += X_UPPER
ii += 2
} else if ii+2 < len(s) && s[ii+1:ii+2] == "-" {
com.DbPrintf("re2", " Matched: %s\n", com.LF())
// xyzzyRune TODO - this code is horribley non-utf8 compatable at this moment in time.
e := s[ii+2]
// fmt.Printf("matched a-b pattern, b=%s e=%s\n", string(s[ii]), string(e))
if s[ii] >= e {
fmt.Printf("Error: Poorly formatted character-class, beginning is larger than or equal to end of CCL\n") // Xyzzy - need line number etc
}
for b := s[ii]; b <= e; b++ {
ccl += string(b)
}
ii += 2
} else {
ccl += s[ii : ii+1]
}
// fmt.Printf("bottom ccl now: ->%s<-\n", ccl)
}
return
}
// mm, nn := lr.parseIterator ( depth+1 )
func (lr *LexReType) parseIterator(depth int) (tree ReTreeNodeType) {
pos := 0
com.DbPrintf("db2", "parseIterator Top: depth=%d\n", depth)
s := ""
c, w := lr.Next()
for w != LR_EOF {
switch w {
case LR_MINUS: // - -- Text if not in CCL and not 1st char in CCL
fallthrough
case LR_CARROT: // ^ -- BOL
fallthrough
case LR_DOT: // . -- Match any char
fallthrough
case LR_STAR: // * -- Error if 1st char, else take prev item from list, star and replace it.
fallthrough
case LR_PLUS: // + -- Error if 1st char
fallthrough
case LR_QUEST: // ? -- Error if 1st char
fallthrough
case LR_OP_PAR: // ( -- Start of Sub_Re
fallthrough
case LR_CL_PAR: // )
fallthrough
case LR_CCL: // [...] -- CCL Node (Above)
fallthrough
case LR_OR: // |
fallthrough
case LR_OP_BR: // {
fallthrough
case LR_DOLLAR: // $
fallthrough
case LR_E_CCL:
fallthrough
case LR_N_CCL: // [^...] -- N_CCL Node
lr.Error = append(lr.Error, errors.New(fmt.Sprintf("in parseIterator, Invalid {m,n} - invalid chars found, %s", com.LF())))
tree.Mm, tree.Nn = 1, 1
tree.LR_Tok = LR_OP_BR
return
case LR_Text: // -- Add a node to list, move right
if c[0] >= '0' && c[0] <= '9' || c[0] == ',' {
s += c // Add To Iterator
} else {
lr.Error = append(lr.Error, errors.New(fmt.Sprintf("Unreacable Code, invalid token in parseIterator, %s", com.LF())))
tree.Mm, tree.Nn = lr.parseIteratorString(s) // Do Something, Return
tree.Item = "{"
tree.LR_Tok = LR_OP_BR
return
}
case LR_COMMA: // ,
s += c // Add To Iterator
case LR_CL_BR: // }
tree.Mm, tree.Nn = lr.parseIteratorString(s) // Do Something, Return
tree.Item = "{"
tree.LR_Tok = LR_OP_BR
return
default:
lr.Error = append(lr.Error, errors.New(fmt.Sprintf("Unreacable Code, invalid token in parseIterator, %s", com.LF())))
tree.Mm, tree.Nn = lr.parseIteratorString(s) // Do Something, Return
tree.Item = "{"
tree.LR_Tok = LR_OP_BR
return
}
pos++
c, w = lr.Next()
}
if w == LR_EOF {
lr.Err("EOF found in Iterator {m,n}.")
}
tree.Mm, tree.Nn = lr.parseIteratorString(s) // Do Something, Return
tree.Item = "{"
tree.LR_Tok = LR_OP_BR
return
}
func (lr *LexReType) parseExpression(depth int, d_depth int, xTree *ReTreeNodeType) []ReTreeNodeType {
//var first *ReTreeNodeType
//var last *ReTreeNodeType
pre := strings.Repeat(" ", depth)
if depth == 0 {
xTree = lr.Tree
com.DbPrintf("parseExpression", "%sat %s !!!top!!!, depth=%d \n", pre, com.LF(), depth)
}
isFirst := true
inOr := false
com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())
c, w := lr.Next()
for w != LR_EOF {
com.DbPrintf("parseExpression", "%sat %s !!!top!!!, depth=%d c=->%s<- w=%d %s -- Loop Top -- xTree=%s\n\n",
pre, com.LF(), depth, c, w, NameOfLR_TokType(w), com.SVarI(xTree))
switch w {
case LR_CL_BR: // }
fallthrough
case LR_COMMA: // ,
fallthrough
case LR_E_CCL:
fallthrough
case LR_MINUS: // - -- Text if not in CCL and not 1st char in CCL
fallthrough
case LR_Text: // -- Add a node to list, move right
//if true {
xTree.Children = append(xTree.Children, ReTreeNodeType{Item: c, LR_Tok: LR_Text})
//} else {
// // Bad Idea - mucks up '*' and other processing - To Simplify Tree needs to be done post-generation with Simp-Rules
// ll := len(lr.Tree.Children) - 1
// if ll >= 0 && lr.Tree.Children[ll].LR_Tok == LR_Text {
// lr.Tree.Children[ll].Item += c
// } else {
// xTree.Children = append(xTree.Children, ReTreeNodeType{Item: c, LR_Tok: LR_Text})
// }
//}
case LR_CARROT: // ^ -- BOL -- If at beginning, or after ( or | then BOL - else just text??
fallthrough
case LR_DOLLAR: // $ -- BOL -- If at end, or just before ) or | the EOL - else just text??
fallthrough
case LR_DOT: // . -- Match any char
xTree.Children = append(xTree.Children, ReTreeNodeType{Item: c, LR_Tok: w})
case LR_OP_BR: // {
if isFirst {
lr.Warn(fmt.Sprintf("Invalid '%s' at beginning of R.E. assumed to be a text character missing esacape.", c))
xTree.Children = append(xTree.Children, ReTreeNodeType{Item: c, LR_Tok: LR_Text})
} else {
ll := len(xTree.Children) - 1
tmp := xTree.Children[ll]
newTree := lr.parseIterator(depth + 1)
if newTree.Mm == 0 && newTree.Nn == InfiniteIteration {
ll := len(xTree.Children) - 1
tmp := xTree.Children[ll]
com.DbPrintf("parseExpression", "%sAT %s, w=%d %s, ll=%d, xTree=%s tmp=%s\n", pre, com.LF(), w, NameOfLR_TokType(w), ll, com.SVarI(xTree), com.SVarI(tmp))
xTree.Children[ll] = ReTreeNodeType{Item: "*", LR_Tok: LR_STAR, Children: []ReTreeNodeType{tmp}}
} else {
if newTree.Mm > newTree.Nn {
lr.Error = append(lr.Error, errors.New(fmt.Sprintf("Invalid Range, Start is bigger than end, {%d,%d}, %s", newTree.Mm, newTree.Nn, com.LF())))
}
com.DbPrintf("parseExpression", "%sAT %s, w=%d %s, ll=%d, xTree=%s tmp=%s\n", pre, com.LF(), w, NameOfLR_TokType(w), ll, com.SVarI(xTree), com.SVarI(tmp))
// xTree.Children[ll] = ReTreeNodeType{Item: c, LR_Tok: LR_OP_BR, Children: []ReTreeNodeType{tmp}, Mm: newTree.Mm, Nn: newTree.Nn}
newTree.Children = []ReTreeNodeType{tmp}
xTree.Children[ll] = newTree
// CCL: xTree.Children = append(xTree.Children, lr.parseCCL(depth+1, w)) // xyzzy needs work ---------------------------------------------------
}
com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())
}
case LR_STAR: // * -- Error if 1st char, else take prev item from list, star and replace it.
fallthrough
case LR_PLUS: // + -- Error if 1st char
fallthrough
case LR_QUEST: // ? -- Error if 1st char
if isFirst {
lr.Warn(fmt.Sprintf("Invalid '%s' at beginning of R.E. assumed to be a text character missing esacape.", c))
xTree.Children = append(xTree.Children, ReTreeNodeType{Item: c, LR_Tok: LR_Text})
} else {
ll := len(xTree.Children) - 1
tmp := xTree.Children[ll]
com.DbPrintf("parseExpression", "%sAT %s, w=%d %s, ll=%d, xTree=%s tmp=%s\n", pre, com.LF(), w, NameOfLR_TokType(w), ll, com.SVarI(xTree), com.SVarI(tmp))
xTree.Children[ll] = ReTreeNodeType{Item: c, LR_Tok: w, Children: []ReTreeNodeType{tmp}}
com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())
}
case LR_OR: // | n-ary or operator
com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())
inOr = true
// Left Machine is collected to be sub-machine == Beginnig-to-current
// left := xTree.Children // change to be left section back to but not including "|" node - or all if no | node.
kk := -1
for jj := len(xTree.Children) - 1; jj >= 0; jj-- {
if xTree.Children[jj].LR_Tok == LR_OR {
kk = jj
break
}
}
if kk == -1 { // No OR tok found
left := xTree.Children // change to be left section back to but not including "|" node - or all if no | node.
ll := len(left)
leftNode := ReTreeNodeType{Item: "", LR_Tok: LR_null, Children: make([]ReTreeNodeType, ll, ll)}
for jj := range left {
leftNode.Children[jj] = left[jj]
}
newTop := ReTreeNodeType{Item: "|", LR_Tok: LR_OR, Children: make([]ReTreeNodeType, 0, 10)}
newTop.Children = append(newTop.Children, leftNode) // only if no "or" node, else ref to "or" node
xTree.Children = xTree.Children[:0]
xTree.Children = append(xTree.Children, newTop)
com.DbPrintf("parseExpression", "%sAT %s, w=%d %s, left=%s\n", pre, com.LF(), w, NameOfLR_TokType(w), com.SVarI(left))
} else {
if kk >= 0 {
if kk < len(xTree.Children) {
tmp := xTree.Children[kk+1:]
xTree.Children = xTree.Children[0 : kk+1]
newNode := ReTreeNodeType{Item: "", LR_Tok: LR_null, Children: make([]ReTreeNodeType, len(tmp), len(tmp))}
for i := 0; i < len(tmp); i++ {
newNode.Children[i] = tmp[i]
}
xTree.Children[kk].Children = append(xTree.Children[kk].Children, newNode)
}
}
}
// Or node is created like (LR_STAR)
// Recursive call to parse rest of items at this level
//newNode := ReTreeNodeType{Item: "", LR_Tok: LR_null, Children: make([]ReTreeNodeType, 1, 10)} // No recursive call
//lr.parseExpression(depth+1, depth, &newNode.Children[0])
//newTop.Children = append(newTop.Children, newNode.Children[0])
// Take results of recursion and put in as RIGHT machine under LR_OR (optimize for N-Tree OR at this point)
//xTree.Children = append(xTree.Children, newTop)
//if depth > d_depth {
//com.DbPrintf("parseExpression", "%sat %s, depth=%d d_detph=%d\n", pre, com.LF(), depth, d_depth)
// return xTree.Children
//}
case LR_OP_PAR: // ( -- Start of Sub_Re
com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())
newNode := ReTreeNodeType{Item: c, LR_Tok: LR_OP_PAR, Children: make([]ReTreeNodeType, 1, 10)}
lr.parseExpression(depth+1, depth+1, &newNode.Children[0])
newNode.Children[0].Item = c
newNode.Children[0].LR_Tok = LR_OP_PAR
xTree.Children = append(xTree.Children, newNode.Children[0])
com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())
case LR_CL_PAR: // )
// If in "or" node set - then collect last section to "or" ------------------------ <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())
if depth == 0 {
com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())
lr.Warn(fmt.Sprintf("Invalid '%s' at not properly nested. Assuming that this was to match a character.", c))
xTree.Children = append(xTree.Children, ReTreeNodeType{Item: c, LR_Tok: LR_Text})
com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())
} else {
com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())
if inOr {
com.DbPrintf("parseExpression", "%sAT Top of new code %s, BOTTOM xTree=%s\n", pre, com.LF(), com.SVarI(xTree))
kk := -1
for jj := len(xTree.Children) - 1; jj >= 0; jj-- {
if xTree.Children[jj].LR_Tok == LR_OR {
kk = jj
break
}
}
if kk >= 0 {
if kk < len(xTree.Children) {
tmp := xTree.Children[kk+1:]
xTree.Children = xTree.Children[0 : kk+1]
newNode := ReTreeNodeType{Item: "", LR_Tok: LR_null, Children: make([]ReTreeNodeType, len(tmp), len(tmp))}
for i := 0; i < len(tmp); i++ {
newNode.Children[i] = tmp[i]
}
xTree.Children[kk].Children = append(xTree.Children[kk].Children, newNode)
}
}
com.DbPrintf("parseExpression", "%sAT Bo5 of new code %s, BOTTOM xTree=%s\n", pre, com.LF(), com.SVarI(xTree))
}
return xTree.Children
}
com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())
inOr = false
case LR_CCL: // [...] -- CCL Node (Above)
fallthrough
case LR_N_CCL: // [^...] -- N_CCL Node
xTree.Children = append(xTree.Children, lr.parseCCL(depth+1, w)) // xyzzy needs work ---------------------------------------------------
default:
lr.Error = append(lr.Error, errors.New(fmt.Sprintf("Invalid LR Token Type, '%d', '%s', %s", w, NameOfLR_TokType(w), com.LF())))
return xTree.Children
}
isFirst = false
com.DbPrintf("parseExpression", "%sAT %s, BOTTOM xTree=%s\n", pre, com.LF(), com.SVarI(xTree))
c, w = lr.Next()
}
// If in "or" node set - then collect last section to "or" ------------------------ <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
if inOr {
com.DbPrintf("parseExpression", "%sAT Top of new code %s, BOTTOM xTree=%s\n", pre, com.LF(), com.SVarI(xTree))
kk := -1
for jj := len(xTree.Children) - 1; jj >= 0; jj-- {
if xTree.Children[jj].LR_Tok == LR_OR {
kk = jj
break
}
}
if kk >= 0 {
if kk < len(xTree.Children) {
tmp := xTree.Children[kk+1:]
xTree.Children = xTree.Children[0 : kk+1]
newNode := ReTreeNodeType{Item: "", LR_Tok: LR_null, Children: make([]ReTreeNodeType, len(tmp), len(tmp))}
for i := 0; i < len(tmp); i++ {
newNode.Children[i] = tmp[i]
}
xTree.Children[kk].Children = append(xTree.Children[kk].Children, newNode)
}
}
com.DbPrintf("parseExpression", "%sAT Bo5 of new code %s, BOTTOM xTree=%s\n", pre, com.LF(), com.SVarI(xTree))
}
return xTree.Children
}
func (lr *LexReType) CalcLength() (int, bool) {
x, h := lr.CalcLengthChild(lr.Tree, 1)
com.DbPrintf("CalcLength", "CalcLength Final Value for Tree = %d, hard=%v\n", x, h)
return x, h
}
func (dfa *DFA_PoolType) ConvNDA_to_DFA(nn *nfa.NFA_PoolType) {
StartState := nn.InitState
dfa.NoneVisited()
nn.Sigma = nn.GenerateSigma()
dfa.Sigma = nn.Sigma
com.DbPrintf("dfa2", "Sigma at top ->%s<-\n", dfa.Sigma)
// Build initial state
dfa_set := nn.LambdaClosure([]int{StartState}) // Find all the lambda closures from specified state
dfa_set = append(dfa_set, StartState) // Add in initial state
dfa_set = com.USortIntSlice(dfa_set) // Make set unique
com.DbPrintf("db_DFAGen", "\nStart: %s, \u03a3 =->%s<-, %s\n", com.SVar(dfa_set), dfa.Sigma, com.LF())
A := dfa.GetDFAName(dfa_set)
if r, is, info, Is0Ch := nn.IsTerminalState(dfa_set); is {
dfa.Pool[A].Rv = r
dfa.Pool[A].Is0Ch = Is0Ch
dfa.Pool[A].Info = info
} else {
dfa.Pool[A].Info = info
}
if com.DbOn("db_DFAGen") {
dfa.DumpPool(false)
}
// Look at all the locaitons we can get to from this "state"
for _, S := range dfa.Sigma {
StateSet := nn.LambdaClosureSet(dfa_set, string(S))
com.DbPrintf("db_DFAGen", "FOR INITIAL state ->%s<- StateSet=%s, %s\n", string(S), com.SVar(StateSet), com.LF())
if len(StateSet) > 0 {
com.DbPrintf("db_DFAGen", "Have a non-empty result, %s\n", com.LF())
com.DbPrintf("db_DFAGen", "<><><> this is the point where we should check to see if 'S' is DOT or NCCL, %s\n", com.LF())
StateSetT := nn.LambdaClosure(StateSet) // need to lambda complete the state set
StateSet = append(StateSet, StateSetT...)
StateSet = com.USortIntSlice(StateSet) // Make set unique
com.DbPrintf("db_DFAGen", " Output Is %s, %s\n", com.SVar(StateSet), com.LF())
B := 0
if t := dfa.HaveStateAlready(StateSet); t != -1 { // Have Already
B = t
com.DbPrintf("db_DFAGen", " Already have this state at location %d, %s\n", t, com.LF())
} else {
B = dfa.GetDFAName(StateSet)
com.DbPrintf("db_DFAGen", " *** New state %d, %s\n", B, com.LF())
}
dfa.AddEdge(A, B, string(S))
com.DbPrintf("db_DFAGen", " *** Before (top) %s\n", com.LF())
if r, is, info, Is0Ch := nn.IsTerminalState(StateSet); is {
dfa.Pool[B].Rv = r
dfa.Pool[B].Is0Ch = Is0Ch
dfa.Pool[B].Info = info
com.DbPrintf("db_DFAGen", " *** New state %d, %s\n", B, com.LF())
} else if _, is, info, Is0Ch := nn.IsNonTerminalPushPopState(StateSet); is {
dfa.Pool[B].Is0Ch = Is0Ch
dfa.Pool[B].Info = info
com.DbPrintf("db_DFAGen", " *** New info for state %d, %s\n", B, com.LF())
} else {
dfa.Pool[B].Info = info
com.DbPrintf("db_DFAGen", " *** NO State Info for state %d, %s\n", B, com.LF())
}
com.DbPrintf("db_DFAGen", " *** After (top) %s\n", com.LF())
if com.DbOn("db_DFAGen") {
fmt.Printf("for %s StateSet=%s, A=%d, B=%s %s\n", string(S), com.SVar(StateSet), A, com.SVar(B), com.LF())
dfa.DumpPool(false)
}
}
}
dfa.Pool[A].Visited = true
com.DbPrintf("db_DFAGen", "\nBefore Main Loop, %s\n", com.LF())
limit := 0
for stateToDo := dfa.FindNotVisited(); stateToDo != -1; stateToDo = dfa.FindNotVisited() {
com.DbPrintf("db_DFAGen", "\nMain Loop: !!TOP!! State:%d\n", stateToDo)
// -----------------------------------------------------------------------------------------------------------
if !dfa.Pool[stateToDo].Visited {
dfa_set := nn.LambdaClosure(dfa.Pool[stateToDo].StateSet) // Find all the lambda closures from specified state
dfa_set = append(dfa_set, dfa.Pool[stateToDo].StateSet...) // Add in initial state
dfa_set = com.USortIntSlice(dfa_set) // Make set unique
for _, S := range dfa.Sigma {
StateSet := nn.LambdaClosureSet(dfa_set, string(S))
com.DbPrintf("db_DFAGen", " for initial state %s StateSet=%s, %s\n", string(S), com.SVar(StateSet), com.LF())
com.DbPrintf("db_DFAGen", "<><><> this is the point where we should check to see if 'S' is DOT or NCCL, %s\n", com.LF())
if len(StateSet) > 0 {
com.DbPrintf("db_DFAGen", " >>> Have a non-empty result, Input Is %s, %s\n", com.SVar(StateSet), com.LF())
StateSetT := nn.LambdaClosure(StateSet) // need to lambda complete the state set
StateSet = append(StateSet, StateSetT...)
StateSet = com.USortIntSlice(StateSet) // Make set unique
com.DbPrintf("db_DFAGen", " >>> Output Is %s, %s\n", com.SVar(StateSet), com.LF())
B := 0
if t := dfa.HaveStateAlready(StateSet); t != -1 { // Have Already
B = t
com.DbPrintf("db_DFAGen", " Already have this state at location %d, %s\n", t, com.LF())
} else {
B = dfa.GetDFAName(StateSet)
}
dfa.AddEdge(stateToDo, B, string(S))
com.DbPrintf("db_DFAGen", " *** Before %s\n", com.LF())
if r, is, info, Is0Ch := nn.IsTerminalState(StateSet); is {
dfa.Pool[B].Rv = r
dfa.Pool[B].Is0Ch = Is0Ch
dfa.Pool[B].Info = info
com.DbPrintf("db_DFAGen", " *** New state %d, %s\n", B, com.LF())
} else if _, is, info, Is0Ch := nn.IsNonTerminalPushPopState(StateSet); is {
dfa.Pool[B].Is0Ch = Is0Ch
dfa.Pool[B].Info = info
com.DbPrintf("db_DFAGen", " *** New info for state %d, %s\n", B, com.LF())
} else {
com.DbPrintf("db_DFAGen", " *** NO State Info for state %d, %s\n", B, com.LF())
dfa.Pool[B].Info = info
}
com.DbPrintf("db_DFAGen", " *** After %s\n", com.LF())
if com.DbOn("db_DFAGen") {
fmt.Printf(" Add New Edge on %s fr %d to %d, %s\n", string(S), stateToDo, B, com.LF())
fmt.Printf(" for %s StateSet=%s, A(stateToDo)=%d, %s\n", string(S), com.SVar(StateSet), stateToDo, com.LF())
dfa.DumpPool(false)
}
}
}
}
// -----------------------------------------------------------------------------------------------------------
dfa.Pool[stateToDo].Visited = true
limit++
if limit > 50000 {
break
}
}
dfa.VerifyMachine()
}
// -----------------------------------------------------------------------------------------------------------------------------------------------
func (dfa *DFA_PoolType) ConvertToTable() (rv dfaTable) {
rv.InitState = dfa.InitState
rv.SMap = smap.NewSMapType(dfa.Sigma, re.R_not_CH)
rv.N_States = dfa.NumberOfStates()
rv.Machine = make([]*MachineStatesType, rv.N_States, rv.N_States)
XLen := rv.SMap.Length()
rv.Width = XLen
jj := 0
for ii, vv := range dfa.Pool {
if vv.IsUsed {
rv.Machine[jj] = &MachineStatesType{Rv: vv.Rv, Info: vv.Info, Tau: vv.Is0Ch}
rv.Machine[jj].To = make([]int, XLen, XLen)
for kk := 0; kk < XLen; kk++ {
rv.Machine[jj].To[kk] = -1
}
dot := false
dotTo := 0
alpha := false
alphaTo := 0
num := false
numTo := 0
for _, ww := range vv.Next2 {
//if s == re.X_DOT {
// return "\u2022" // Middle Bullet
if ww.On == re.X_DOT {
// fmt.Printf("Found a dot, at jj=%d\n", jj)
dot = true
dotTo = ww.To
} else if ww.On == re.X_ALPHA {
// fmt.Printf("Found a alpha, at jj=%d\n", jj)
alpha = true
alphaTo = ww.To
rr, _ := utf8.DecodeRune([]byte(ww.On))
xx := rv.SMap.MapRune(rr)
rv.Machine[jj].To[xx] = ww.To
} else if ww.On == re.X_NUMERIC {
// com.DbPrintf("dfa2", "Found a numeric, at jj=%d\n", jj)
num = true
numTo = ww.To
rr, _ := utf8.DecodeRune([]byte(ww.On))
xx := rv.SMap.MapRune(rr)
rv.Machine[jj].To[xx] = ww.To
} else {
rr, _ := utf8.DecodeRune([]byte(ww.On))
xx := rv.SMap.MapRune(rr)
rv.Machine[jj].To[xx] = ww.To
}
}
if num {
for ii := 0; ii < rv.Width; ii++ {
// rn := rune(ii + rv.SMap.MinV)
rn := rv.SMap.SigmaRN[ii]
// func (smap *SMapType) ReverseMapRune(x int) rune {
// com.DbPrintf("dfa2", "numeric: ->%s<-\n", string(rn))
if rv.Machine[jj].To[ii] == -1 && unicode.IsDigit(rn) {
rv.Machine[jj].To[ii] = numTo
}
}
}
if alpha {
for ii := 0; ii < rv.Width; ii++ {
// rn := rune(ii + rv.SMap.MinV)
rn := rv.SMap.SigmaRN[ii]
if rv.Machine[jj].To[ii] == -1 && (unicode.IsUpper(rn) || unicode.IsLower(rn)) {
rv.Machine[jj].To[ii] = alphaTo
}
}
}
if dot {
for ii := 0; ii < rv.Width; ii++ {
if rv.Machine[jj].To[ii] == -1 {
rv.Machine[jj].To[ii] = dotTo
}
}
}
if ((vv.Info.Action&com.A_Pop) != 0 || (vv.Info.Action&com.A_Push) != 0 || (vv.Info.Action&com.A_Reset) != 0) && !vv.Info.HardMatch { // xyzzy8
com.DbPrintf("dfa7", "Info2-in-TabGen: State[%d] a Push/Call or a Pop/Return/Reset optration Must be a terminal state with fixed string matched, 'a*' or 'a?' is not a legitimate match.\n", ii)
//for ii := 0; ii < rv.Width; ii++ {
// if rv.Machine[jj].To[ii] == -1 {
// rv.Machine[jj].To[ii] = dotTo
// }
//}
}
jj++
}
}
return
}
func (lr *LexReType) CalcLengthChild(tree *ReTreeNodeType, d int) (x int, hard bool) {
t := 0
hard = false
if d == 1 {
com.DbPrintf("CalcLength", "CalcLengthChild at top: %s\n\n", com.SVarI(tree))
}
switch tree.LR_Tok {
case LR_null: //
for jj := range tree.Children {
t, hard = lr.CalcLengthChild(&tree.Children[jj], d+1)
x += t
}
case LR_Text: //
// com.DbPrintf("CalcLength", "Len of item(%s) = %d, %s\n", tree.Item, len(tree.Item), com.LF())
x += len(tree.Item)
hard = true
case LR_EOF: //
hard = true
case LR_DOT: // .
x += 1
case LR_STAR: // *
x = 0
// com.DbPrintf("CalcLength", "After * x = %d, hard=%v\n", x, hard)
case LR_PLUS: // +
// patch to fix the problem with [0-9]+ not working -- In reality the length is only if it is a "FIXED" length, 0 else
// if len(tree.Children) > 0 {
// t, hard = lr.CalcLengthChild(&tree.Children[0], d+1)
// x += t
// }
// hard = true
x = 0
case LR_QUEST: // ?
x = 0
case LR_OP_BR: // { // {m,n} - need to calculate length of ( m times, length of children
x = 0
case LR_OP_PAR: // (
if len(tree.Children) > 0 {
t, hard = lr.CalcLengthChild(&tree.Children[0], d+1)
x += t
}
// com.DbPrintf("CalcLength", "After ( x = %d, hard=%v\n", x, hard)
case LR_CL_PAR: // )
x = 0
case LR_CCL: // [...]
x += 1
hard = true
case LR_N_CCL: // [^...]
x += 1
case LR_E_CCL: // ]
x += 1
case LR_CARROT: // ^
x += 0
hard = true
case LR_MINUS: // -
x += 1
hard = true
case LR_DOLLAR: // $
hard = true
case LR_OR: // |
y := -1
z := 0
hard = false
if len(tree.Children) > 0 {
hard = true
h := false
for jj := range tree.Children {
z, h = lr.CalcLengthChild(&tree.Children[jj], d+1)
if y == -1 {
y = z
} else if y < z {
y = z
}
if !h {
hard = false
}
}
}
x += y
// com.DbPrintf("CalcLength", "After | x = %d, hard = %v\n", x, hard)
}
return
}