Example #1
0
File: re.go Project: pschlump/lexie
func (lr *LexReType) ParseRe(ss string) {
	com.DbPrintf("db2", "at %s\n", com.LF())
	lr.SetBuf(ss)
	com.DbPrintf("db2", "at %s\n", com.LF())
	lr.parseExpression(0, 0, nil)
	com.DbPrintf("db2", "at %s\n", com.LF())
}
Example #2
0
File: re.go Project: pschlump/lexie
//
// What can I see at the top of a RE
//
//	LR_Text                     //
//	LR_EOF                      //
//	LR_DOT                      // .		-- Match any char
//	LR_STAR                     // *		-- Error if 1st char
//	LR_PLUS                     // +		-- Error if 1st char
//	LR_QUEST                    // ?		-- Error if 1st char
//	LR_B_CCL                    // [		-- Start of CCL Node
//	LR_E_CCL                    // ]
//	LR_OP_PAR                   // (		-- Start of Sub_Re
//	LR_CL_PAR                   // )
//	LR_CCL                      // [...]	-- CCL Node (Above)
//	LR_N_CCL                    // [^...]	-- N_CCL Node
//	LR_CARROT                   // ^		-- BOL
//	LR_MINUS                    // -		-- Text if not in CCL and not 1st char in CCL
//
//	Item     string
//	LR_Tok   LR_TokType
//	Children []*ReTreeNodeType
//	Next     *ReTreeNodeType
//
func (lr *LexReType) DumpParseNodesChild(ch []ReTreeNodeType, d int) {
	com.DbPrintf("DumpParseNodes", "\n%sDumpParseNodesChild: At %s\n", N4Blanks(d), com.LF())
	for ii, vv := range ch {
		com.DbPrintf("DumpParseNodes", "%sat %s [step %d] ", N4Blanks(d), com.LF(), ii)
		com.DbPrintf("DumpParseNodes", "Item: [%s] %d=%s, N-Children=%d\n", vv.Item, vv.LR_Tok, NameOfLR_TokType(vv.LR_Tok), len(vv.Children))
		if len(vv.Children) > 0 {
			lr.DumpParseNodesChild(vv.Children, d+1)
		}
	}
	com.DbPrintf("DumpParseNodes", "%sDumpParseNodesChild: Done %s\n\n", N4Blanks(d), com.LF())
}
Example #3
0
// dfa.TokList.ReplaceToken ( dfa.MTab.Machine[ctx.St].Info.MatchLength, dfa.MTab.Machine[ctx.St].Info.ReplStr )
func (tl *TokenList) ReplaceToken(l int, s string) {
	ii := len(tl.TL) - 1
	lv := len(tl.TL[ii].AToken.Val)
	tl.TL[ii].AToken.IsRepl = true
	tl.TL[ii].AToken.ReplStr = s
	if lv-l >= 1 {
		tl.TL[ii].AToken.Val = tl.TL[ii].AToken.Val[0:lv-l] + s
	} else {
		com.DbPrintf("db_tok01", "Error: ReplaceToken has invalid data, %s\n", com.LF())
	}
	com.DbPrintf("db_tok01", "ReplaceToken: Match: ->%s<- Val: ->%s<-\n", tl.TL[ii].AToken.Match, tl.TL[ii].AToken.Val)
}
Example #4
0
File: re.go Project: pschlump/lexie
func (lr *LexReType) DumpParseNodes() {
	com.DbPrintf("DumpParseNodes", "\nDumpParseNodes: At %s\n", com.LF())
	for ii, vv := range lr.Tree.Children {
		com.DbPrintf("DumpParseNodes", "at %s [step %d] ", com.LF(), ii)
		com.DbPrintf("DumpParseNodes", "Item: [%s] %d=%s, N-Children=%d\n", vv.Item, vv.LR_Tok, NameOfLR_TokType(vv.LR_Tok), len(vv.Children))
		if len(vv.Children) > 0 {
			lr.DumpParseNodesChild(vv.Children, 1)
		}
	}
	com.DbPrintf("DumpParseNodes", "DumpParseNodes: Done %s\n\n", com.LF())
	com.DbPrintf("DumpParseNodesX", "DumpParseNodes: %s\n\n", com.SVarI(lr.Tree))
}
Example #5
0
func (lex *Lexie) OutputActionFlags(dfa *DFA_PoolType) {
	com.DbPrintf("match", "Action Flags Are:\n")
	// com. ConvertActionFlagToString(kk int) (rv string) {
	dn := make(map[int]bool)
	for _, vv := range dfa.Pool {
		if vv.IsUsed {
			if vv.Info.Action != 0 {
				if _, ok := dn[vv.Info.Action]; !ok {
					com.DbPrintf("match", "    %2x: %s\n", vv.Info.Action, com.ConvertActionFlagToString(vv.Info.Action))
					dn[vv.Info.Action] = true
				}
			}
		}
	}
}
Example #6
0
// Get the current line/col no and file name
func (pb *PBReadType) GetPos() (LineNo int, ColNo int, FileName string) {
	com.DbPrintf("pbbuf02", "At: %s\n", com.LF())
	if len(pb.PbBuffer) > 0 {
		com.DbPrintf("pbbuf02", "From Buffer At: %s\n", com.LF())
		LineNo = pb.PbBuffer[0].LineNo
		ColNo = pb.PbBuffer[0].ColNo
		FileName = pb.PbBuffer[0].FileName
	} else {
		com.DbPrintf("pbbuf02", "Not set At: %s\n", com.LF())
		LineNo = 1
		ColNo = 1
		FileName = ""
	}
	return
}
Example #7
0
// Open a file - this puts the file at the end of the input.   This is used on the command line for a list of files
// in order.  Each opened and added to the end of the list.
func (pb *PBReadType) OpenFile(fn string) (err error) {
	com.DbPrintf("pbbuf01", "At: %s\n", com.LF())
	pb.FileName = fn
	pb.AbsFileName, _ = filepath.Abs(fn)
	pb.FilesOpened[pb.AbsFileName] = true

	// read file -> PbBuffer
	b := &ABuffer{
		FileName:    fn,
		AbsFileName: pb.AbsFileName,
		LineNo:      1,
		ColNo:       1,
	}
	pb.PbBuffer = append(pb.PbBuffer, b)

	bb, err := ioutil.ReadFile(fn)
	if err != nil {
		return
	}
	b.EofOnFile = true
	b.Pos = 0
	var rn rune
	var sz int
	b.Buffer = make([]rune, 0, len(bb))
	for ii := 0; ii < len(bb); ii += sz {
		rn, sz = utf8.DecodeRune(bb[ii:])
		b.Buffer = append(b.Buffer, rn)
	}

	return nil
}
Example #8
0
// Return the next rune.  If runes have been pushed back then use those first.
func (pb *PBReadType) NextRune() (rn rune, done bool) {
	com.DbPrintf("pbbuf01", "At: %s\n", com.LF())
	done = false

	if pb.PbTop > 0 {
		pb.PbTop--
		rn = pb.PbAFew[pb.PbTop]
	} else if len(pb.PbBuffer) <= 0 {
		done = true
		// } else if len(pb.PbBuffer) == 1 && pb.PbBuffer[0].Pos >= len(pb.PbBuffer[0].Buffer) && !pb.PbBuffer[0].EofOnFile {
		// Xyzzy - read in more form file - append
		// so far case never happens because EofOnFile is constant true at init time.
	} else if len(pb.PbBuffer) == 1 && pb.PbBuffer[0].Pos >= len(pb.PbBuffer[0].Buffer) && pb.PbBuffer[0].EofOnFile {
		done = true
	} else if len(pb.PbBuffer) > 1 && pb.PbBuffer[0].Pos >= len(pb.PbBuffer[0].Buffer) && pb.PbBuffer[0].EofOnFile {
		pb.PbBuffer = pb.PbBuffer[1:]
		return pb.NextRune()
	} else {
		//fmt.Printf("Just before core, Pos=%d\n", pb.PbBuffer[0].Pos)
		//fmt.Printf("Just before core, Len 1=%d\n", len(pb.PbBuffer[0].Buffer))
		if pb.PbBuffer[0].Pos >= len(pb.PbBuffer[0].Buffer) { // xyzzy --------------------- pjs - new code - not certain if correct ---------------------------------
			done = true
		} else {
			rn = pb.PbBuffer[0].Buffer[pb.PbBuffer[0].Pos]
			pb.PbBuffer[0].Pos++
			if rn == '\n' {
				pb.PbBuffer[0].LineNo++
				pb.PbBuffer[0].ColNo = 1
			} else {
				pb.PbBuffer[0].ColNo++
			}
		}
	}
	return
}
Example #9
0
// Have we already seen the specified file.  Useful for require(fn)
func (pb *PBReadType) FileSeen(fn string) bool {
	com.DbPrintf("pbbuf01", "At: %s\n", com.LF())
	a, _ := filepath.Abs(fn)
	if t, ok := pb.FilesOpened[a]; ok && t {
		return true
	}
	return false
}
Example #10
0
// Set the line/col/file-name for the current buffer - Useful for constructing something like C/Pre processor's #line
func (pb *PBReadType) SetPos(LineNo int, ColNo int, FileName string) {
	com.DbPrintf("pbbuf01", "At: %s\n", com.LF())
	pb.pushbackIntoBuffer()
	if len(pb.PbBuffer) > 0 {
		pb.PbBuffer[0].LineNo = LineNo
		pb.PbBuffer[0].ColNo = ColNo
		pb.PbBuffer[0].FileName = FileName
	}
	return
}
Example #11
0
// Push back a single rune onto input.  You can call this more than one time.
func (pb *PBReadType) PbRune(rn rune) {
	com.DbPrintf("pbbuf01", "At: %s\n", com.LF())

	if pb.PbTop >= MaxAFew { // Buffer is full
		pb.pushbackIntoBuffer()
	}

	pb.PbAFew[pb.PbTop] = rn
	pb.PbTop++
}
Example #12
0
// Push back a string.  Will be converted from an array of byte to an array of runes.
func (pb *PBReadType) PbByteArray(s []byte) {
	com.DbPrintf("pbbuf01", "At: %s\n", com.LF())
	rns := make([]rune, 0, len(s))
	var rn rune
	var sz int
	for ii := 0; ii < len(s); ii += sz {
		rn, sz = utf8.DecodeRune(s[ii:])
		rns = append(rns, rn)
	}
	pb.PbRuneArray(rns)
}
Example #13
0
File: in.go Project: pschlump/lexie
func ParsePlist(pl string) (aa []string) {
	t1 := pl_re.FindAllStringSubmatch(pl, -1)
	if t1 != nil {
		com.DbPrintf("in", "t1=%s\n", com.SVarI(t1))
		for _, vv := range t1 {
			if len(vv) > 3 && vv[2] != "" {
				aa = append(aa, vv[2])
			}
		}
	}
	return
}
Example #14
0
// Map an input rune to one of the possible output subscripts
func (smap *SMapType) MapRune(rn rune) int {
	x := int(rn) - smap.MinV
	// fmt.Printf("x=%d, rn=%04x ( %s ),  %s\n", x, rn, string(rn), com.LF())
	if x > smap.MaxV {
		// fmt.Printf("At %s\n", com.LF())
		if y, ok := smap.M1[rn]; ok {
			// fmt.Printf("At %s\n", com.LF())
			return y
		}
		// fmt.Printf("At %s\n", com.LF())
		return smap.NoMapTo
	} else if x >= 0 {
		com.DbPrintf("smap", "********************************** this one **************************, %s\n", string(rn))
		v := smap.M0[x]
		if v == smap.NoMapTo {
			com.DbPrintf("smap", "********************************** No MAP - 28 case \n")
			if unicode.IsDigit(rn) {
				if y, ok := smap.M1[re.R_NUMERIC]; ok {
					com.DbPrintf("smap", "********************************** case - numeric \n")
					return y
				}
			} else if unicode.IsUpper(rn) {
				if y, ok := smap.M1[re.R_UPPER]; ok {
					com.DbPrintf("smap", "********************************** case - upper\n")
					return y
				}
			} else if unicode.IsLower(rn) {
				if y, ok := smap.M1[re.R_LOWER]; ok {
					com.DbPrintf("smap", "********************************** case - lower\n")
					return y
				}
			}
		}
		// fmt.Printf("At %s\n", com.LF())
		return v
	} else {
		// fmt.Printf("At %s\n", com.LF())
		return smap.NoMapTo
	}
}
Example #15
0
File: in.go Project: pschlump/lexie
// Tok_Name=1 Tok_Name "T O K"
func ParseNameValue(nv string) (name string, value string) {
	name, value = "", ""
	t1 := pnv_re.FindAllStringSubmatch(nv, -1)
	com.DbPrintf("in", "t1=%s\n", com.SVarI(t1))
	if t1 != nil && len(t1[0]) > 0 {
		name = t1[0][1]
		if len(t1[0]) > 3 {
			value = t1[0][3]
		}
	} else {
		name = nv
	}
	return
}
Example #16
0
File: in.go Project: pschlump/lexie
// Rv(Name) Ignore(Xxx)
func ParseActionItem(act string) (aa string, pp string) {
	aa, pp = "", ""
	t1 := fx_re.FindAllStringSubmatch(act, -1)
	if t1 != nil {
		com.DbPrintf("in", "t1=%s\n", com.SVarI(t1))
		aa = t1[0][1]
		if len(t1[0]) > 1 {
			pp = t1[0][2]
		}
	} else {
		aa = act
	}
	return
}
Example #17
0
File: in.go Project: pschlump/lexie
// -----------------------------------------------------------------------------------------------------------------------------
// -----------------------------------------------------------------------------------------------------------------------------
//type ImRuleType struct {
//	PatternType  int    // Pattern, Str0,1,2, $eof etc.  // Pattern Stuff --------------------------------------------------------------------------------
//	Pattern      string //
func (Im *ImType) LocatePattern(ff *ImRuleType, in []*ImRuleType) (rv int) {
	rv = -1
	// fmt.Printf("LocatePattern for %s %d\n", ff.Pattern, ff.PatternType)
	for kk, tt := range in {
		// fmt.Printf("    Compare to %s %d\n", tt.Pattern, tt.PatternType)
		if tt.PatternType == ff.PatternType && tt.Pattern == ff.Pattern {
			com.DbPrintf("in", "    Found\n")
			return kk
		}
	}
	// fmt.Printf("    NOT NOT NOT Found\n")
	// Xyzzy - should add an error at this point?
	//		2. Errors about missing machines in mixin not reported at all - see xyzzyMixin01
	return
}
Example #18
0
File: in.go Project: pschlump/lexie
// -----------------------------------------------------------------------------------------------------------------------------
// -----------------------------------------------------------------------------------------------------------------------------
func (Im *ImType) SaveDef(DefType string, Defs []string, line_no int, file_name string) {
	if validateDefType(DefType) {
		for _, nm := range Defs {
			dd, ok := Im.Def.DefsAre[DefType]
			if !ok {
				dd = ImDefinedValueType{
					Seq:          1,
					WhoAmI:       DefType,
					NameValue:    make(map[string]int),
					NameValueStr: make(map[string]string),
					Reverse:      make(map[int]string),
					SeenAt:       make(map[string]ImSeenAtType),
				}
			}
			// seq := dd.Seq
			n, v := ParseNameValue(nm)
			if n == "" && v == "" { // xyzzy100
				return
			}
			com.DbPrintf("in", "Input: ->%s<- n >%s< v >%s<\n", nm, n, v)
			if v != "" {
				dd.NameValue[n] = -2 //							//
				if vv, ok1 := dd.NameValueStr[n]; !ok1 {
					dd.NameValueStr[n] = v //						//
				} else {
					if vv != v {
						fmt.Printf("Error: Attempt to redfine %s from %s to %s - Probably an error\n", n, vv, v)
					}
				}
			} else {
				dd.NameValue[n] = -1 //							//
				if _, ok1 := dd.NameValueStr[n]; !ok1 {
					dd.NameValueStr[n] = "" //							//
				}
			}
			// dd.Seq = seq + 1
			sa := dd.SeenAt[n]
			sa.LineNo = append(sa.LineNo, line_no)
			sa.FileName = append(sa.FileName, file_name)
			dd.SeenAt[n] = sa
			Im.Def.DefsAre[DefType] = dd
		}
	}
	// fmt.Printf("It Is:%+v\n", Im)
}
Example #19
0
// Place the contents of a file in buffers at the head so NextRune will pull from this next.
func (pb *PBReadType) PbFile(fn string) (err error) {
	com.DbPrintf("pbbuf01", "At: %s\n", com.LF())
	err = nil

	pb.pushbackIntoBuffer()

	pb.FileName = fn
	pb.AbsFileName, _ = filepath.Abs(fn)
	pb.FilesOpened[pb.AbsFileName] = true

	// read file -> PbBuffer
	b := &ABuffer{
		FileName:    fn,
		AbsFileName: pb.AbsFileName,
		LineNo:      1,
		ColNo:       1,
	}
	// pb.PbBuffer = append(pb.PbBuffer, b)
	// data = append([]string{"Prepend Item"}, data...)
	pb.PbBuffer = append([]*ABuffer{b}, pb.PbBuffer...) // prepend

	bb, err := ioutil.ReadFile(fn)
	if err != nil {
		return
	}
	b.EofOnFile = true
	b.Pos = 0
	var rn rune
	var sz int
	b.Buffer = make([]rune, 0, len(bb))
	for ii := 0; ii < len(bb); ii += sz {
		rn, sz = utf8.DecodeRune(bb[ii:])
		b.Buffer = append(b.Buffer, rn)
	}

	return
}
Example #20
0
func (dfa *DFA_PoolType) DumpPool(all bool) {
	if all {
		com.DbPrintf("db_DumpDFAPool", "Cur: %d Top: %d NextFree %d\n", dfa.Cur, dfa.Top, dfa.NextFree)
	}
	com.DbPrintf("db_DumpDFAPool", "\n---------------------------- DFA Output -----------------------------------------------\n")
	com.DbPrintf("db_DumpDFAPool", "\nDFA InitState: %d, Sigma ->%s<-\n\n", dfa.InitState, dfa.Sigma)
	pLnNo := com.DbOn("db_DFA_LnNo")
	IfLnNo := func(s string) string {
		if pLnNo {
			t := fmt.Sprintf("[%3s]", s)
			return t
		}
		return ""
	}
	com.DbPrintf("db_DumpDFAPool", "%3s%s: ", "St", IfLnNo("/Ln"))
	com.DbPrintf("db_DumpDFAPool", " %12s %12s \u2714              \tEdges", "StateName", "StateSet")
	com.DbPrintf("db_DumpDFAPool", "\n\n")
	for ii, vv := range dfa.Pool {
		if all || vv.IsUsed {
			com.DbPrintf("db_DumpDFAPool", "%3d%s: ", ii, IfLnNo(vv.LineNo))
			com.DbPrintf("db_DumpDFAPool", " %12s %12s %s :", vv.StateName, com.SVar(vv.StateSet), com.ChkOrBlank(vv.Visited))
			if vv.Rv > 0 {
				if vv.Is0Ch {
					com.DbPrintf("db_DumpDFAPool", " \u03c4:Tau:%04d ", vv.Rv)
				} else {
					com.DbPrintf("db_DumpDFAPool", " T:%04d ", vv.Rv)
				}
			} else {
				com.DbPrintf("db_DumpDFAPool", "        ")
			}
			if com.DbOn("db_DumpDFAPool") {
				fmt.Printf("\t E:")
				for _, ww := range vv.Next2 {
					if ww.Is0ChMatch {
						fmt.Printf("//Found  \u03c4 (%s) //", com.LF()) // Show a Tau(t) for a lambda that matchiens on else conditions.
					}
					if ww.IsLambda {
						fmt.Printf("{  ERROR!! \u03bb  %2d -> %2d  %s}  ", ww.From, ww.To, ww.LineNo)
					} else {
						// fmt.Printf("{ \"%s\" %2d -> %2d  %s}  ", ww.On, ww.From, ww.To, IfLnNo(ww.LineNo))
						on, _ := utf8.DecodeRune([]byte(ww.On))
						son := fmt.Sprintf("%q", ww.On)
						switch on {
						case re.R_DOT: // = '\uF8FA' // Any char in Sigma
							son = "DOT/uF8FA"
						case re.R_BOL: // = '\uF8F3' // Beginning of line
							son = "BOL/uF8F3"
						case re.R_EOL: // = '\uF8F4' // End of line
							son = "EOL/uF8F4"
						case re.R_NUMERIC: // = '\uF8F5'
							son = "NUMERIC/uF8F5"
						case re.R_LOWER: // = '\uF8F6'
							son = "LOWER/uF8F6"
						case re.R_UPPER: // = '\uF8F7'
							son = "UPPER/uF8F7"
						case re.R_ALPHA: // = '\uF8F8'
							son = "ALPHA/uF8F8"
						case re.R_ALPHNUM: // = '\uF8F9'
							son = "ALPHANUM/uF8F9"
						case re.R_EOF: // = '\uF8FB'
							son = "EOF/uF8FB"
						case re.R_not_CH: // = '\uF8FC' // On input lookup if the char is NOT in Signa then it is returned as this.
							son = "else_CH/uF8Fc"
						case re.R_N_CCL: // = '\uF8FD' // If char is not matched in this state then take this path
							son = "N_CCL/uF8Fd"
						case re.R_LAMBDA_MATCH: // = '\uF8FE'
							son = "LambdaM/uF8FE"
						}
						fmt.Printf("{ %s  %2d -> %2d  %s}  ", son, ww.From, ww.To, IfLnNo(ww.LineNo))
					}
				}
				fmt.Printf("\n")
				if vv.Info.Action != 0 || vv.Info.MatchLength != 0 {
					// fmt.Printf("\t\t\tInfo: %s\n", com.SVar(vv.Info))		// xyzzy - output Info
					// xyzzy - NextState info
					fmt.Printf("\t\t\tDFA.Info: %s", nfa.DumpInfo(vv.Info))
					// if ((vv.Info.Action&com.A_Pop) != 0 || (vv.Info.Action&com.A_Push) != 0 || (vv.Info.Action&com.A_Reset) != 0) && !vv.Info.HardMatch {		// xyzzy8
					fmt.Printf(" IsHard=%v (((false imples else case Rv!)))\n", vv.Info.HardMatch)
				}
				fmt.Printf("\n")
			}
		}
	}
}
Example #21
0
func Test_PbBufer01(t *testing.T) {

	SymbolTable := make([]*MacroDefTestType, 0, 100)
	Define := func(name rune, body string) {
		for ii := 0; ii < len(SymbolTable); ii++ {
			// fmt.Printf("Search at %d, for %s\n", ii, string(name))
			if SymbolTable[ii] != nil && SymbolTable[ii].Rn == name {
				SymbolTable[ii].Body = body
				return
			}
		}
		// fmt.Printf("Append\n")
		SymbolTable = append(SymbolTable, &MacroDefTestType{Rn: name, Body: body})
	}
	ResetST := func() {
		// SymbolTable = make([]*MacroDefTestType, 0, 100)
		SymbolTable = SymbolTable[:1]
	}
	HaveMacro := func(name rune) (body string, found bool) {
		body = ""
		found = false
		for ii := 0; ii < len(SymbolTable); ii++ {
			if SymbolTable[ii] != nil && SymbolTable[ii].Rn == name {
				body, found = SymbolTable[ii].Body, true
				return
			}
		}
		return
	}

	for ii, vv := range Pb01Test {

		if !vv.SkipTest {

			// Implement a quick - fetch execute macine to test - the PbBuffer - commands/opcodes are the Cmd* constants above.
			ss := ""
			pb := NewPbRead()
			ResetST()
			for pc, ww := range vv.Actions {

				switch ww.OpCode {
				case CmdOpenFile: // Open a file , at the tail end of list of input
					pb.OpenFile(ww.Fn)
					com.DbPrintf("testCode", "Open file %s At: %s\n", ww.Fn, com.LF())
					if com.DbOn("testDump") {
						pb.Dump01(os.Stdout)
					}
				case CmdPbString: // Push back a string
					pb.PbString(ww.Data)
					if com.DbOn("testDump") {
						pb.Dump01(os.Stdout)
					}
				case CmdPbRune: // Push back a rune
					pb.PbRune(ww.Rn)
					if com.DbOn("testDump") {
						pb.Dump01(os.Stdout)
					}
				case CmdPbRuneArray: // Push back a rune array
					pb.PbRuneArray(ww.RnS)
					if com.DbOn("testDump") {
						pb.Dump01(os.Stdout)
					}
				case CmdNextNChar:
					for ll := 0; ll < ww.X; ll++ {
						rn, done := pb.NextRune()
						if !done {
							ss = ss + string(rn)
						}
						com.DbPrintf("testCode", "Case 5: At: ->%s<- ll=%d ss >>>%s<<< %s\n", string(rn), ll, ss, com.LF())
					}
					if com.DbOn("testDump") {
						pb.Dump01(os.Stdout)
					}
				case CmdPeek:
					rn, done := pb.PeekRune()
					if done || rn != ww.Rn {
						t.Errorf("%04s: Peek at [pc=%d] in test [%s] did not work, got %s expected %s, done=%v\n", pc, ii, string(rn), string(ww.Rn), done)
					}
				case CmdOutputToEof:
					com.DbPrintf("testCode", "All Done: ss >>>%s<<< before At: %s\n", ss, com.LF())
					if com.DbOn("testDump") {
						pb.Dump01(os.Stdout)
					}
					for rn, done := pb.NextRune(); !done; rn, done = pb.NextRune() {
						ss = ss + string(rn)
					}
					com.DbPrintf("testCode", "All Done: ss >>>%s<<< after At: %s\n", ss, com.LF())
				case CmdPbByteArray: // Push back a byte array
					pb.PbByteArray([]byte(ww.Data))
					if com.DbOn("testDump") {
						pb.Dump01(os.Stdout)
					}
				case CmdPbFile: // Open file and push contents back onto input at head of list. (Macro file, Include, Require)
					pb.PbFile(ww.Fn)
					com.DbPrintf("testCode", "Pb file %s At: %s\n", ww.Fn, com.LF())
					if com.DbOn("testDump") {
						pb.Dump01(os.Stdout)
					}
				case CmdFileSeen:
					fs := pb.FileSeen(ww.Fn)
					if fs != ww.FileSeenFlag {
						t.Errorf("%04s: Peek at [pc=%d] in test [%s] did not work, got %v expected %s for file seen flagv\n", pc, ii, fs, ww.FileSeenFlag)
					}
				case CmdGetPos: // Check get file name
					ln, cn, fn := pb.GetPos()
					com.DbPrintf("testCode", "fn=%s ln=%d cn=%d\n", fn, ln, cn)
					if ln != ww.LineNo {
						t.Errorf("%04s: %d: did not match line no Expected ->%d<-, Got ->%d<-\n", vv.Test, pc, ww.LineNo, ln)
					}
					if cn != ww.ColNo {
						t.Errorf("%04s: %d: did not match col no Expected ->%d<-, Got ->%d<-\n", vv.Test, pc, ww.ColNo, cn)
					}
					if fn != ww.Fn {
						t.Errorf("%04s: %d: did not match file name Expected ->%s<-, Got ->%s<-\n", vv.Test, pc, ww.Fn, fn)
					}
				case CmdSetPos: // Check get file name
					pb.SetPos(ww.LineNo, ww.ColNo, ww.Fn)
				case CmdResetST: // Reset symbol table
					ResetST()
				case CmdMacroProc: // Apply 1 char macros to input and process
					for rn, done := pb.NextRune(); !done; rn, done = pb.NextRune() {
						if m_body, m_found := HaveMacro(rn); m_found {
							pb.PbString(m_body)
						} else {
							ss = ss + string(rn)
						}
					}
				case CmdDefineMacro: // Define
					Define(ww.Rn, ww.Data)
				case CmdDumpBuffer: // Dump the buffer  - debuging
					if ww.Fn == "" {
						pb.Dump01(os.Stdout)
					} else {
						fp, err := com.Fopen(ww.Fn, "w")
						if err == nil {
							pb.Dump01(fp)
							fp.Close()
						} else {
							pb.Dump01(os.Stdout)
							t.Errorf("%04s: Unable to open file for output ->%s<-, error: %s\n", vv.Test, ww.Fn, err)
						}
					}
				case CmdResetOutput: // Reset output
					ss = ""
				case CmdPushBackXCopies: // Special test to pub back more than buffer of 'x'
					x := strings.Repeat(ww.Data, ww.X)
					pb.PbString(x)
				}
			}
			if ss != vv.Results {
				t.Errorf("%04s: did not match Expected ->%s<-, Got ->%s<-\n", vv.Test, vv.Results, ss)
			} else {
				com.DbPrintf("testCode", "%04s: Passed ------------------------------------------------------------------------------------------------\n\n", vv.Test)
			}
		}
	}

}
Example #22
0
File: re.go Project: pschlump/lexie
func (lr *LexReType) parseCCL(depth int, ww LR_TokType) (tree ReTreeNodeType) {
	pos := 0
	com.DbPrintf("db2", "parseCCL Top: depth=%d,  %d=%s\n", depth, ww, NameOfLR_TokType(ww))
	s := ""
	dx := 0
	marked := false
	c, w := lr.Next()
	for w != LR_EOF {
		com.DbPrintf("re2", "Top of parseCCL dx=%d ->%s<-\n", dx, c)
		switch w {
		case LR_MINUS: // -		-- Text if not in CCL and not 1st char in CCL
			fallthrough
		case LR_Text: //			-- Add a node to list, move right
			fallthrough
		case LR_CARROT: // ^		-- BOL
			fallthrough
		case LR_DOT: // .		-- Match any char
			fallthrough
		case LR_STAR: // *		-- Error if 1st char, else take prev item from list, star and replace it.
			fallthrough
		case LR_PLUS: // +		-- Error if 1st char
			fallthrough
		case LR_QUEST: // ?		-- Error if 1st char
			fallthrough
		case LR_OP_PAR: // (		-- Start of Sub_Re
			fallthrough
		case LR_CL_PAR: // )
			fallthrough
		case LR_OR: // |
			fallthrough
		case LR_OP_BR: // {
			fallthrough
		case LR_CL_BR: // }
			fallthrough
		case LR_COMMA: // ,
			fallthrough
		case LR_DOLLAR: // $
			s += c

		case LR_N_CCL: // [^...]	-- N_CCL Node
			marked = true
			com.DbPrintf("re2", "    incr to %d, %s\n", dx, com.LF())
			s += c // Add To CCL

		case LR_CCL: // [...]	-- CCL Node (Above)
			marked = true
			com.DbPrintf("re2", "    incr to %d, %s\n", dx, com.LF())
			s += c // Add To CCL

		case LR_E_CCL:
			dx--
			com.DbPrintf("re2", "    decr to %d, %s\n", dx, com.LF())
			if dx < 0 {
				tree.Item = expandCCL(s) // Do Something, Return
				tree.LR_Tok = ww
				return
			} else {
				s += c //
			}

		default:
			lr.Error = append(lr.Error, errors.New(fmt.Sprintf("Unreacable Code, invalid token in parseCCL = %d, %s", w, com.LF())))
			tree.Item = expandCCL(s) // Do Something, Return
			tree.LR_Tok = ww
			return
		}
		pos++
		c, w = lr.Next()

		if c == ":" && marked {
			marked = false
			dx++
		}
	}
	if w == LR_EOF {
		lr.Err("EOF found in Character Class [...] or [^...].")
	}
	tree.Item = expandCCL(s)
	tree.LR_Tok = ww
	return
}
Example #23
0
func (s *ReTesteSuite) TestLexie(c *C) {

	// Test of Parseing REs into RE-TParseTrees
	// return
	fmt.Fprintf(os.Stderr, "Test Parsing of REs, %s\n", com.LF())

	com.DbOnFlags["db_DumpPool"] = true
	com.DbOnFlags["parseExpression"] = true
	com.DbOnFlags["CalcLength"] = true
	com.DbOnFlags["DumpParseNodes"] = true
	com.DbOnFlags["DumpParseNodesX"] = true

	fmt.Printf("**** In Test RE\n")

	n_err := 0
	n_skip := 0

	for ii, vv := range Lexie00Data {
		if !vv.SkipTest {
			fmt.Printf("\n\n--- %d Test: %s -----------------------------------------------------------------------------\n\n", ii, vv.Test)
			lr := NewLexReType()
			lr.SetBuf(vv.Re)
			tn := 0
			cc, ww := lr.Next()
			for ww != LR_EOF {
				fmt.Printf("    %s % x = %d %s\n", string(cc), string(cc), ww, LR_TokTypeLookup[ww])

				if len(vv.SM) > 0 {
					// Check correct token
					if vv.SM[tn].Tok != ww {
						fmt.Printf("     Failed to return the correct token, expecting %d/%s, got %d/%s\n", vv.SM[tn].Tok, LR_TokTypeLookup[vv.SM[tn].Tok], ww,
							LR_TokTypeLookup[ww])
						c.Check(int(vv.SM[tn].Tok), Equals, int(ww))
						n_err++
					}
					// Check correct string/rune returned
					if !CmpByteArr(vv.SM[tn].Dat, []byte(cc)) {
						fmt.Printf("     The returned runes did not match\n")
						c.Check(string(vv.SM[tn].Dat), Equals, cc)
						n_err++
					}

				}

				tn++
				cc, ww = lr.Next()
			}
			fmt.Printf("\n--- %d End : %s -----------------------------------------------------------------------------\n\n", ii, vv.Test)
		}
	}

	for ii, vv := range Lexie01Data {
		if !vv.SkipTest {
			fmt.Printf("\n\n--- %d Test: %s -----------------------------------------------------------------------------\n\n", ii, vv.Test)

			lr := NewLexReType()
			lr.ParseRe(vv.Re)

			lr.Sigma = lr.GenerateSigma()
			fmt.Printf("Sigma: ->%s<-\n", lr.Sigma)

			if vv.Sigma != "" {
				if vv.Sigma != lr.Sigma {
					fmt.Printf("     The calculated and reference Sigma did not match\n")
					c.Check(vv.Sigma, Equals, lr.Sigma)
					n_err++
				}
			}

			fmt.Printf("\n--- %d End : %s -----------------------------------------------------------------------------\n\n", ii, vv.Test)
		}
	}

	// -------------------------------------------------------------------------------------------------------------------------------------------------
	lr := NewLexReType()
	com.DbOnFlags["DumpParseNodes"] = true
	com.DbOnFlags["DumpParseNodesX"] = true
	com.DbOnFlags["db_DumpPool"] = true
	com.DbOnFlags["parseExpression"] = true
	for i, v := range Test6Data {
		com.DbPrintf("debug", "\nTest[%03d]: `%s` %s\n\n", i, v.Re, strings.Repeat("-", 120-len(v.Re)))
		// fmt.Printf("%s *** com.Red *** %s\n", ansi.ColorCode("red"), ansi.ColorCode("reset"))
		lr.ParseRe(v.Re)
		lr.DumpParseNodes()
		if len(v.TopTok) > 0 {
			if len(v.TopTok) != len(lr.Tree.Children) {
				com.DbPrintf("debug", "%sError%s: wrong number of tokens prsed, Expected: %d Got %d\n", com.Red, com.Reset, len(v.TopTok), len(lr.Tree.Children))
				n_err++
			} else {
				for i := 0; i < len(v.TopTok); i++ {
					if v.TopTok[i] != lr.Tree.Children[i].LR_Tok {
						com.DbPrintf("debug", "%sError%s: invalid token returnd at postion %d\n", com.Red, com.Reset, i)
						c.Check(v.TopTok[i], Equals, lr.Tree.Children[i].LR_Tok)
						n_err++
					}
				}
			}
		}
		if len(v.TopVal) > 0 {
			if len(v.TopVal) != len(lr.Tree.Children) {
				com.DbPrintf("debug", "%sError%s: wrong number of tokens prsed, Expected: %d Got %d - Based on number of values, TopVal\n", com.Red, com.Reset, len(v.TopVal), len(lr.Tree.Children))
				n_err++
			} else {
				for i := 0; i < len(v.TopVal); i++ {
					if v.TopVal[i] != lr.Tree.Children[i].Item {
						com.DbPrintf("debug", "%sError%s: invalid value at postion %d, %s\n", com.Red, com.Reset, i, com.LF())
						n_err++
					}
				}
			}
		}
		if len(lr.Error) > v.NExpectedErr {
			com.DbPrintf("debug", "%sError%s: Errors reported in R.E. parsing %d\n", com.Red, com.Reset, len(lr.Error))
			n_err++
		} else if len(lr.Error) > 0 {
			com.DbPrintf("debug", "%sNote%s: Errors reported in R.E. parsing %d\n", com.Green, com.Reset, len(lr.Error))
		}
		lr.Error = lr.Error[:0]
		com.DbPrintf("debug", "\nDone[%03d]: `%s` %s\n\n", i, v.Re, strings.Repeat("-", 120-len(v.Re)))
	}
	if n_err > 0 {
		fmt.Fprintf(os.Stderr, "%sFailed, # of errors = %d%s\n", com.Red, n_err, com.Reset)
		com.DbPrintf("debug", "\n\n%sFailed, # of errors = %d%s\n", com.Red, n_err, com.Reset)
	} else {
		fmt.Fprintf(os.Stderr, "%sPASS%s\n", com.Green, com.Reset)
		com.DbPrintf("debug", "\n\n%sPASS%s\n", com.Green, com.Reset)
	}

	if n_skip > 0 {
		fmt.Fprintf(os.Stderr, "%sSkipped, # of files without automated checks = %d%s\n", com.Yellow, n_skip, com.Reset)
		com.DbPrintf("debug", "\n\n%sSkipped, # of files without automated checks = %d%s\n", com.Yellow, n_skip, com.Reset)
	}
	if n_err > 0 {
		c.Check(n_err, Equals, 0)
		fmt.Fprintf(os.Stderr, "%sFailed, # of errors = %d%s\n", com.Red, n_err, com.Reset)
		com.DbPrintf("debug", "\n\n%sFailed, # of errors = %d%s\n", com.Red, n_err, com.Reset)
	} else {
		fmt.Fprintf(os.Stderr, "%sPASS%s\n", com.Green, com.Reset)
		com.DbPrintf("debug", "\n\n%sPASS%s\n", com.Green, com.Reset)
	}
}
Example #24
0
File: re.go Project: pschlump/lexie
func expandCCL(s string) (ccl string) {
	ccl = ""
	com.DbPrintf("db2", "at %s\n", com.LF())

	pos := 0
	if len(s) > 0 && s[0:1] == "-" { // Check for leading '-' include in CCL
		ccl += "-"
		pos = 1
	}

	for ii := pos; ii < len(s); ii++ {
		com.DbPrintf("re2", "ii=%d remaining ->%s<-, %s\n", ii, s[ii:], com.LF())
		if strings.HasPrefix(s[ii:], "[:alphnum:]") {
			com.DbPrintf("re2", "   Matched: %s\n", com.LF())
			// ccl += X_ALPHA
			ccl += X_LOWER
			ccl += X_UPPER
			ccl += X_NUMERIC
			ii += len("[:alphnum:]") - 1
		} else if strings.HasPrefix(s[ii:], "[:alpha:]") {
			com.DbPrintf("re2", "   Matched: %s\n", com.LF())
			// ccl += X_ALPHA
			ccl += X_LOWER
			ccl += X_UPPER
			ii += len("[:alpha:]") - 1
		} else if strings.HasPrefix(s[ii:], "[:lower:]") {
			com.DbPrintf("re2", "   Matched: %s\n", com.LF())
			ccl += X_LOWER
			ii += len("[:lower:]") - 1
		} else if strings.HasPrefix(s[ii:], "[:upper:]") {
			com.DbPrintf("re2", "   Matched: %s\n", com.LF())
			ccl += X_UPPER
			ii += len("[:upper:]") - 1
		} else if strings.HasPrefix(s[ii:], "[:numeric:]") {
			com.DbPrintf("re2", "   Matched: %s\n", com.LF())
			ccl += X_NUMERIC
			ii += len("[:numeric:]") - 1
		} else if ii+9 <= len(s) && s[ii:ii+9] == "a-zA-Z0-9" {
			com.DbPrintf("re2", "   Matched: %s\n", com.LF())
			// ccl += X_ALPHA
			ccl += X_LOWER
			ccl += X_UPPER
			ccl += X_NUMERIC
			ii += 8
		} else if ii+6 <= len(s) && s[ii:ii+6] == "a-zA-Z" {
			com.DbPrintf("re2", "   Matched: %s\n", com.LF())
			// ccl += X_ALPHA
			ccl += X_LOWER
			ccl += X_UPPER
			ii += 5
		} else if ii+3 <= len(s) && s[ii:ii+3] == "0-9" {
			com.DbPrintf("re2", "   Matched: %s\n", com.LF())
			// fmt.Printf("matched 0-9 pattern\n")
			ccl += X_NUMERIC
			ii += 2
		} else if ii+3 <= len(s) && s[ii:ii+3] == "a-z" {
			com.DbPrintf("re2", "   Matched: %s\n", com.LF())
			ccl += X_LOWER
			ii += 2
		} else if ii+3 <= len(s) && s[ii:ii+3] == "A-Z" {
			com.DbPrintf("re2", "   Matched: %s\n", com.LF())
			ccl += X_UPPER
			ii += 2
		} else if ii+2 < len(s) && s[ii+1:ii+2] == "-" {
			com.DbPrintf("re2", "   Matched: %s\n", com.LF())
			// xyzzyRune  TODO - this code is horribley non-utf8 compatable at this moment in time.
			e := s[ii+2]
			// fmt.Printf("matched a-b pattern, b=%s e=%s\n", string(s[ii]), string(e))
			if s[ii] >= e {
				fmt.Printf("Error: Poorly formatted character-class, beginning is larger than or equal to end of CCL\n") // Xyzzy - need line number etc
			}
			for b := s[ii]; b <= e; b++ {
				ccl += string(b)
			}
			ii += 2
		} else {
			ccl += s[ii : ii+1]
		}
		// fmt.Printf("bottom ccl now: ->%s<-\n", ccl)
	}

	return
}
Example #25
0
File: re.go Project: pschlump/lexie
// mm, nn := lr.parseIterator ( depth+1 )
func (lr *LexReType) parseIterator(depth int) (tree ReTreeNodeType) {
	pos := 0
	com.DbPrintf("db2", "parseIterator Top: depth=%d\n", depth)
	s := ""
	c, w := lr.Next()
	for w != LR_EOF {
		switch w {
		case LR_MINUS: // -		-- Text if not in CCL and not 1st char in CCL
			fallthrough
		case LR_CARROT: // ^		-- BOL
			fallthrough
		case LR_DOT: // .		-- Match any char
			fallthrough
		case LR_STAR: // *		-- Error if 1st char, else take prev item from list, star and replace it.
			fallthrough
		case LR_PLUS: // +		-- Error if 1st char
			fallthrough
		case LR_QUEST: // ?		-- Error if 1st char
			fallthrough
		case LR_OP_PAR: // (		-- Start of Sub_Re
			fallthrough
		case LR_CL_PAR: // )
			fallthrough
		case LR_CCL: // [...]	-- CCL Node (Above)
			fallthrough
		case LR_OR: // |
			fallthrough
		case LR_OP_BR: // {
			fallthrough
		case LR_DOLLAR: // $
			fallthrough
		case LR_E_CCL:
			fallthrough
		case LR_N_CCL: // [^...]	-- N_CCL Node
			lr.Error = append(lr.Error, errors.New(fmt.Sprintf("in parseIterator, Invalid {m,n} - invalid chars found, %s", com.LF())))
			tree.Mm, tree.Nn = 1, 1
			tree.LR_Tok = LR_OP_BR
			return

		case LR_Text: //			-- Add a node to list, move right
			if c[0] >= '0' && c[0] <= '9' || c[0] == ',' {
				s += c // Add To Iterator
			} else {
				lr.Error = append(lr.Error, errors.New(fmt.Sprintf("Unreacable Code, invalid token in parseIterator, %s", com.LF())))
				tree.Mm, tree.Nn = lr.parseIteratorString(s) // Do Something, Return
				tree.Item = "{"
				tree.LR_Tok = LR_OP_BR
				return
			}
		case LR_COMMA: // ,
			s += c // Add To Iterator
		case LR_CL_BR: // }
			tree.Mm, tree.Nn = lr.parseIteratorString(s) // Do Something, Return
			tree.Item = "{"
			tree.LR_Tok = LR_OP_BR
			return
		default:
			lr.Error = append(lr.Error, errors.New(fmt.Sprintf("Unreacable Code, invalid token in parseIterator, %s", com.LF())))
			tree.Mm, tree.Nn = lr.parseIteratorString(s) // Do Something, Return
			tree.Item = "{"
			tree.LR_Tok = LR_OP_BR
			return
		}
		pos++
		c, w = lr.Next()
	}
	if w == LR_EOF {
		lr.Err("EOF found in Iterator {m,n}.")
	}
	tree.Mm, tree.Nn = lr.parseIteratorString(s) // Do Something, Return
	tree.Item = "{"
	tree.LR_Tok = LR_OP_BR
	return
}
Example #26
0
File: re.go Project: pschlump/lexie
func (lr *LexReType) parseExpression(depth int, d_depth int, xTree *ReTreeNodeType) []ReTreeNodeType {
	//var first *ReTreeNodeType
	//var last *ReTreeNodeType
	pre := strings.Repeat("    ", depth)
	if depth == 0 {
		xTree = lr.Tree
		com.DbPrintf("parseExpression", "%sat %s !!!top!!!, depth=%d \n", pre, com.LF(), depth)
	}
	isFirst := true
	inOr := false
	com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())
	c, w := lr.Next()
	for w != LR_EOF {
		com.DbPrintf("parseExpression", "%sat %s !!!top!!!, depth=%d c=->%s<- w=%d %s -- Loop Top -- xTree=%s\n\n",
			pre, com.LF(), depth, c, w, NameOfLR_TokType(w), com.SVarI(xTree))
		switch w {
		case LR_CL_BR: // }
			fallthrough
		case LR_COMMA: // ,
			fallthrough
		case LR_E_CCL:
			fallthrough
		case LR_MINUS: // -		-- Text if not in CCL and not 1st char in CCL
			fallthrough
		case LR_Text: //			-- Add a node to list, move right
			//if true {
			xTree.Children = append(xTree.Children, ReTreeNodeType{Item: c, LR_Tok: LR_Text})
			//} else {
			// // Bad Idea - mucks up '*' and other processing -  To Simplify Tree needs to be done post-generation with Simp-Rules
			//	ll := len(lr.Tree.Children) - 1
			//	if ll >= 0 && lr.Tree.Children[ll].LR_Tok == LR_Text {
			//		lr.Tree.Children[ll].Item += c
			//	} else {
			//		xTree.Children = append(xTree.Children, ReTreeNodeType{Item: c, LR_Tok: LR_Text})
			//	}
			//}

		case LR_CARROT: // ^		-- BOL		-- If at beginning, or after ( or | then BOL - else just text??
			fallthrough
		case LR_DOLLAR: // $		-- BOL		-- If at end, or just before ) or | the EOL - else just text??
			fallthrough
		case LR_DOT: // .		-- Match any char
			xTree.Children = append(xTree.Children, ReTreeNodeType{Item: c, LR_Tok: w})

		case LR_OP_BR: // {
			if isFirst {
				lr.Warn(fmt.Sprintf("Invalid '%s' at beginning of R.E. assumed to be a text character missing esacape.", c))
				xTree.Children = append(xTree.Children, ReTreeNodeType{Item: c, LR_Tok: LR_Text})
			} else {
				ll := len(xTree.Children) - 1
				tmp := xTree.Children[ll]
				newTree := lr.parseIterator(depth + 1)
				if newTree.Mm == 0 && newTree.Nn == InfiniteIteration {
					ll := len(xTree.Children) - 1
					tmp := xTree.Children[ll]
					com.DbPrintf("parseExpression", "%sAT %s, w=%d %s, ll=%d, xTree=%s tmp=%s\n", pre, com.LF(), w, NameOfLR_TokType(w), ll, com.SVarI(xTree), com.SVarI(tmp))
					xTree.Children[ll] = ReTreeNodeType{Item: "*", LR_Tok: LR_STAR, Children: []ReTreeNodeType{tmp}}
				} else {
					if newTree.Mm > newTree.Nn {
						lr.Error = append(lr.Error, errors.New(fmt.Sprintf("Invalid Range, Start is bigger than end, {%d,%d}, %s", newTree.Mm, newTree.Nn, com.LF())))
					}
					com.DbPrintf("parseExpression", "%sAT %s, w=%d %s, ll=%d, xTree=%s tmp=%s\n", pre, com.LF(), w, NameOfLR_TokType(w), ll, com.SVarI(xTree), com.SVarI(tmp))
					// xTree.Children[ll] = ReTreeNodeType{Item: c, LR_Tok: LR_OP_BR, Children: []ReTreeNodeType{tmp}, Mm: newTree.Mm, Nn: newTree.Nn}
					newTree.Children = []ReTreeNodeType{tmp}
					xTree.Children[ll] = newTree
					// CCL: xTree.Children = append(xTree.Children, lr.parseCCL(depth+1, w)) // xyzzy needs work ---------------------------------------------------
				}
				com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())
			}

		case LR_STAR: // *		-- Error if 1st char, else take prev item from list, star and replace it.
			fallthrough
		case LR_PLUS: // +		-- Error if 1st char
			fallthrough
		case LR_QUEST: // ?		-- Error if 1st char
			if isFirst {
				lr.Warn(fmt.Sprintf("Invalid '%s' at beginning of R.E. assumed to be a text character missing esacape.", c))
				xTree.Children = append(xTree.Children, ReTreeNodeType{Item: c, LR_Tok: LR_Text})
			} else {
				ll := len(xTree.Children) - 1
				tmp := xTree.Children[ll]
				com.DbPrintf("parseExpression", "%sAT %s, w=%d %s, ll=%d, xTree=%s tmp=%s\n", pre, com.LF(), w, NameOfLR_TokType(w), ll, com.SVarI(xTree), com.SVarI(tmp))
				xTree.Children[ll] = ReTreeNodeType{Item: c, LR_Tok: w, Children: []ReTreeNodeType{tmp}}
				com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())
			}

		case LR_OR: // |		n-ary or operator
			com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())
			inOr = true

			// Left Machine is collected to be sub-machine == Beginnig-to-current
			// left := xTree.Children // change to be left section back to but not including "|" node - or all if no | node.
			kk := -1
			for jj := len(xTree.Children) - 1; jj >= 0; jj-- {
				if xTree.Children[jj].LR_Tok == LR_OR {
					kk = jj
					break
				}
			}
			if kk == -1 { // No OR tok found
				left := xTree.Children // change to be left section back to but not including "|" node - or all if no | node.
				ll := len(left)
				leftNode := ReTreeNodeType{Item: "", LR_Tok: LR_null, Children: make([]ReTreeNodeType, ll, ll)}
				for jj := range left {
					leftNode.Children[jj] = left[jj]
				}

				newTop := ReTreeNodeType{Item: "|", LR_Tok: LR_OR, Children: make([]ReTreeNodeType, 0, 10)}
				newTop.Children = append(newTop.Children, leftNode) // only if no "or" node, else ref to "or" node
				xTree.Children = xTree.Children[:0]
				xTree.Children = append(xTree.Children, newTop)
				com.DbPrintf("parseExpression", "%sAT %s, w=%d %s, left=%s\n", pre, com.LF(), w, NameOfLR_TokType(w), com.SVarI(left))
			} else {
				if kk >= 0 {
					if kk < len(xTree.Children) {
						tmp := xTree.Children[kk+1:]
						xTree.Children = xTree.Children[0 : kk+1]
						newNode := ReTreeNodeType{Item: "", LR_Tok: LR_null, Children: make([]ReTreeNodeType, len(tmp), len(tmp))}
						for i := 0; i < len(tmp); i++ {
							newNode.Children[i] = tmp[i]
						}
						xTree.Children[kk].Children = append(xTree.Children[kk].Children, newNode)
					}
				}
			}

			// Or node is created like (LR_STAR)
			// Recursive call to parse rest of items at this level
			//newNode := ReTreeNodeType{Item: "", LR_Tok: LR_null, Children: make([]ReTreeNodeType, 1, 10)} // No recursive call
			//lr.parseExpression(depth+1, depth, &newNode.Children[0])
			//newTop.Children = append(newTop.Children, newNode.Children[0])

			// Take results of recursion and put in as RIGHT machine under LR_OR (optimize for N-Tree OR at this point)
			//xTree.Children = append(xTree.Children, newTop)
			//if depth > d_depth {
			//com.DbPrintf("parseExpression", "%sat %s, depth=%d d_detph=%d\n", pre, com.LF(), depth, d_depth)
			//	return xTree.Children
			//}

		case LR_OP_PAR: // (		-- Start of Sub_Re
			com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())

			newNode := ReTreeNodeType{Item: c, LR_Tok: LR_OP_PAR, Children: make([]ReTreeNodeType, 1, 10)}

			lr.parseExpression(depth+1, depth+1, &newNode.Children[0])

			newNode.Children[0].Item = c
			newNode.Children[0].LR_Tok = LR_OP_PAR

			xTree.Children = append(xTree.Children, newNode.Children[0])

			com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())

		case LR_CL_PAR: // )
			// If in "or" node set - then collect last section to "or" ------------------------ <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
			com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())
			if depth == 0 {
				com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())
				lr.Warn(fmt.Sprintf("Invalid '%s' at not properly nested.   Assuming that this was to match a character.", c))
				xTree.Children = append(xTree.Children, ReTreeNodeType{Item: c, LR_Tok: LR_Text})
				com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())
			} else {
				com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())
				if inOr {
					com.DbPrintf("parseExpression", "%sAT Top of new code %s, BOTTOM xTree=%s\n", pre, com.LF(), com.SVarI(xTree))
					kk := -1
					for jj := len(xTree.Children) - 1; jj >= 0; jj-- {
						if xTree.Children[jj].LR_Tok == LR_OR {
							kk = jj
							break
						}
					}
					if kk >= 0 {
						if kk < len(xTree.Children) {
							tmp := xTree.Children[kk+1:]
							xTree.Children = xTree.Children[0 : kk+1]
							newNode := ReTreeNodeType{Item: "", LR_Tok: LR_null, Children: make([]ReTreeNodeType, len(tmp), len(tmp))}
							for i := 0; i < len(tmp); i++ {
								newNode.Children[i] = tmp[i]
							}
							xTree.Children[kk].Children = append(xTree.Children[kk].Children, newNode)
						}
					}
					com.DbPrintf("parseExpression", "%sAT Bo5 of new code %s, BOTTOM xTree=%s\n", pre, com.LF(), com.SVarI(xTree))
				}
				return xTree.Children
			}
			com.DbPrintf("parseExpression", "%sat %s\n", pre, com.LF())
			inOr = false

		case LR_CCL: // [...]	-- CCL Node (Above)
			fallthrough
		case LR_N_CCL: // [^...]	-- N_CCL Node
			xTree.Children = append(xTree.Children, lr.parseCCL(depth+1, w)) // xyzzy needs work ---------------------------------------------------

		default:
			lr.Error = append(lr.Error, errors.New(fmt.Sprintf("Invalid LR Token Type, '%d', '%s', %s", w, NameOfLR_TokType(w), com.LF())))
			return xTree.Children
		}
		isFirst = false
		com.DbPrintf("parseExpression", "%sAT %s, BOTTOM xTree=%s\n", pre, com.LF(), com.SVarI(xTree))
		c, w = lr.Next()
	}
	// If in "or" node set - then collect last section to "or" ------------------------ <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
	if inOr {
		com.DbPrintf("parseExpression", "%sAT Top of new code %s, BOTTOM xTree=%s\n", pre, com.LF(), com.SVarI(xTree))
		kk := -1
		for jj := len(xTree.Children) - 1; jj >= 0; jj-- {
			if xTree.Children[jj].LR_Tok == LR_OR {
				kk = jj
				break
			}
		}
		if kk >= 0 {
			if kk < len(xTree.Children) {
				tmp := xTree.Children[kk+1:]
				xTree.Children = xTree.Children[0 : kk+1]
				newNode := ReTreeNodeType{Item: "", LR_Tok: LR_null, Children: make([]ReTreeNodeType, len(tmp), len(tmp))}
				for i := 0; i < len(tmp); i++ {
					newNode.Children[i] = tmp[i]
				}
				xTree.Children[kk].Children = append(xTree.Children[kk].Children, newNode)
			}
		}
		com.DbPrintf("parseExpression", "%sAT Bo5 of new code %s, BOTTOM xTree=%s\n", pre, com.LF(), com.SVarI(xTree))
	}
	return xTree.Children
}
Example #27
0
File: re.go Project: pschlump/lexie
func (lr *LexReType) CalcLength() (int, bool) {
	x, h := lr.CalcLengthChild(lr.Tree, 1)
	com.DbPrintf("CalcLength", "CalcLength Final Value for Tree = %d, hard=%v\n", x, h)
	return x, h
}
Example #28
0
func (dfa *DFA_PoolType) ConvNDA_to_DFA(nn *nfa.NFA_PoolType) {
	StartState := nn.InitState
	dfa.NoneVisited()

	nn.Sigma = nn.GenerateSigma()
	dfa.Sigma = nn.Sigma
	com.DbPrintf("dfa2", "Sigma at top ->%s<-\n", dfa.Sigma)

	// Build initial state
	dfa_set := nn.LambdaClosure([]int{StartState}) // Find all the lambda closures from specified state
	dfa_set = append(dfa_set, StartState)          // Add in initial state
	dfa_set = com.USortIntSlice(dfa_set)           // Make set unique
	com.DbPrintf("db_DFAGen", "\nStart: %s, \u03a3 =->%s<-, %s\n", com.SVar(dfa_set), dfa.Sigma, com.LF())
	A := dfa.GetDFAName(dfa_set)
	if r, is, info, Is0Ch := nn.IsTerminalState(dfa_set); is {
		dfa.Pool[A].Rv = r
		dfa.Pool[A].Is0Ch = Is0Ch
		dfa.Pool[A].Info = info
	} else {
		dfa.Pool[A].Info = info
	}
	if com.DbOn("db_DFAGen") {
		dfa.DumpPool(false)
	}
	// Look at all the locaitons we can get to from this "state"
	for _, S := range dfa.Sigma {
		StateSet := nn.LambdaClosureSet(dfa_set, string(S))
		com.DbPrintf("db_DFAGen", "FOR INITIAL state ->%s<- StateSet=%s, %s\n", string(S), com.SVar(StateSet), com.LF())
		if len(StateSet) > 0 {
			com.DbPrintf("db_DFAGen", "Have a non-empty result, %s\n", com.LF())
			com.DbPrintf("db_DFAGen", "<><><> this is the point where we should check to see if 'S' is DOT or NCCL, %s\n", com.LF())

			StateSetT := nn.LambdaClosure(StateSet) // need to lambda complete the state set
			StateSet = append(StateSet, StateSetT...)
			StateSet = com.USortIntSlice(StateSet) // Make set unique
			com.DbPrintf("db_DFAGen", "    Output Is %s, %s\n", com.SVar(StateSet), com.LF())
			B := 0
			if t := dfa.HaveStateAlready(StateSet); t != -1 { // Have Already
				B = t
				com.DbPrintf("db_DFAGen", "    Already have this state at location %d, %s\n", t, com.LF())
			} else {
				B = dfa.GetDFAName(StateSet)
				com.DbPrintf("db_DFAGen", "    *** New state %d, %s\n", B, com.LF())
			}
			dfa.AddEdge(A, B, string(S))
			com.DbPrintf("db_DFAGen", "    *** Before (top) %s\n", com.LF())
			if r, is, info, Is0Ch := nn.IsTerminalState(StateSet); is {
				dfa.Pool[B].Rv = r
				dfa.Pool[B].Is0Ch = Is0Ch
				dfa.Pool[B].Info = info
				com.DbPrintf("db_DFAGen", "    *** New state %d, %s\n", B, com.LF())
			} else if _, is, info, Is0Ch := nn.IsNonTerminalPushPopState(StateSet); is {
				dfa.Pool[B].Is0Ch = Is0Ch
				dfa.Pool[B].Info = info
				com.DbPrintf("db_DFAGen", "    *** New info for state %d, %s\n", B, com.LF())
			} else {
				dfa.Pool[B].Info = info
				com.DbPrintf("db_DFAGen", "    *** NO State Info for state %d, %s\n", B, com.LF())
			}
			com.DbPrintf("db_DFAGen", "    *** After (top) %s\n", com.LF())
			if com.DbOn("db_DFAGen") {
				fmt.Printf("for %s StateSet=%s, A=%d, B=%s %s\n", string(S), com.SVar(StateSet), A, com.SVar(B), com.LF())
				dfa.DumpPool(false)
			}
		}
	}
	dfa.Pool[A].Visited = true

	com.DbPrintf("db_DFAGen", "\nBefore Main Loop, %s\n", com.LF())
	limit := 0
	for stateToDo := dfa.FindNotVisited(); stateToDo != -1; stateToDo = dfa.FindNotVisited() {
		com.DbPrintf("db_DFAGen", "\nMain Loop: !!TOP!! State:%d\n", stateToDo)
		// -----------------------------------------------------------------------------------------------------------
		if !dfa.Pool[stateToDo].Visited {
			dfa_set := nn.LambdaClosure(dfa.Pool[stateToDo].StateSet)  // Find all the lambda closures from specified state
			dfa_set = append(dfa_set, dfa.Pool[stateToDo].StateSet...) // Add in initial state
			dfa_set = com.USortIntSlice(dfa_set)                       // Make set unique
			for _, S := range dfa.Sigma {
				StateSet := nn.LambdaClosureSet(dfa_set, string(S))
				com.DbPrintf("db_DFAGen", "    for initial state %s StateSet=%s, %s\n", string(S), com.SVar(StateSet), com.LF())
				com.DbPrintf("db_DFAGen", "<><><> this is the point where we should check to see if 'S' is DOT or NCCL, %s\n", com.LF())
				if len(StateSet) > 0 {
					com.DbPrintf("db_DFAGen", "    >>> Have a non-empty result, Input Is %s, %s\n", com.SVar(StateSet), com.LF())
					StateSetT := nn.LambdaClosure(StateSet) // need to lambda complete the state set
					StateSet = append(StateSet, StateSetT...)
					StateSet = com.USortIntSlice(StateSet) // Make set unique
					com.DbPrintf("db_DFAGen", "    >>> Output Is %s, %s\n", com.SVar(StateSet), com.LF())
					B := 0
					if t := dfa.HaveStateAlready(StateSet); t != -1 { // Have Already
						B = t
						com.DbPrintf("db_DFAGen", "    Already have this state at location %d, %s\n", t, com.LF())
					} else {
						B = dfa.GetDFAName(StateSet)
					}
					dfa.AddEdge(stateToDo, B, string(S))
					com.DbPrintf("db_DFAGen", "    *** Before %s\n", com.LF())
					if r, is, info, Is0Ch := nn.IsTerminalState(StateSet); is {
						dfa.Pool[B].Rv = r
						dfa.Pool[B].Is0Ch = Is0Ch
						dfa.Pool[B].Info = info
						com.DbPrintf("db_DFAGen", "    *** New state %d, %s\n", B, com.LF())
					} else if _, is, info, Is0Ch := nn.IsNonTerminalPushPopState(StateSet); is {
						dfa.Pool[B].Is0Ch = Is0Ch
						dfa.Pool[B].Info = info
						com.DbPrintf("db_DFAGen", "    *** New info for state %d, %s\n", B, com.LF())
					} else {
						com.DbPrintf("db_DFAGen", "    *** NO State Info for state %d, %s\n", B, com.LF())
						dfa.Pool[B].Info = info
					}
					com.DbPrintf("db_DFAGen", "    *** After %s\n", com.LF())
					if com.DbOn("db_DFAGen") {
						fmt.Printf("    Add New Edge on %s fr %d to %d, %s\n", string(S), stateToDo, B, com.LF())
						fmt.Printf("    for %s StateSet=%s, A(stateToDo)=%d, %s\n", string(S), com.SVar(StateSet), stateToDo, com.LF())
						dfa.DumpPool(false)
					}
				}
			}
		}
		// -----------------------------------------------------------------------------------------------------------
		dfa.Pool[stateToDo].Visited = true
		limit++
		if limit > 50000 {
			break
		}
	}
	dfa.VerifyMachine()

}
Example #29
0
// -----------------------------------------------------------------------------------------------------------------------------------------------
func (dfa *DFA_PoolType) ConvertToTable() (rv dfaTable) {
	rv.InitState = dfa.InitState
	rv.SMap = smap.NewSMapType(dfa.Sigma, re.R_not_CH)
	rv.N_States = dfa.NumberOfStates()
	rv.Machine = make([]*MachineStatesType, rv.N_States, rv.N_States)
	XLen := rv.SMap.Length()
	rv.Width = XLen

	jj := 0
	for ii, vv := range dfa.Pool {
		if vv.IsUsed {
			rv.Machine[jj] = &MachineStatesType{Rv: vv.Rv, Info: vv.Info, Tau: vv.Is0Ch}
			rv.Machine[jj].To = make([]int, XLen, XLen)
			for kk := 0; kk < XLen; kk++ {
				rv.Machine[jj].To[kk] = -1
			}
			dot := false
			dotTo := 0
			alpha := false
			alphaTo := 0
			num := false
			numTo := 0
			for _, ww := range vv.Next2 {
				//if s == re.X_DOT {
				// return "\u2022"	// Middle Bullet
				if ww.On == re.X_DOT {
					// fmt.Printf("Found a dot, at jj=%d\n", jj)
					dot = true
					dotTo = ww.To
				} else if ww.On == re.X_ALPHA {
					// fmt.Printf("Found a alpha, at jj=%d\n", jj)
					alpha = true
					alphaTo = ww.To
					rr, _ := utf8.DecodeRune([]byte(ww.On))
					xx := rv.SMap.MapRune(rr)
					rv.Machine[jj].To[xx] = ww.To
				} else if ww.On == re.X_NUMERIC {
					// com.DbPrintf("dfa2", "Found a numeric, at jj=%d\n", jj)
					num = true
					numTo = ww.To
					rr, _ := utf8.DecodeRune([]byte(ww.On))
					xx := rv.SMap.MapRune(rr)
					rv.Machine[jj].To[xx] = ww.To
				} else {
					rr, _ := utf8.DecodeRune([]byte(ww.On))
					xx := rv.SMap.MapRune(rr)
					rv.Machine[jj].To[xx] = ww.To
				}
			}
			if num {
				for ii := 0; ii < rv.Width; ii++ {
					// rn := rune(ii + rv.SMap.MinV)
					rn := rv.SMap.SigmaRN[ii]
					// func (smap *SMapType) ReverseMapRune(x int) rune {
					// com.DbPrintf("dfa2", "numeric: ->%s<-\n", string(rn))
					if rv.Machine[jj].To[ii] == -1 && unicode.IsDigit(rn) {
						rv.Machine[jj].To[ii] = numTo
					}
				}
			}
			if alpha {
				for ii := 0; ii < rv.Width; ii++ {
					// rn := rune(ii + rv.SMap.MinV)
					rn := rv.SMap.SigmaRN[ii]
					if rv.Machine[jj].To[ii] == -1 && (unicode.IsUpper(rn) || unicode.IsLower(rn)) {
						rv.Machine[jj].To[ii] = alphaTo
					}
				}
			}
			if dot {
				for ii := 0; ii < rv.Width; ii++ {
					if rv.Machine[jj].To[ii] == -1 {
						rv.Machine[jj].To[ii] = dotTo
					}
				}
			}
			if ((vv.Info.Action&com.A_Pop) != 0 || (vv.Info.Action&com.A_Push) != 0 || (vv.Info.Action&com.A_Reset) != 0) && !vv.Info.HardMatch { // xyzzy8
				com.DbPrintf("dfa7", "Info2-in-TabGen: State[%d] a Push/Call or a Pop/Return/Reset optration Must be a terminal state with fixed string matched, 'a*' or 'a?' is not a legitimate match.\n", ii)
				//for ii := 0; ii < rv.Width; ii++ {
				//	if rv.Machine[jj].To[ii] == -1 {
				//		rv.Machine[jj].To[ii] = dotTo
				//	}
				//}
			}
			jj++
		}
	}
	return
}
Example #30
0
File: re.go Project: pschlump/lexie
func (lr *LexReType) CalcLengthChild(tree *ReTreeNodeType, d int) (x int, hard bool) {
	t := 0

	hard = false
	if d == 1 {
		com.DbPrintf("CalcLength", "CalcLengthChild at top: %s\n\n", com.SVarI(tree))
	}

	switch tree.LR_Tok {
	case LR_null: //
		for jj := range tree.Children {
			t, hard = lr.CalcLengthChild(&tree.Children[jj], d+1)
			x += t
		}
	case LR_Text: //
		// com.DbPrintf("CalcLength", "Len of item(%s) = %d, %s\n", tree.Item, len(tree.Item), com.LF())
		x += len(tree.Item)
		hard = true
	case LR_EOF: //
		hard = true
	case LR_DOT: // .
		x += 1
	case LR_STAR: // *
		x = 0
		// com.DbPrintf("CalcLength", "After * x = %d, hard=%v\n", x, hard)
	case LR_PLUS: // +
		// patch to fix the problem with [0-9]+ not working -- In reality the length is only if it is a "FIXED" length, 0 else
		//		if len(tree.Children) > 0 {
		//			t, hard = lr.CalcLengthChild(&tree.Children[0], d+1)
		//			x += t
		//		}
		//		hard = true
		x = 0
	case LR_QUEST: // ?
		x = 0
	case LR_OP_BR: // { 			// {m,n} - need to calculate length of ( m times, length of children
		x = 0
	case LR_OP_PAR: // (
		if len(tree.Children) > 0 {
			t, hard = lr.CalcLengthChild(&tree.Children[0], d+1)
			x += t
		}
		// com.DbPrintf("CalcLength", "After ( x = %d, hard=%v\n", x, hard)
	case LR_CL_PAR: // )
		x = 0
	case LR_CCL: // [...]
		x += 1
		hard = true
	case LR_N_CCL: // [^...]
		x += 1
	case LR_E_CCL: // ]
		x += 1
	case LR_CARROT: // ^
		x += 0
		hard = true
	case LR_MINUS: // -
		x += 1
		hard = true
	case LR_DOLLAR: // $
		hard = true

	case LR_OR: // |
		y := -1
		z := 0
		hard = false
		if len(tree.Children) > 0 {
			hard = true
			h := false
			for jj := range tree.Children {
				z, h = lr.CalcLengthChild(&tree.Children[jj], d+1)
				if y == -1 {
					y = z
				} else if y < z {
					y = z
				}
				if !h {
					hard = false
				}
			}
		}
		x += y
		// com.DbPrintf("CalcLength", "After | x = %d, hard = %v\n", x, hard)
	}

	return
}