func xfol(ctxt *liblink.Link, p *liblink.Prog, last **liblink.Prog) { var q *liblink.Prog var i int var a int loop: if p == nil { return } if p.As == AJMP { q = p.Pcond if q != nil && q.As != ATEXT { /* mark instruction as done and continue layout at target of jump */ p.Mark = 1 p = q if p.Mark == 0 { goto loop } } } if p.Mark != 0 { /* * p goes here, but already used it elsewhere. * copy up to 4 instructions or else branch to other copy. */ i = 0 q = p for ; i < 4; (func() { i++; q = q.Link })() { if q == nil { break } if q == *last { break } a = int(q.As) if a == ANOP { i-- continue } if nofollow(a) != 0 || pushpop(a) != 0 { break // NOTE(rsc): arm does goto copy } if q.Pcond == nil || q.Pcond.Mark != 0 { continue } if a == ACALL || a == ALOOP { continue } for { if p.As == ANOP { p = p.Link continue } q = liblink.Copyp(ctxt, p) p = p.Link q.Mark = 1 (*last).Link = q *last = q if int(q.As) != a || q.Pcond == nil || q.Pcond.Mark != 0 { continue } q.As = int16(relinv(int(q.As))) p = q.Pcond q.Pcond = q.Link q.Link = p xfol(ctxt, q.Link, last) p = q.Link if p.Mark != 0 { return } goto loop /* */ } } q = ctxt.Arch.Prg() q.As = AJMP q.Lineno = p.Lineno q.To.Type_ = D_BRANCH q.To.Offset = p.Pc q.Pcond = p p = q } /* emit p */ p.Mark = 1 (*last).Link = p *last = p a = int(p.As) /* continue loop with what comes after p */ if nofollow(a) != 0 { return } if p.Pcond != nil && a != ACALL { /* * some kind of conditional branch. * recurse to follow one path. * continue loop on the other. */ q = liblink.Brchain(ctxt, p.Pcond) if q != nil { p.Pcond = q } q = liblink.Brchain(ctxt, p.Link) if q != nil { p.Link = q } if p.From.Type_ == D_CONST { if p.From.Offset == 1 { /* * expect conditional jump to be taken. * rewrite so that's the fall-through case. */ p.As = int16(relinv(a)) q = p.Link p.Link = p.Pcond p.Pcond = q } } else { q = p.Link if q.Mark != 0 { if a != ALOOP { p.As = int16(relinv(a)) p.Link = p.Pcond p.Pcond = q } } } xfol(ctxt, p.Link, last) if p.Pcond.Mark != 0 { return } p = p.Pcond goto loop } p = p.Link goto loop }
func xfol(ctxt *liblink.Link, p *liblink.Prog, last **liblink.Prog) { var q *liblink.Prog var r *liblink.Prog var a int var i int loop: if p == nil { return } a = int(p.As) if a == AB { q = p.Pcond if q != nil && q.As != ATEXT { p.Mark |= FOLL p = q if !(p.Mark&FOLL != 0) { goto loop } } } if p.Mark&FOLL != 0 { i = 0 q = p for ; i < 4; (func() { i++; q = q.Link })() { if q == *last || q == nil { break } a = int(q.As) if a == ANOP { i-- continue } if a == AB || (a == ARET && q.Scond == C_SCOND_NONE) || a == ARFE || a == AUNDEF { goto copy } if q.Pcond == nil || (q.Pcond.Mark&FOLL != 0) { continue } if a != ABEQ && a != ABNE { continue } copy: for { r = ctxt.Arch.Prg() *r = *p if !(r.Mark&FOLL != 0) { fmt.Printf("can't happen 1\n") } r.Mark |= FOLL if p != q { p = p.Link (*last).Link = r *last = r continue } (*last).Link = r *last = r if a == AB || (a == ARET && q.Scond == C_SCOND_NONE) || a == ARFE || a == AUNDEF { return } r.As = ABNE if a == ABNE { r.As = ABEQ } r.Pcond = p.Link r.Link = p.Pcond if !(r.Link.Mark&FOLL != 0) { xfol(ctxt, r.Link, last) } if !(r.Pcond.Mark&FOLL != 0) { fmt.Printf("can't happen 2\n") } return } } a = AB q = ctxt.Arch.Prg() q.As = int16(a) q.Lineno = p.Lineno q.To.Type_ = D_BRANCH q.To.Offset = p.Pc q.Pcond = p p = q } p.Mark |= FOLL (*last).Link = p *last = p if a == AB || (a == ARET && p.Scond == C_SCOND_NONE) || a == ARFE || a == AUNDEF { return } if p.Pcond != nil { if a != ABL && a != ABX && p.Link != nil { q = liblink.Brchain(ctxt, p.Link) if a != ATEXT && a != ABCASE { if q != nil && (q.Mark&FOLL != 0) { p.As = int16(relinv(a)) p.Link = p.Pcond p.Pcond = q } } xfol(ctxt, p.Link, last) q = liblink.Brchain(ctxt, p.Pcond) if q == nil { q = p.Pcond } if q.Mark&FOLL != 0 { p.Pcond = q return } p = q goto loop } } p = p.Link goto loop }
func addstacksplit(ctxt *liblink.Link, cursym *liblink.LSym) { var p *liblink.Prog var q *liblink.Prog var p1 *liblink.Prog var p2 *liblink.Prog var autoffset int32 var deltasp int32 var a int var pcsize int var textstksiz int64 var textarg int64 if ctxt.Tlsg == nil { ctxt.Tlsg = liblink.Linklookup(ctxt, "runtime.tlsg", 0) } if ctxt.Symmorestack[0] == nil { ctxt.Symmorestack[0] = liblink.Linklookup(ctxt, "runtime.morestack", 0) ctxt.Symmorestack[1] = liblink.Linklookup(ctxt, "runtime.morestack_noctxt", 0) } if ctxt.Headtype == liblink.Hplan9 && ctxt.Plan9privates == nil { ctxt.Plan9privates = liblink.Linklookup(ctxt, "_privates", 0) } ctxt.Cursym = cursym if cursym.Text == nil || cursym.Text.Link == nil { return } p = cursym.Text parsetextconst(p.To.Offset, &textstksiz, &textarg) autoffset = int32(textstksiz) if autoffset < 0 { autoffset = 0 } cursym.Args = int32(p.To.Offset >> 32) cursym.Locals = int32(textstksiz) if autoffset < liblink.StackSmall && !(p.From.Scale&liblink.NOSPLIT != 0) { for q = p; q != nil; q = q.Link { if q.As == ACALL { goto noleaf } if (q.As == ADUFFCOPY || q.As == ADUFFZERO) && autoffset >= liblink.StackSmall-8 { goto noleaf } } p.From.Scale |= liblink.NOSPLIT noleaf: } q = nil if !(p.From.Scale&liblink.NOSPLIT != 0) || (p.From.Scale&liblink.WRAPPER != 0) { p = liblink.Appendp(ctxt, p) p = load_g_cx(ctxt, p) // load g into CX } if !(cursym.Text.From.Scale&liblink.NOSPLIT != 0) { p = stacksplit(ctxt, p, autoffset, int32(textarg), bool2int(!(cursym.Text.From.Scale&liblink.NEEDCTXT != 0)), &q) // emit split check } if autoffset != 0 { if autoffset%int32(ctxt.Arch.Regsize) != 0 { ctxt.Diag("unaligned stack size %d", autoffset) } p = liblink.Appendp(ctxt, p) p.As = AADJSP p.From.Type_ = D_CONST p.From.Offset = int64(autoffset) p.Spadj = autoffset } else { // zero-byte stack adjustment. // Insert a fake non-zero adjustment so that stkcheck can // recognize the end of the stack-splitting prolog. p = liblink.Appendp(ctxt, p) p.As = ANOP p.Spadj = int32(-ctxt.Arch.Ptrsize) p = liblink.Appendp(ctxt, p) p.As = ANOP p.Spadj = int32(ctxt.Arch.Ptrsize) } if q != nil { q.Pcond = p } deltasp = autoffset if cursym.Text.From.Scale&liblink.WRAPPER != 0 { // if(g->panic != nil && g->panic->argp == FP) g->panic->argp = bottom-of-frame // // MOVQ g_panic(CX), BX // TESTQ BX, BX // JEQ end // LEAQ (autoffset+8)(SP), DI // CMPQ panic_argp(BX), DI // JNE end // MOVQ SP, panic_argp(BX) // end: // NOP // // The NOP is needed to give the jumps somewhere to land. // It is a liblink NOP, not an x86 NOP: it encodes to 0 instruction bytes. p = liblink.Appendp(ctxt, p) p.As = AMOVQ p.From.Type_ = D_INDIR + D_CX p.From.Offset = 4 * int64(ctxt.Arch.Ptrsize) // G.panic p.To.Type_ = D_BX if ctxt.Headtype == liblink.Hnacl { p.As = AMOVL p.From.Type_ = D_INDIR + D_R15 p.From.Scale = 1 p.From.Index = D_CX } p = liblink.Appendp(ctxt, p) p.As = ATESTQ p.From.Type_ = D_BX p.To.Type_ = D_BX if ctxt.Headtype == liblink.Hnacl { p.As = ATESTL } p = liblink.Appendp(ctxt, p) p.As = AJEQ p.To.Type_ = D_BRANCH p1 = p p = liblink.Appendp(ctxt, p) p.As = ALEAQ p.From.Type_ = D_INDIR + D_SP p.From.Offset = int64(autoffset) + 8 p.To.Type_ = D_DI if ctxt.Headtype == liblink.Hnacl { p.As = ALEAL } p = liblink.Appendp(ctxt, p) p.As = ACMPQ p.From.Type_ = D_INDIR + D_BX p.From.Offset = 0 // Panic.argp p.To.Type_ = D_DI if ctxt.Headtype == liblink.Hnacl { p.As = ACMPL p.From.Type_ = D_INDIR + D_R15 p.From.Scale = 1 p.From.Index = D_BX } p = liblink.Appendp(ctxt, p) p.As = AJNE p.To.Type_ = D_BRANCH p2 = p p = liblink.Appendp(ctxt, p) p.As = AMOVQ p.From.Type_ = D_SP p.To.Type_ = D_INDIR + D_BX p.To.Offset = 0 // Panic.argp if ctxt.Headtype == liblink.Hnacl { p.As = AMOVL p.To.Type_ = D_INDIR + D_R15 p.To.Scale = 1 p.To.Index = D_BX } p = liblink.Appendp(ctxt, p) p.As = ANOP p1.Pcond = p p2.Pcond = p } if ctxt.Debugzerostack != 0 && autoffset != 0 && !(cursym.Text.From.Scale&liblink.NOSPLIT != 0) { // 6l -Z means zero the stack frame on entry. // This slows down function calls but can help avoid // false positives in garbage collection. p = liblink.Appendp(ctxt, p) p.As = AMOVQ p.From.Type_ = D_SP p.To.Type_ = D_DI p = liblink.Appendp(ctxt, p) p.As = AMOVQ p.From.Type_ = D_CONST p.From.Offset = int64(autoffset) / 8 p.To.Type_ = D_CX p = liblink.Appendp(ctxt, p) p.As = AMOVQ p.From.Type_ = D_CONST p.From.Offset = 0 p.To.Type_ = D_AX p = liblink.Appendp(ctxt, p) p.As = AREP p = liblink.Appendp(ctxt, p) p.As = ASTOSQ } for ; p != nil; p = p.Link { pcsize = int(p.Mode) / 8 a = int(p.From.Type_) if a == D_AUTO { p.From.Offset += int64(deltasp) } if a == D_PARAM { p.From.Offset += int64(deltasp) + int64(pcsize) } a = int(p.To.Type_) if a == D_AUTO { p.To.Offset += int64(deltasp) } if a == D_PARAM { p.To.Offset += int64(deltasp) + int64(pcsize) } switch p.As { default: continue case APUSHL, APUSHFL: deltasp += 4 p.Spadj = 4 continue case APUSHQ, APUSHFQ: deltasp += 8 p.Spadj = 8 continue case APUSHW, APUSHFW: deltasp += 2 p.Spadj = 2 continue case APOPL, APOPFL: deltasp -= 4 p.Spadj = -4 continue case APOPQ, APOPFQ: deltasp -= 8 p.Spadj = -8 continue case APOPW, APOPFW: deltasp -= 2 p.Spadj = -2 continue case ARET: break } if autoffset != deltasp { ctxt.Diag("unbalanced PUSH/POP") } if autoffset != 0 { p.As = AADJSP p.From.Type_ = D_CONST p.From.Offset = int64(-autoffset) p.Spadj = -autoffset p = liblink.Appendp(ctxt, p) p.As = ARET // If there are instructions following // this ARET, they come from a branch // with the same stackframe, so undo // the cleanup. p.Spadj = +autoffset } if p.To.Sym != nil { // retjmp p.As = AJMP } } } func indir_cx(ctxt *liblink.Link, a *liblink.Addr) { if ctxt.Headtype == liblink.Hnacl { a.Type_ = D_INDIR + D_R15 a.Index = D_CX a.Scale = 1 return } a.Type_ = D_INDIR + D_CX } // Append code to p to load g into cx. // Overwrites p with the first instruction (no first appendp). // Overwriting p is unusual but it lets use this in both the // prologue (caller must call appendp first) and in the epilogue. // Returns last new instruction. func load_g_cx(ctxt *liblink.Link, p *liblink.Prog) *liblink.Prog { var next *liblink.Prog p.As = AMOVQ if ctxt.Arch.Ptrsize == 4 { p.As = AMOVL } p.From.Type_ = D_INDIR + D_TLS p.From.Offset = 0 p.To.Type_ = D_CX next = p.Link progedit(ctxt, p) for p.Link != next { p = p.Link } if p.From.Index == D_TLS { p.From.Scale = 2 } return p } // Append code to p to check for stack split. // Appends to (does not overwrite) p. // Assumes g is in CX. // Returns last new instruction. // On return, *jmpok is the instruction that should jump // to the stack frame allocation if no split is needed. func stacksplit(ctxt *liblink.Link, p *liblink.Prog, framesize int32, textarg int32, noctxt int, jmpok **liblink.Prog) *liblink.Prog { var q *liblink.Prog var q1 *liblink.Prog var cmp int var lea int var mov int var sub int cmp = ACMPQ lea = ALEAQ mov = AMOVQ sub = ASUBQ if ctxt.Headtype == liblink.Hnacl { cmp = ACMPL lea = ALEAL mov = AMOVL sub = ASUBL } q1 = nil if framesize <= liblink.StackSmall { // small stack: SP <= stackguard // CMPQ SP, stackguard p = liblink.Appendp(ctxt, p) p.As = int16(cmp) p.From.Type_ = D_SP indir_cx(ctxt, &p.To) p.To.Offset = 2 * int64(ctxt.Arch.Ptrsize) // G.stackguard0 if ctxt.Cursym.Cfunc != 0 { p.To.Offset = 3 * int64(ctxt.Arch.Ptrsize) // G.stackguard1 } } else if framesize <= liblink.StackBig { // large stack: SP-framesize <= stackguard-StackSmall // LEAQ -xxx(SP), AX // CMPQ AX, stackguard p = liblink.Appendp(ctxt, p) p.As = int16(lea) p.From.Type_ = D_INDIR + D_SP p.From.Offset = -(int64(framesize) - liblink.StackSmall) p.To.Type_ = D_AX p = liblink.Appendp(ctxt, p) p.As = int16(cmp) p.From.Type_ = D_AX indir_cx(ctxt, &p.To) p.To.Offset = 2 * int64(ctxt.Arch.Ptrsize) // G.stackguard0 if ctxt.Cursym.Cfunc != 0 { p.To.Offset = 3 * int64(ctxt.Arch.Ptrsize) // G.stackguard1 } } else { // Such a large stack we need to protect against wraparound. // If SP is close to zero: // SP-stackguard+StackGuard <= framesize + (StackGuard-StackSmall) // The +StackGuard on both sides is required to keep the left side positive: // SP is allowed to be slightly below stackguard. See stack.h. // // Preemption sets stackguard to StackPreempt, a very large value. // That breaks the math above, so we have to check for that explicitly. // MOVQ stackguard, CX // CMPQ CX, $StackPreempt // JEQ label-of-call-to-morestack // LEAQ StackGuard(SP), AX // SUBQ CX, AX // CMPQ AX, $(framesize+(StackGuard-StackSmall)) p = liblink.Appendp(ctxt, p) p.As = int16(mov) indir_cx(ctxt, &p.From) p.From.Offset = 2 * int64(ctxt.Arch.Ptrsize) // G.stackguard0 if ctxt.Cursym.Cfunc != 0 { p.From.Offset = 3 * int64(ctxt.Arch.Ptrsize) // G.stackguard1 } p.To.Type_ = D_SI p = liblink.Appendp(ctxt, p) p.As = int16(cmp) p.From.Type_ = D_SI p.To.Type_ = D_CONST p.To.Offset = liblink.StackPreempt p = liblink.Appendp(ctxt, p) p.As = AJEQ p.To.Type_ = D_BRANCH q1 = p p = liblink.Appendp(ctxt, p) p.As = int16(lea) p.From.Type_ = D_INDIR + D_SP p.From.Offset = liblink.StackGuard p.To.Type_ = D_AX p = liblink.Appendp(ctxt, p) p.As = int16(sub) p.From.Type_ = D_SI p.To.Type_ = D_AX p = liblink.Appendp(ctxt, p) p.As = int16(cmp) p.From.Type_ = D_AX p.To.Type_ = D_CONST p.To.Offset = int64(framesize) + (liblink.StackGuard - liblink.StackSmall) } // common p = liblink.Appendp(ctxt, p) p.As = AJHI p.To.Type_ = D_BRANCH q = p p = liblink.Appendp(ctxt, p) p.As = ACALL p.To.Type_ = D_BRANCH if ctxt.Cursym.Cfunc != 0 { p.To.Sym = liblink.Linklookup(ctxt, "runtime.morestackc", 0) } else { p.To.Sym = ctxt.Symmorestack[noctxt] } p = liblink.Appendp(ctxt, p) p.As = AJMP p.To.Type_ = D_BRANCH p.Pcond = ctxt.Cursym.Text.Link if q != nil { q.Pcond = p.Link } if q1 != nil { q1.Pcond = q.Link } *jmpok = q return p } func follow(ctxt *liblink.Link, s *liblink.LSym) { var firstp *liblink.Prog var lastp *liblink.Prog ctxt.Cursym = s firstp = ctxt.Arch.Prg() lastp = firstp xfol(ctxt, s.Text, &lastp) lastp.Link = nil s.Text = firstp.Link } func nofollow(a int) int { switch a { case AJMP, ARET, AIRETL, AIRETQ, AIRETW, ARETFL, ARETFQ, ARETFW, AUNDEF: return 1 } return 0 } func pushpop(a int) int { switch a { case APUSHL, APUSHFL, APUSHQ, APUSHFQ, APUSHW, APUSHFW, APOPL, APOPFL, APOPQ, APOPFQ, APOPW, APOPFW: return 1 } return 0 } func relinv(a int) int { switch a { case AJEQ: return AJNE case AJNE: return AJEQ case AJLE: return AJGT case AJLS: return AJHI case AJLT: return AJGE case AJMI: return AJPL case AJGE: return AJLT case AJPL: return AJMI case AJGT: return AJLE case AJHI: return AJLS case AJCS: return AJCC case AJCC: return AJCS case AJPS: return AJPC case AJPC: return AJPS case AJOS: return AJOC case AJOC: return AJOS } log.Fatalf("unknown relation: %s", anames6[a]) return 0 } func xfol(ctxt *liblink.Link, p *liblink.Prog, last **liblink.Prog) { var q *liblink.Prog var i int var a int loop: if p == nil { return } if p.As == AJMP { q = p.Pcond if q != nil && q.As != ATEXT { /* mark instruction as done and continue layout at target of jump */ p.Mark = 1 p = q if p.Mark == 0 { goto loop } } } if p.Mark != 0 { /* * p goes here, but already used it elsewhere. * copy up to 4 instructions or else branch to other copy. */ i = 0 q = p for ; i < 4; (func() { i++; q = q.Link })() { if q == nil { break } if q == *last { break } a = int(q.As) if a == ANOP { i-- continue } if nofollow(a) != 0 || pushpop(a) != 0 { break // NOTE(rsc): arm does goto copy } if q.Pcond == nil || q.Pcond.Mark != 0 { continue } if a == ACALL || a == ALOOP { continue } for { if p.As == ANOP { p = p.Link continue } q = liblink.Copyp(ctxt, p) p = p.Link q.Mark = 1 (*last).Link = q *last = q if int(q.As) != a || q.Pcond == nil || q.Pcond.Mark != 0 { continue } q.As = int16(relinv(int(q.As))) p = q.Pcond q.Pcond = q.Link q.Link = p xfol(ctxt, q.Link, last) p = q.Link if p.Mark != 0 { return } goto loop /* */ } } q = ctxt.Arch.Prg() q.As = AJMP q.Lineno = p.Lineno q.To.Type_ = D_BRANCH q.To.Offset = p.Pc q.Pcond = p p = q } /* emit p */ p.Mark = 1 (*last).Link = p *last = p a = int(p.As) /* continue loop with what comes after p */ if nofollow(a) != 0 { return } if p.Pcond != nil && a != ACALL { /* * some kind of conditional branch. * recurse to follow one path. * continue loop on the other. */ q = liblink.Brchain(ctxt, p.Pcond) if q != nil { p.Pcond = q } q = liblink.Brchain(ctxt, p.Link) if q != nil { p.Link = q } if p.From.Type_ == D_CONST { if p.From.Offset == 1 { /* * expect conditional jump to be taken. * rewrite so that's the fall-through case. */ p.As = int16(relinv(a)) q = p.Link p.Link = p.Pcond p.Pcond = q } } else { q = p.Link if q.Mark != 0 { if a != ALOOP { p.As = int16(relinv(a)) p.Link = p.Pcond p.Pcond = q } } } xfol(ctxt, p.Link, last) if p.Pcond.Mark != 0 { return } p = p.Pcond goto loop } p = p.Link goto loop } func prg() *liblink.Prog { var p *liblink.Prog p = new(liblink.Prog) *p = zprg return p } var Linkamd64 = liblink.LinkArch{ ByteOrder: binary.LittleEndian, Pconv: Pconv, Name: "amd64", Thechar: '6', Endian: liblink.LittleEndian, Addstacksplit: addstacksplit, Assemble: span6, Datasize: datasize, Follow: follow, Iscall: iscall, Isdata: isdata, Prg: prg, Progedit: progedit, Settextflag: settextflag, Symtype: symtype, Textflag: textflag, Minlc: 1, Ptrsize: 8, Regsize: 8, D_ADDR: D_ADDR, D_AUTO: D_AUTO, D_BRANCH: D_BRANCH, D_CONST: D_CONST, D_EXTERN: D_EXTERN, D_FCONST: D_FCONST, D_NONE: D_NONE, D_PARAM: D_PARAM, D_SCONST: D_SCONST, D_STATIC: D_STATIC, ACALL: ACALL, ADATA: ADATA, AEND: AEND, AFUNCDATA: AFUNCDATA, AGLOBL: AGLOBL, AJMP: AJMP, ANOP: ANOP, APCDATA: APCDATA, ARET: ARET, ATEXT: ATEXT, ATYPE: ATYPE, AUSEFIELD: AUSEFIELD, } var Linkamd64p32 = liblink.LinkArch{ ByteOrder: binary.LittleEndian, Pconv: Pconv, Name: "amd64p32", Thechar: '6', Endian: liblink.LittleEndian, Addstacksplit: addstacksplit, Assemble: span6, Datasize: datasize, Follow: follow, Iscall: iscall, Isdata: isdata, Prg: prg, Progedit: progedit, Settextflag: settextflag, Symtype: symtype, Textflag: textflag, Minlc: 1, Ptrsize: 4, Regsize: 8, D_ADDR: D_ADDR, D_AUTO: D_AUTO, D_BRANCH: D_BRANCH, D_CONST: D_CONST, D_EXTERN: D_EXTERN, D_FCONST: D_FCONST, D_NONE: D_NONE, D_PARAM: D_PARAM, D_SCONST: D_SCONST, D_STATIC: D_STATIC, ACALL: ACALL, ADATA: ADATA, AEND: AEND, AFUNCDATA: AFUNCDATA, AGLOBL: AGLOBL, AJMP: AJMP, ANOP: ANOP, APCDATA: APCDATA, ARET: ARET, ATEXT: ATEXT, ATYPE: ATYPE, AUSEFIELD: AUSEFIELD, }