func clearfat(nl *gc.Node) {
/* clear a fat object */
if gc.Debug['g'] != 0 {
fmt.Printf("clearfat %v (%v, size: %d)\n", nl, nl.Type, nl.Type.Width)
}
w := uint64(uint64(nl.Type.Width))
// Avoid taking the address for simple enough types.
if gc.Componentgen(nil, nl) {
return
}
c := uint64(w % 8) // bytes
q := uint64(w / 8) // dwords
if gc.Reginuse(ppc64.REGRT1) {
gc.Fatal("%v in use during clearfat", obj.Rconv(ppc64.REGRT1))
}
var r0 gc.Node
gc.Nodreg(&r0, gc.Types[gc.TUINT64], ppc64.REGZERO)
var dst gc.Node
gc.Nodreg(&dst, gc.Types[gc.Tptr], ppc64.REGRT1)
gc.Regrealloc(&dst)
gc.Agen(nl, &dst)
var boff uint64
if q > 128 {
p := gins(ppc64.ASUB, nil, &dst)
p.From.Type = obj.TYPE_CONST
p.From.Offset = 8
var end gc.Node
gc.Regalloc(&end, gc.Types[gc.Tptr], nil)
p = gins(ppc64.AMOVD, &dst, &end)
p.From.Type = obj.TYPE_ADDR
p.From.Offset = int64(q * 8)
p = gins(ppc64.AMOVDU, &r0, &dst)
p.To.Type = obj.TYPE_MEM
p.To.Offset = 8
pl := (*obj.Prog)(p)
p = gins(ppc64.ACMP, &dst, &end)
gc.Patch(gc.Gbranch(ppc64.ABNE, nil, 0), pl)
gc.Regfree(&end)
// The loop leaves R3 on the last zeroed dword
boff = 8
} else if q >= 4 {
p := gins(ppc64.ASUB, nil, &dst)
p.From.Type = obj.TYPE_CONST
p.From.Offset = 8
f := (*gc.Node)(gc.Sysfunc("duffzero"))
p = gins(obj.ADUFFZERO, nil, f)
gc.Afunclit(&p.To, f)
// 4 and 128 = magic constants: see ../../runtime/asm_ppc64x.s
p.To.Offset = int64(4 * (128 - q))
// duffzero leaves R3 on the last zeroed dword
boff = 8
} else {
var p *obj.Prog
for t := uint64(0); t < q; t++ {
p = gins(ppc64.AMOVD, &r0, &dst)
p.To.Type = obj.TYPE_MEM
p.To.Offset = int64(8 * t)
}
boff = 8 * q
}
var p *obj.Prog
for t := uint64(0); t < c; t++ {
p = gins(ppc64.AMOVB, &r0, &dst)
p.To.Type = obj.TYPE_MEM
p.To.Offset = int64(t + boff)
}
gc.Regfree(&dst)
}
func blockcopy(n, res *gc.Node, osrc, odst, w int64) {
// determine alignment.
// want to avoid unaligned access, so have to use
// smaller operations for less aligned types.
// for example moving [4]byte must use 4 MOVB not 1 MOVW.
align := int(n.Type.Align)
var op int
switch align {
default:
gc.Fatalf("sgen: invalid alignment %d for %v", align, n.Type)
case 1:
op = mips.AMOVB
case 2:
op = mips.AMOVH
case 4:
op = mips.AMOVW
case 8:
op = mips.AMOVV
}
if w%int64(align) != 0 {
gc.Fatalf("sgen: unaligned size %d (align=%d) for %v", w, align, n.Type)
}
c := int32(w / int64(align))
// if we are copying forward on the stack and
// the src and dst overlap, then reverse direction
dir := align
if osrc < odst && odst < osrc+w {
dir = -dir
}
var dst gc.Node
var src gc.Node
if n.Ullman >= res.Ullman {
gc.Agenr(n, &dst, res) // temporarily use dst
gc.Regalloc(&src, gc.Types[gc.Tptr], nil)
gins(mips.AMOVV, &dst, &src)
if res.Op == gc.ONAME {
gc.Gvardef(res)
}
gc.Agen(res, &dst)
} else {
if res.Op == gc.ONAME {
gc.Gvardef(res)
}
gc.Agenr(res, &dst, res)
gc.Agenr(n, &src, nil)
}
var tmp gc.Node
gc.Regalloc(&tmp, gc.Types[gc.Tptr], nil)
// set up end marker
var nend gc.Node
// move src and dest to the end of block if necessary
if dir < 0 {
if c >= 4 {
gc.Regalloc(&nend, gc.Types[gc.Tptr], nil)
gins(mips.AMOVV, &src, &nend)
}
p := gins(mips.AADDV, nil, &src)
p.From.Type = obj.TYPE_CONST
p.From.Offset = w
p = gins(mips.AADDV, nil, &dst)
p.From.Type = obj.TYPE_CONST
p.From.Offset = w
} else {
p := gins(mips.AADDV, nil, &src)
p.From.Type = obj.TYPE_CONST
p.From.Offset = int64(-dir)
p = gins(mips.AADDV, nil, &dst)
p.From.Type = obj.TYPE_CONST
p.From.Offset = int64(-dir)
if c >= 4 {
gc.Regalloc(&nend, gc.Types[gc.Tptr], nil)
p := gins(mips.AMOVV, &src, &nend)
p.From.Type = obj.TYPE_ADDR
p.From.Offset = w
}
}
// move
// TODO: enable duffcopy for larger copies.
if c >= 4 {
p := gins(op, &src, &tmp)
p.From.Type = obj.TYPE_MEM
p.From.Offset = int64(dir)
ploop := p
p = gins(mips.AADDV, nil, &src)
p.From.Type = obj.TYPE_CONST
p.From.Offset = int64(dir)
p = gins(op, &tmp, &dst)
p.To.Type = obj.TYPE_MEM
p.To.Offset = int64(dir)
p = gins(mips.AADDV, nil, &dst)
p.From.Type = obj.TYPE_CONST
p.From.Offset = int64(dir)
gc.Patch(ginsbranch(mips.ABNE, nil, &src, &nend, 0), ploop)
gc.Regfree(&nend)
} else {
// TODO: Instead of generating ADDV $-8,R8; ADDV
// $-8,R7; n*(MOVV 8(R8),R9; ADDV $8,R8; MOVV R9,8(R7);
// ADDV $8,R7;) just generate the offsets directly and
// eliminate the ADDs. That will produce shorter, more
// pipeline-able code.
var p *obj.Prog
for ; c > 0; c-- {
p = gins(op, &src, &tmp)
p.From.Type = obj.TYPE_MEM
p.From.Offset = int64(dir)
p = gins(mips.AADDV, nil, &src)
p.From.Type = obj.TYPE_CONST
p.From.Offset = int64(dir)
p = gins(op, &tmp, &dst)
p.To.Type = obj.TYPE_MEM
p.To.Offset = int64(dir)
p = gins(mips.AADDV, nil, &dst)
p.From.Type = obj.TYPE_CONST
p.From.Offset = int64(dir)
}
}
gc.Regfree(&dst)
gc.Regfree(&src)
gc.Regfree(&tmp)
}
func blockcopy(n, res *gc.Node, osrc, odst, w int64) {
// determine alignment.
// want to avoid unaligned access, so have to use
// smaller operations for less aligned types.
// for example moving [4]byte must use 4 MOVB not 1 MOVW.
align := int(n.Type.Align)
var op int
switch align {
default:
gc.Fatalf("sgen: invalid alignment %d for %v", align, n.Type)
case 1:
op = arm.AMOVB
case 2:
op = arm.AMOVH
case 4:
op = arm.AMOVW
}
if w%int64(align) != 0 {
gc.Fatalf("sgen: unaligned size %d (align=%d) for %v", w, align, n.Type)
}
c := int32(w / int64(align))
if osrc%int64(align) != 0 || odst%int64(align) != 0 {
gc.Fatalf("sgen: unaligned offset src %d or dst %d (align %d)", osrc, odst, align)
}
// if we are copying forward on the stack and
// the src and dst overlap, then reverse direction
dir := align
if osrc < odst && int64(odst) < int64(osrc)+w {
dir = -dir
}
if op == arm.AMOVW && !gc.Nacl && dir > 0 && c >= 4 && c <= 128 {
var r0 gc.Node
r0.Op = gc.OREGISTER
r0.Reg = arm.REG_R0
var r1 gc.Node
r1.Op = gc.OREGISTER
r1.Reg = arm.REG_R0 + 1
var r2 gc.Node
r2.Op = gc.OREGISTER
r2.Reg = arm.REG_R0 + 2
var src gc.Node
gc.Regalloc(&src, gc.Types[gc.Tptr], &r1)
var dst gc.Node
gc.Regalloc(&dst, gc.Types[gc.Tptr], &r2)
if n.Ullman >= res.Ullman {
// eval n first
gc.Agen(n, &src)
if res.Op == gc.ONAME {
gc.Gvardef(res)
}
gc.Agen(res, &dst)
} else {
// eval res first
if res.Op == gc.ONAME {
gc.Gvardef(res)
}
gc.Agen(res, &dst)
gc.Agen(n, &src)
}
var tmp gc.Node
gc.Regalloc(&tmp, gc.Types[gc.Tptr], &r0)
f := gc.Sysfunc("duffcopy")
p := gins(obj.ADUFFCOPY, nil, f)
gc.Afunclit(&p.To, f)
// 8 and 128 = magic constants: see ../../runtime/asm_arm.s
p.To.Offset = 8 * (128 - int64(c))
gc.Regfree(&tmp)
gc.Regfree(&src)
gc.Regfree(&dst)
return
}
var dst gc.Node
var src gc.Node
if n.Ullman >= res.Ullman {
gc.Agenr(n, &dst, res) // temporarily use dst
gc.Regalloc(&src, gc.Types[gc.Tptr], nil)
gins(arm.AMOVW, &dst, &src)
if res.Op == gc.ONAME {
gc.Gvardef(res)
}
gc.Agen(res, &dst)
} else {
if res.Op == gc.ONAME {
gc.Gvardef(res)
}
gc.Agenr(res, &dst, res)
gc.Agenr(n, &src, nil)
}
var tmp gc.Node
gc.Regalloc(&tmp, gc.Types[gc.TUINT32], nil)
// set up end marker
var nend gc.Node
if c >= 4 {
gc.Regalloc(&nend, gc.Types[gc.TUINT32], nil)
p := gins(arm.AMOVW, &src, &nend)
p.From.Type = obj.TYPE_ADDR
if dir < 0 {
p.From.Offset = int64(dir)
} else {
p.From.Offset = w
}
}
// move src and dest to the end of block if necessary
if dir < 0 {
p := gins(arm.AMOVW, &src, &src)
p.From.Type = obj.TYPE_ADDR
p.From.Offset = w + int64(dir)
p = gins(arm.AMOVW, &dst, &dst)
p.From.Type = obj.TYPE_ADDR
p.From.Offset = w + int64(dir)
}
// move
if c >= 4 {
p := gins(op, &src, &tmp)
p.From.Type = obj.TYPE_MEM
p.From.Offset = int64(dir)
p.Scond |= arm.C_PBIT
ploop := p
p = gins(op, &tmp, &dst)
p.To.Type = obj.TYPE_MEM
p.To.Offset = int64(dir)
p.Scond |= arm.C_PBIT
p = gins(arm.ACMP, &src, nil)
raddr(&nend, p)
gc.Patch(gc.Gbranch(arm.ABNE, nil, 0), ploop)
gc.Regfree(&nend)
} else {
var p *obj.Prog
for {
tmp14 := c
c--
if tmp14 <= 0 {
break
}
p = gins(op, &src, &tmp)
p.From.Type = obj.TYPE_MEM
p.From.Offset = int64(dir)
p.Scond |= arm.C_PBIT
p = gins(op, &tmp, &dst)
p.To.Type = obj.TYPE_MEM
p.To.Offset = int64(dir)
p.Scond |= arm.C_PBIT
}
}
gc.Regfree(&dst)
gc.Regfree(&src)
gc.Regfree(&tmp)
}
func clearfat(nl *gc.Node) {
/* clear a fat object */
if gc.Debug['g'] != 0 {
gc.Dump("\nclearfat", nl)
}
w := nl.Type.Width
// Avoid taking the address for simple enough types.
if gc.Componentgen(nil, nl) {
return
}
c := w % 8 // bytes
q := w / 8 // quads
if q < 4 {
// Write sequence of MOV 0, off(base) instead of using STOSQ.
// The hope is that although the code will be slightly longer,
// the MOVs will have no dependencies and pipeline better
// than the unrolled STOSQ loop.
// NOTE: Must use agen, not igen, so that optimizer sees address
// being taken. We are not writing on field boundaries.
var n1 gc.Node
gc.Agenr(nl, &n1, nil)
n1.Op = gc.OINDREG
var z gc.Node
gc.Nodconst(&z, gc.Types[gc.TUINT64], 0)
for {
tmp14 := q
q--
if tmp14 <= 0 {
break
}
n1.Type = z.Type
gins(x86.AMOVQ, &z, &n1)
n1.Xoffset += 8
}
if c >= 4 {
gc.Nodconst(&z, gc.Types[gc.TUINT32], 0)
n1.Type = z.Type
gins(x86.AMOVL, &z, &n1)
n1.Xoffset += 4
c -= 4
}
gc.Nodconst(&z, gc.Types[gc.TUINT8], 0)
for {
tmp15 := c
c--
if tmp15 <= 0 {
break
}
n1.Type = z.Type
gins(x86.AMOVB, &z, &n1)
n1.Xoffset++
}
gc.Regfree(&n1)
return
}
var oldn1 gc.Node
var n1 gc.Node
savex(x86.REG_DI, &n1, &oldn1, nil, gc.Types[gc.Tptr])
gc.Agen(nl, &n1)
var ax gc.Node
var oldax gc.Node
savex(x86.REG_AX, &ax, &oldax, nil, gc.Types[gc.Tptr])
gconreg(x86.AMOVL, 0, x86.REG_AX)
if q > 128 || gc.Nacl {
gconreg(movptr, q, x86.REG_CX)
gins(x86.AREP, nil, nil) // repeat
gins(x86.ASTOSQ, nil, nil) // STOQ AL,*(DI)+
} else {
if di := dzDI(q); di != 0 {
gconreg(addptr, di, x86.REG_DI)
}
p := gins(obj.ADUFFZERO, nil, nil)
p.To.Type = obj.TYPE_ADDR
p.To.Sym = gc.Linksym(gc.Pkglookup("duffzero", gc.Runtimepkg))
p.To.Offset = dzOff(q)
}
z := ax
di := n1
if w >= 8 && c >= 4 {
di.Op = gc.OINDREG
z.Type = gc.Types[gc.TINT64]
di.Type = z.Type
p := gins(x86.AMOVQ, &z, &di)
p.To.Scale = 1
p.To.Offset = c - 8
} else if c >= 4 {
di.Op = gc.OINDREG
z.Type = gc.Types[gc.TINT32]
di.Type = z.Type
gins(x86.AMOVL, &z, &di)
if c > 4 {
p := gins(x86.AMOVL, &z, &di)
p.To.Scale = 1
p.To.Offset = c - 4
}
} else {
for c > 0 {
gins(x86.ASTOSB, nil, nil) // STOB AL,*(DI)+
c--
}
}
restx(&n1, &oldn1)
restx(&ax, &oldax)
}
func clearfat(nl *gc.Node) {
/* clear a fat object */
if gc.Debug['g'] != 0 {
fmt.Printf("clearfat %v (%v, size: %d)\n", nl, nl.Type, nl.Type.Width)
}
w := uint64(nl.Type.Width)
// Avoid taking the address for simple enough types.
if gc.Componentgen(nil, nl) {
return
}
c := w % 8 // bytes
q := w / 8 // dwords
if gc.Reginuse(mips.REGRT1) {
gc.Fatalf("%v in use during clearfat", obj.Rconv(mips.REGRT1))
}
var r0 gc.Node
gc.Nodreg(&r0, gc.Types[gc.TUINT64], mips.REGZERO)
var dst gc.Node
gc.Nodreg(&dst, gc.Types[gc.Tptr], mips.REGRT1)
gc.Regrealloc(&dst)
gc.Agen(nl, &dst)
var boff uint64
if q > 128 {
p := gins(mips.ASUBV, nil, &dst)
p.From.Type = obj.TYPE_CONST
p.From.Offset = 8
var end gc.Node
gc.Regalloc(&end, gc.Types[gc.Tptr], nil)
p = gins(mips.AMOVV, &dst, &end)
p.From.Type = obj.TYPE_ADDR
p.From.Offset = int64(q * 8)
p = gins(mips.AMOVV, &r0, &dst)
p.To.Type = obj.TYPE_MEM
p.To.Offset = 8
pl := p
p = gins(mips.AADDV, nil, &dst)
p.From.Type = obj.TYPE_CONST
p.From.Offset = 8
gc.Patch(ginsbranch(mips.ABNE, nil, &dst, &end, 0), pl)
gc.Regfree(&end)
// The loop leaves R1 on the last zeroed dword
boff = 8
// TODO(dfc): https://golang.org/issue/12108
// If DUFFZERO is used inside a tail call (see genwrapper) it will
// overwrite the link register.
} else if false && q >= 4 {
p := gins(mips.ASUBV, nil, &dst)
p.From.Type = obj.TYPE_CONST
p.From.Offset = 8
f := gc.Sysfunc("duffzero")
p = gins(obj.ADUFFZERO, nil, f)
gc.Afunclit(&p.To, f)
// 8 and 128 = magic constants: see ../../runtime/asm_mips64x.s
p.To.Offset = int64(8 * (128 - q))
// duffzero leaves R1 on the last zeroed dword
boff = 8
} else {
var p *obj.Prog
for t := uint64(0); t < q; t++ {
p = gins(mips.AMOVV, &r0, &dst)
p.To.Type = obj.TYPE_MEM
p.To.Offset = int64(8 * t)
}
boff = 8 * q
}
var p *obj.Prog
for t := uint64(0); t < c; t++ {
p = gins(mips.AMOVB, &r0, &dst)
p.To.Type = obj.TYPE_MEM
p.To.Offset = int64(t + boff)
}
gc.Regfree(&dst)
}
func clearfat(nl *gc.Node) {
/* clear a fat object */
if gc.Debug['g'] != 0 {
fmt.Printf("clearfat %v (%v, size: %d)\n", nl, nl.Type, nl.Type.Width)
}
w := uint64(uint64(nl.Type.Width))
// Avoid taking the address for simple enough types.
if gc.Componentgen(nil, nl) {
return
}
c := uint64(w % 8) // bytes
q := uint64(w / 8) // dwords
var r0 gc.Node
gc.Nodreg(&r0, gc.Types[gc.TUINT64], arm64.REGZERO)
var dst gc.Node
// REGRT1 is reserved on arm64, see arm64/gsubr.go.
gc.Nodreg(&dst, gc.Types[gc.Tptr], arm64.REGRT1)
gc.Agen(nl, &dst)
var boff uint64
if q > 128 {
p := gins(arm64.ASUB, nil, &dst)
p.From.Type = obj.TYPE_CONST
p.From.Offset = 8
var end gc.Node
gc.Regalloc(&end, gc.Types[gc.Tptr], nil)
p = gins(arm64.AMOVD, &dst, &end)
p.From.Type = obj.TYPE_ADDR
p.From.Offset = int64(q * 8)
p = gins(arm64.AMOVD, &r0, &dst)
p.To.Type = obj.TYPE_MEM
p.To.Offset = 8
p.Scond = arm64.C_XPRE
pl := (*obj.Prog)(p)
p = gcmp(arm64.ACMP, &dst, &end)
gc.Patch(gc.Gbranch(arm64.ABNE, nil, 0), pl)
gc.Regfree(&end)
// The loop leaves R16 on the last zeroed dword
boff = 8
} else if q >= 4 && !darwin { // darwin ld64 cannot handle BR26 reloc with non-zero addend
p := gins(arm64.ASUB, nil, &dst)
p.From.Type = obj.TYPE_CONST
p.From.Offset = 8
f := (*gc.Node)(gc.Sysfunc("duffzero"))
p = gins(obj.ADUFFZERO, nil, f)
gc.Afunclit(&p.To, f)
// 4 and 128 = magic constants: see ../../runtime/asm_arm64x.s
p.To.Offset = int64(4 * (128 - q))
// duffzero leaves R16 on the last zeroed dword
boff = 8
} else {
var p *obj.Prog
for t := uint64(0); t < q; t++ {
p = gins(arm64.AMOVD, &r0, &dst)
p.To.Type = obj.TYPE_MEM
p.To.Offset = int64(8 * t)
}
boff = 8 * q
}
var p *obj.Prog
for t := uint64(0); t < c; t++ {
p = gins(arm64.AMOVB, &r0, &dst)
p.To.Type = obj.TYPE_MEM
p.To.Offset = int64(t + boff)
}
}
func clearfat(nl *gc.Node) {
/* clear a fat object */
if gc.Debug['g'] != 0 {
gc.Dump("\nclearfat", nl)
}
w := uint32(nl.Type.Width)
// Avoid taking the address for simple enough types.
if gc.Componentgen(nil, nl) {
return
}
c := w % 4 // bytes
q := w / 4 // quads
var r0 gc.Node
r0.Op = gc.OREGISTER
r0.Reg = arm.REG_R0
var r1 gc.Node
r1.Op = gc.OREGISTER
r1.Reg = arm.REG_R1
var dst gc.Node
gc.Regalloc(&dst, gc.Types[gc.Tptr], &r1)
gc.Agen(nl, &dst)
var nc gc.Node
gc.Nodconst(&nc, gc.Types[gc.TUINT32], 0)
var nz gc.Node
gc.Regalloc(&nz, gc.Types[gc.TUINT32], &r0)
gc.Cgen(&nc, &nz)
if q > 128 {
var end gc.Node
gc.Regalloc(&end, gc.Types[gc.Tptr], nil)
p := gins(arm.AMOVW, &dst, &end)
p.From.Type = obj.TYPE_ADDR
p.From.Offset = int64(q) * 4
p = gins(arm.AMOVW, &nz, &dst)
p.To.Type = obj.TYPE_MEM
p.To.Offset = 4
p.Scond |= arm.C_PBIT
pl := p
p = gins(arm.ACMP, &dst, nil)
raddr(&end, p)
gc.Patch(gc.Gbranch(arm.ABNE, nil, 0), pl)
gc.Regfree(&end)
} else if q >= 4 && !gc.Nacl {
f := gc.Sysfunc("duffzero")
p := gins(obj.ADUFFZERO, nil, f)
gc.Afunclit(&p.To, f)
// 4 and 128 = magic constants: see ../../runtime/asm_arm.s
p.To.Offset = 4 * (128 - int64(q))
} else {
var p *obj.Prog
for q > 0 {
p = gins(arm.AMOVW, &nz, &dst)
p.To.Type = obj.TYPE_MEM
p.To.Offset = 4
p.Scond |= arm.C_PBIT
//print("1. %v\n", p);
q--
}
}
var p *obj.Prog
for c > 0 {
p = gins(arm.AMOVB, &nz, &dst)
p.To.Type = obj.TYPE_MEM
p.To.Offset = 1
p.Scond |= arm.C_PBIT
//print("2. %v\n", p);
c--
}
gc.Regfree(&dst)
gc.Regfree(&nz)
}
func clearfat(nl *gc.Node) {
/* clear a fat object */
if gc.Debug['g'] != 0 {
gc.Dump("\nclearfat", nl)
}
w := uint32(nl.Type.Width)
// Avoid taking the address for simple enough types.
if gc.Componentgen(nil, nl) {
return
}
c := w % 4 // bytes
q := w / 4 // quads
if q < 4 {
// Write sequence of MOV 0, off(base) instead of using STOSL.
// The hope is that although the code will be slightly longer,
// the MOVs will have no dependencies and pipeline better
// than the unrolled STOSL loop.
// NOTE: Must use agen, not igen, so that optimizer sees address
// being taken. We are not writing on field boundaries.
var n1 gc.Node
gc.Regalloc(&n1, gc.Types[gc.Tptr], nil)
gc.Agen(nl, &n1)
n1.Op = gc.OINDREG
var z gc.Node
gc.Nodconst(&z, gc.Types[gc.TUINT64], 0)
for ; q > 0; q-- {
n1.Type = z.Type
gins(x86.AMOVL, &z, &n1)
n1.Xoffset += 4
}
gc.Nodconst(&z, gc.Types[gc.TUINT8], 0)
for ; c > 0; c-- {
n1.Type = z.Type
gins(x86.AMOVB, &z, &n1)
n1.Xoffset++
}
gc.Regfree(&n1)
return
}
var n1 gc.Node
gc.Nodreg(&n1, gc.Types[gc.Tptr], x86.REG_DI)
gc.Agen(nl, &n1)
gconreg(x86.AMOVL, 0, x86.REG_AX)
if q > 128 || (q >= 4 && gc.Nacl) {
gconreg(x86.AMOVL, int64(q), x86.REG_CX)
gins(x86.AREP, nil, nil) // repeat
gins(x86.ASTOSL, nil, nil) // STOL AL,*(DI)+
} else if q >= 4 {
p := gins(obj.ADUFFZERO, nil, nil)
p.To.Type = obj.TYPE_ADDR
p.To.Sym = gc.Linksym(gc.Pkglookup("duffzero", gc.Runtimepkg))
// 1 and 128 = magic constants: see ../../runtime/asm_386.s
p.To.Offset = 1 * (128 - int64(q))
} else {
for q > 0 {
gins(x86.ASTOSL, nil, nil) // STOL AL,*(DI)+
q--
}
}
for c > 0 {
gins(x86.ASTOSB, nil, nil) // STOB AL,*(DI)+
c--
}
}
// clearfat clears (i.e. replaces with zeros) the value pointed to by nl.
func clearfat(nl *gc.Node) {
if gc.Debug['g'] != 0 {
fmt.Printf("clearfat %v (%v, size: %d)\n", nl, nl.Type, nl.Type.Width)
}
// Avoid taking the address for simple enough types.
if gc.Componentgen(nil, nl) {
return
}
var dst gc.Node
gc.Regalloc(&dst, gc.Types[gc.Tptr], nil)
gc.Agen(nl, &dst)
var boff int64
w := nl.Type.Width
if w > clearLoopCutoff {
// Generate a loop clearing 256 bytes per iteration using XCs.
var end gc.Node
gc.Regalloc(&end, gc.Types[gc.Tptr], nil)
p := gins(s390x.AMOVD, &dst, &end)
p.From.Type = obj.TYPE_ADDR
p.From.Offset = w - (w % 256)
p = gins(s390x.AXC, &dst, &dst)
p.From.Type = obj.TYPE_MEM
p.From.Offset = 0
p.To.Type = obj.TYPE_MEM
p.To.Offset = 0
p.From3 = new(obj.Addr)
p.From3.Offset = 256
p.From3.Type = obj.TYPE_CONST
pl := p
ginscon(s390x.AADD, 256, &dst)
gins(s390x.ACMP, &dst, &end)
gc.Patch(gc.Gbranch(s390x.ABNE, nil, 0), pl)
gc.Regfree(&end)
w = w % 256
}
// Generate instructions to clear the remaining memory.
for w > 0 {
n := w
// Can clear at most 256 bytes per instruction.
if n > 256 {
n = 256
}
switch n {
// Handle very small clears using moves.
case 8, 4, 2, 1:
ins := s390x.AMOVB
switch n {
case 8:
ins = s390x.AMOVD
case 4:
ins = s390x.AMOVW
case 2:
ins = s390x.AMOVH
}
p := gins(ins, nil, &dst)
p.From.Type = obj.TYPE_CONST
p.From.Offset = 0
p.To.Type = obj.TYPE_MEM
p.To.Offset = boff
// Handle clears that would require multiple moves with a XC.
default:
p := gins(s390x.AXC, &dst, &dst)
p.From.Type = obj.TYPE_MEM
p.From.Offset = boff
p.To.Type = obj.TYPE_MEM
p.To.Offset = boff
p.From3 = new(obj.Addr)
p.From3.Offset = n
p.From3.Type = obj.TYPE_CONST
}
boff += n
w -= n
}
gc.Regfree(&dst)
}
/*
* generate code to compute address of n,
* a reference to a (perhaps nested) field inside
* an array or struct.
* return 0 on failure, 1 on success.
* on success, leaves usable address in a.
*
* caller is responsible for calling sudoclean
* after successful sudoaddable,
* to release the register used for a.
*/
func sudoaddable(as int, n *gc.Node, a *obj.Addr) bool {
if n.Type == nil {
return false
}
*a = obj.Addr{}
switch n.Op {
case gc.OLITERAL:
if !gc.Isconst(n, gc.CTINT) {
break
}
v := n.Int()
if v >= 32000 || v <= -32000 {
break
}
switch as {
default:
return false
case arm.AADD,
arm.ASUB,
arm.AAND,
arm.AORR,
arm.AEOR,
arm.AMOVB,
arm.AMOVBS,
arm.AMOVBU,
arm.AMOVH,
arm.AMOVHS,
arm.AMOVHU,
arm.AMOVW:
break
}
cleani += 2
reg := &clean[cleani-1]
reg1 := &clean[cleani-2]
reg.Op = gc.OEMPTY
reg1.Op = gc.OEMPTY
gc.Naddr(a, n)
return true
case gc.ODOT,
gc.ODOTPTR:
cleani += 2
reg := &clean[cleani-1]
reg1 := &clean[cleani-2]
reg.Op = gc.OEMPTY
reg1.Op = gc.OEMPTY
var nn *gc.Node
var oary [10]int64
o := gc.Dotoffset(n, oary[:], &nn)
if nn == nil {
sudoclean()
return false
}
if nn.Addable && o == 1 && oary[0] >= 0 {
// directly addressable set of DOTs
n1 := *nn
n1.Type = n.Type
n1.Xoffset += oary[0]
gc.Naddr(a, &n1)
return true
}
gc.Regalloc(reg, gc.Types[gc.Tptr], nil)
n1 := *reg
n1.Op = gc.OINDREG
if oary[0] >= 0 {
gc.Agen(nn, reg)
n1.Xoffset = oary[0]
} else {
gc.Cgen(nn, reg)
gc.Cgen_checknil(reg)
n1.Xoffset = -(oary[0] + 1)
}
for i := 1; i < o; i++ {
if oary[i] >= 0 {
gc.Fatal("can't happen")
}
gins(arm.AMOVW, &n1, reg)
gc.Cgen_checknil(reg)
n1.Xoffset = -(oary[i] + 1)
}
a.Type = obj.TYPE_NONE
a.Name = obj.NAME_NONE
n1.Type = n.Type
gc.Naddr(a, &n1)
return true
case gc.OINDEX:
return false
}
return false
}
func clearfat(nl *gc.Node) {
/* clear a fat object */
if gc.Debug['g'] != 0 {
gc.Dump("\nclearfat", nl)
}
// Avoid taking the address for simple enough types.
if gc.Componentgen(nil, nl) {
return
}
w := nl.Type.Width
if w > 1024 || (w >= 64 && (gc.Nacl || isPlan9)) {
var oldn1 gc.Node
var n1 gc.Node
savex(x86.REG_DI, &n1, &oldn1, nil, gc.Types[gc.Tptr])
gc.Agen(nl, &n1)
var ax gc.Node
var oldax gc.Node
savex(x86.REG_AX, &ax, &oldax, nil, gc.Types[gc.Tptr])
gconreg(x86.AMOVL, 0, x86.REG_AX)
gconreg(movptr, w/8, x86.REG_CX)
gins(x86.AREP, nil, nil) // repeat
gins(x86.ASTOSQ, nil, nil) // STOQ AL,*(DI)+
if w%8 != 0 {
n1.Op = gc.OINDREG
clearfat_tail(&n1, w%8)
}
restx(&n1, &oldn1)
restx(&ax, &oldax)
return
}
if w >= 64 {
var oldn1 gc.Node
var n1 gc.Node
savex(x86.REG_DI, &n1, &oldn1, nil, gc.Types[gc.Tptr])
gc.Agen(nl, &n1)
var vec_zero gc.Node
var old_x0 gc.Node
savex(x86.REG_X0, &vec_zero, &old_x0, nil, gc.Types[gc.TFLOAT64])
gins(x86.AXORPS, &vec_zero, &vec_zero)
if di := dzDI(w); di != 0 {
gconreg(addptr, di, x86.REG_DI)
}
p := gins(obj.ADUFFZERO, nil, nil)
p.To.Type = obj.TYPE_ADDR
p.To.Sym = gc.Linksym(gc.Pkglookup("duffzero", gc.Runtimepkg))
p.To.Offset = dzOff(w)
if w%16 != 0 {
n1.Op = gc.OINDREG
n1.Xoffset -= 16 - w%16
gins(x86.AMOVUPS, &vec_zero, &n1)
}
restx(&vec_zero, &old_x0)
restx(&n1, &oldn1)
return
}
// NOTE: Must use agen, not igen, so that optimizer sees address
// being taken. We are not writing on field boundaries.
var n1 gc.Node
gc.Agenr(nl, &n1, nil)
n1.Op = gc.OINDREG
clearfat_tail(&n1, w)
gc.Regfree(&n1)
}
/*
* generate code to compute address of n,
* a reference to a (perhaps nested) field inside
* an array or struct.
* return 0 on failure, 1 on success.
* on success, leaves usable address in a.
*
* caller is responsible for calling sudoclean
* after successful sudoaddable,
* to release the register used for a.
*/
func sudoaddable(as int, n *gc.Node, a *obj.Addr) bool {
if n.Type == nil {
return false
}
*a = obj.Addr{}
switch n.Op {
case gc.OLITERAL:
if !gc.Isconst(n, gc.CTINT) {
break
}
v := n.Int()
if v >= 32000 || v <= -32000 {
break
}
switch as {
default:
return false
case x86.AADDB,
x86.AADDW,
x86.AADDL,
x86.AADDQ,
x86.ASUBB,
x86.ASUBW,
x86.ASUBL,
x86.ASUBQ,
x86.AANDB,
x86.AANDW,
x86.AANDL,
x86.AANDQ,
x86.AORB,
x86.AORW,
x86.AORL,
x86.AORQ,
x86.AXORB,
x86.AXORW,
x86.AXORL,
x86.AXORQ,
x86.AINCB,
x86.AINCW,
x86.AINCL,
x86.AINCQ,
x86.ADECB,
x86.ADECW,
x86.ADECL,
x86.ADECQ,
x86.AMOVB,
x86.AMOVW,
x86.AMOVL,
x86.AMOVQ:
break
}
cleani += 2
reg := &clean[cleani-1]
reg1 := &clean[cleani-2]
reg.Op = gc.OEMPTY
reg1.Op = gc.OEMPTY
gc.Naddr(a, n)
return true
case gc.ODOT,
gc.ODOTPTR:
cleani += 2
reg := &clean[cleani-1]
reg1 := &clean[cleani-2]
reg.Op = gc.OEMPTY
reg1.Op = gc.OEMPTY
var nn *gc.Node
var oary [10]int64
o := gc.Dotoffset(n, oary[:], &nn)
if nn == nil {
sudoclean()
return false
}
if nn.Addable && o == 1 && oary[0] >= 0 {
// directly addressable set of DOTs
n1 := *nn
n1.Type = n.Type
n1.Xoffset += oary[0]
gc.Naddr(a, &n1)
return true
}
gc.Regalloc(reg, gc.Types[gc.Tptr], nil)
n1 := *reg
n1.Op = gc.OINDREG
if oary[0] >= 0 {
gc.Agen(nn, reg)
n1.Xoffset = oary[0]
} else {
gc.Cgen(nn, reg)
gc.Cgen_checknil(reg)
n1.Xoffset = -(oary[0] + 1)
}
for i := 1; i < o; i++ {
if oary[i] >= 0 {
gc.Fatal("can't happen")
}
gins(movptr, &n1, reg)
gc.Cgen_checknil(reg)
n1.Xoffset = -(oary[i] + 1)
}
a.Type = obj.TYPE_NONE
a.Index = obj.TYPE_NONE
gc.Fixlargeoffset(&n1)
gc.Naddr(a, &n1)
return true
case gc.OINDEX:
return false
}
return false
}
/*
* generate code to compute address of n,
* a reference to a (perhaps nested) field inside
* an array or struct.
* return 0 on failure, 1 on success.
* on success, leaves usable address in a.
*
* caller is responsible for calling sudoclean
* after successful sudoaddable,
* to release the register used for a.
*/
func sudoaddable(as obj.As, n *gc.Node, a *obj.Addr) bool {
if n.Type == nil {
return false
}
*a = obj.Addr{}
switch n.Op {
case gc.OLITERAL:
if !gc.Isconst(n, gc.CTINT) {
return false
}
v := n.Int64()
switch as {
default:
return false
// operations that can cope with a 32-bit immediate
// TODO(mundaym): logical operations can work on high bits
case s390x.AADD,
s390x.AADDC,
s390x.ASUB,
s390x.AMULLW,
s390x.AAND,
s390x.AOR,
s390x.AXOR,
s390x.ASLD,
s390x.ASLW,
s390x.ASRAW,
s390x.ASRAD,
s390x.ASRW,
s390x.ASRD,
s390x.AMOVB,
s390x.AMOVBZ,
s390x.AMOVH,
s390x.AMOVHZ,
s390x.AMOVW,
s390x.AMOVWZ,
s390x.AMOVD:
if int64(int32(v)) != v {
return false
}
// for comparisons avoid immediates unless they can
// fit into a int8/uint8
// this favours combined compare and branch instructions
case s390x.ACMP:
if int64(int8(v)) != v {
return false
}
case s390x.ACMPU:
if int64(uint8(v)) != v {
return false
}
}
cleani += 2
reg := &clean[cleani-1]
reg1 := &clean[cleani-2]
reg.Op = gc.OEMPTY
reg1.Op = gc.OEMPTY
gc.Naddr(a, n)
return true
case gc.ODOT,
gc.ODOTPTR:
cleani += 2
reg := &clean[cleani-1]
reg1 := &clean[cleani-2]
reg.Op = gc.OEMPTY
reg1.Op = gc.OEMPTY
var nn *gc.Node
var oary [10]int64
o := gc.Dotoffset(n, oary[:], &nn)
if nn == nil {
sudoclean()
return false
}
if nn.Addable && o == 1 && oary[0] >= 0 {
// directly addressable set of DOTs
n1 := *nn
n1.Type = n.Type
n1.Xoffset += oary[0]
// check that the offset fits into a 12-bit displacement
if n1.Xoffset < 0 || n1.Xoffset >= (1<<12)-8 {
sudoclean()
return false
}
gc.Naddr(a, &n1)
return true
}
gc.Regalloc(reg, gc.Types[gc.Tptr], nil)
n1 := *reg
n1.Op = gc.OINDREG
if oary[0] >= 0 {
gc.Agen(nn, reg)
n1.Xoffset = oary[0]
} else {
gc.Cgen(nn, reg)
gc.Cgen_checknil(reg)
n1.Xoffset = -(oary[0] + 1)
}
for i := 1; i < o; i++ {
if oary[i] >= 0 {
gc.Fatalf("can't happen")
}
gins(s390x.AMOVD, &n1, reg)
gc.Cgen_checknil(reg)
n1.Xoffset = -(oary[i] + 1)
}
a.Type = obj.TYPE_NONE
a.Index = 0
// check that the offset fits into a 12-bit displacement
if n1.Xoffset < 0 || n1.Xoffset >= (1<<12)-8 {
tmp := n1
tmp.Op = gc.OREGISTER
tmp.Type = gc.Types[gc.Tptr]
tmp.Xoffset = 0
gc.Cgen_checknil(&tmp)
ginscon(s390x.AADD, n1.Xoffset, &tmp)
n1.Xoffset = 0
}
gc.Naddr(a, &n1)
return true
}
return false
}
// blockcopy copies w bytes from &n to &res
func blockcopy(n, res *gc.Node, osrc, odst, w int64) {
var dst gc.Node
var src gc.Node
if n.Ullman >= res.Ullman {
gc.Agenr(n, &dst, res) // temporarily use dst
gc.Regalloc(&src, gc.Types[gc.Tptr], nil)
gins(s390x.AMOVD, &dst, &src)
if res.Op == gc.ONAME {
gc.Gvardef(res)
}
gc.Agen(res, &dst)
} else {
if res.Op == gc.ONAME {
gc.Gvardef(res)
}
gc.Agenr(res, &dst, res)
gc.Agenr(n, &src, nil)
}
defer gc.Regfree(&src)
defer gc.Regfree(&dst)
var tmp gc.Node
gc.Regalloc(&tmp, gc.Types[gc.Tptr], nil)
defer gc.Regfree(&tmp)
offset := int64(0)
dir := _FORWARDS
if osrc < odst && odst < osrc+w {
// Reverse. Can't use MVC, fall back onto basic moves.
dir = _BACKWARDS
const copiesPerIter = 2
if w >= 8*copiesPerIter {
cnt := w - (w % (8 * copiesPerIter))
ginscon(s390x.AADD, w, &src)
ginscon(s390x.AADD, w, &dst)
var end gc.Node
gc.Regalloc(&end, gc.Types[gc.Tptr], nil)
p := gins(s390x.ASUB, nil, &end)
p.From.Type = obj.TYPE_CONST
p.From.Offset = cnt
p.Reg = src.Reg
var label *obj.Prog
for i := 0; i < copiesPerIter; i++ {
offset := int64(-8 * (i + 1))
p := gins(s390x.AMOVD, &src, &tmp)
p.From.Type = obj.TYPE_MEM
p.From.Offset = offset
if i == 0 {
label = p
}
p = gins(s390x.AMOVD, &tmp, &dst)
p.To.Type = obj.TYPE_MEM
p.To.Offset = offset
}
ginscon(s390x.ASUB, 8*copiesPerIter, &src)
ginscon(s390x.ASUB, 8*copiesPerIter, &dst)
gins(s390x.ACMP, &src, &end)
gc.Patch(gc.Gbranch(s390x.ABNE, nil, 0), label)
gc.Regfree(&end)
w -= cnt
} else {
offset = w
}
}
if dir == _FORWARDS && w > 1024 {
// Loop over MVCs
cnt := w - (w % 256)
var end gc.Node
gc.Regalloc(&end, gc.Types[gc.Tptr], nil)
add := gins(s390x.AADD, nil, &end)
add.From.Type = obj.TYPE_CONST
add.From.Offset = cnt
add.Reg = src.Reg
mvc := gins(s390x.AMVC, &src, &dst)
mvc.From.Type = obj.TYPE_MEM
mvc.From.Offset = 0
mvc.To.Type = obj.TYPE_MEM
mvc.To.Offset = 0
mvc.From3 = new(obj.Addr)
mvc.From3.Type = obj.TYPE_CONST
mvc.From3.Offset = 256
ginscon(s390x.AADD, 256, &src)
ginscon(s390x.AADD, 256, &dst)
gins(s390x.ACMP, &src, &end)
gc.Patch(gc.Gbranch(s390x.ABNE, nil, 0), mvc)
gc.Regfree(&end)
w -= cnt
}
for w > 0 {
cnt := w
// If in reverse we can only do 8, 4, 2 or 1 bytes at a time.
if dir == _BACKWARDS {
switch {
case cnt >= 8:
cnt = 8
case cnt >= 4:
cnt = 4
case cnt >= 2:
cnt = 2
}
} else if cnt > 256 {
cnt = 256
}
switch cnt {
case 8, 4, 2, 1:
op := s390x.AMOVB
switch cnt {
case 8:
op = s390x.AMOVD
case 4:
op = s390x.AMOVW
case 2:
op = s390x.AMOVH
}
load := gins(op, &src, &tmp)
load.From.Type = obj.TYPE_MEM
load.From.Offset = offset
store := gins(op, &tmp, &dst)
store.To.Type = obj.TYPE_MEM
store.To.Offset = offset
if dir == _BACKWARDS {
load.From.Offset -= cnt
store.To.Offset -= cnt
}
default:
p := gins(s390x.AMVC, &src, &dst)
p.From.Type = obj.TYPE_MEM
p.From.Offset = offset
p.To.Type = obj.TYPE_MEM
p.To.Offset = offset
p.From3 = new(obj.Addr)
p.From3.Type = obj.TYPE_CONST
p.From3.Offset = cnt
}
switch dir {
case _FORWARDS:
offset += cnt
case _BACKWARDS:
offset -= cnt
}
w -= cnt
}
}
func blockcopy(n, res *gc.Node, osrc, odst, w int64) {
var dst gc.Node
gc.Nodreg(&dst, gc.Types[gc.Tptr], x86.REG_DI)
var src gc.Node
gc.Nodreg(&src, gc.Types[gc.Tptr], x86.REG_SI)
var tsrc gc.Node
gc.Tempname(&tsrc, gc.Types[gc.Tptr])
var tdst gc.Node
gc.Tempname(&tdst, gc.Types[gc.Tptr])
if !n.Addable {
gc.Agen(n, &tsrc)
}
if !res.Addable {
gc.Agen(res, &tdst)
}
if n.Addable {
gc.Agen(n, &src)
} else {
gmove(&tsrc, &src)
}
if res.Op == gc.ONAME {
gc.Gvardef(res)
}
if res.Addable {
gc.Agen(res, &dst)
} else {
gmove(&tdst, &dst)
}
c := int32(w % 4) // bytes
q := int32(w / 4) // doublewords
// if we are copying forward on the stack and
// the src and dst overlap, then reverse direction
if osrc < odst && odst < osrc+w {
// reverse direction
gins(x86.ASTD, nil, nil) // set direction flag
if c > 0 {
gconreg(x86.AADDL, w-1, x86.REG_SI)
gconreg(x86.AADDL, w-1, x86.REG_DI)
gconreg(x86.AMOVL, int64(c), x86.REG_CX)
gins(x86.AREP, nil, nil) // repeat
gins(x86.AMOVSB, nil, nil) // MOVB *(SI)-,*(DI)-
}
if q > 0 {
if c > 0 {
gconreg(x86.AADDL, -3, x86.REG_SI)
gconreg(x86.AADDL, -3, x86.REG_DI)
} else {
gconreg(x86.AADDL, w-4, x86.REG_SI)
gconreg(x86.AADDL, w-4, x86.REG_DI)
}
gconreg(x86.AMOVL, int64(q), x86.REG_CX)
gins(x86.AREP, nil, nil) // repeat
gins(x86.AMOVSL, nil, nil) // MOVL *(SI)-,*(DI)-
}
// we leave with the flag clear
gins(x86.ACLD, nil, nil)
} else {
gins(x86.ACLD, nil, nil) // paranoia. TODO(rsc): remove?
// normal direction
if q > 128 || (q >= 4 && gc.Nacl) {
gconreg(x86.AMOVL, int64(q), x86.REG_CX)
gins(x86.AREP, nil, nil) // repeat
gins(x86.AMOVSL, nil, nil) // MOVL *(SI)+,*(DI)+
} else if q >= 4 {
p := gins(obj.ADUFFCOPY, nil, nil)
p.To.Type = obj.TYPE_ADDR
p.To.Sym = gc.Linksym(gc.Pkglookup("duffcopy", gc.Runtimepkg))
// 10 and 128 = magic constants: see ../../runtime/asm_386.s
p.To.Offset = 10 * (128 - int64(q))
} else if !gc.Nacl && c == 0 {
var cx gc.Node
gc.Nodreg(&cx, gc.Types[gc.TINT32], x86.REG_CX)
// We don't need the MOVSL side-effect of updating SI and DI,
// and issuing a sequence of MOVLs directly is faster.
src.Op = gc.OINDREG
dst.Op = gc.OINDREG
for q > 0 {
gmove(&src, &cx) // MOVL x+(SI),CX
gmove(&cx, &dst) // MOVL CX,x+(DI)
src.Xoffset += 4
dst.Xoffset += 4
q--
}
} else {
for q > 0 {
gins(x86.AMOVSL, nil, nil) // MOVL *(SI)+,*(DI)+
q--
}
}
for c > 0 {
gins(x86.AMOVSB, nil, nil) // MOVB *(SI)+,*(DI)+
c--
}
}
}