// opBorrowMaybeResultReg returns either the register of the given operand, or
// the reserved result register with the value of the operand.
func (mach X86) opBorrowMaybeResultReg(code gen.RegCoder, x values.Operand, preserveFlags bool) (reg regs.R, zeroExt, own bool) {
if x.Storage == values.VarReg {
reg = x.Reg()
zeroExt = x.RegZeroExt()
} else {
reg, zeroExt = mach.opMaybeResultReg(code, x, preserveFlags)
own = (reg != regResult)
}
return
}
// opBorrowMaybeScratchReg returns either the register of the given operand, or
// the reserved scratch register with the value of the operand.
func (mach X86) opBorrowMaybeScratchReg(code gen.Coder, x values.Operand, preserveFlags bool) (reg regs.R, zeroExt, own bool) {
if x.Storage.IsReg() {
reg = x.Reg()
zeroExt = x.RegZeroExt()
} else {
reg = regScratch
zeroExt = mach.OpMove(code, reg, x, preserveFlags)
}
own = (x.Storage == values.TempReg)
return
}
func (mach X86) ConversionOp(code gen.RegCoder, oper uint16, resultType types.T, source values.Operand) (result values.Operand) {
if oper == opers.Wrap {
source.Type = types.I32 // short mov; useful zeroExt flag
reg, zeroExt := mach.opMaybeResultReg(code, source, false)
return values.TempRegOperand(resultType, reg, zeroExt)
}
reg, zeroExt := mach.opMaybeResultReg(code, source, false)
// TODO: for int<->float ops: borrow source reg, allocate target reg
switch oper {
case opers.ExtendS:
Movsxd.opFromReg(code, 0, reg, reg)
result = values.TempRegOperand(resultType, reg, false)
case opers.ExtendU:
if !zeroExt {
Mov.opFromReg(code, types.I32, reg, reg)
}
result = values.TempRegOperand(resultType, reg, false)
case opers.Mote:
Cvts2sSSE.opFromReg(code, source.Type, reg, reg)
result = values.TempRegOperand(resultType, reg, false)
case opers.TruncS:
panic(fmt.Errorf("%s.trunc_s/%s not implemented", resultType, source.Type))
case opers.TruncU:
panic(fmt.Errorf("%s.trunc_u/%s not implemented", resultType, source.Type))
case opers.ConvertS:
Cvtsi2sSSE.opReg(code, resultType, source.Type, regResult, reg)
code.FreeReg(source.Type, reg)
result = values.TempRegOperand(resultType, regResult, false)
case opers.ConvertU:
if source.Type == types.I32 {
if !zeroExt {
Mov.opFromReg(code, types.I32, reg, reg)
}
Cvtsi2sSSE.opReg(code, resultType, types.I64, regResult, reg)
} else {
mach.opConvertUnsignedI64ToFloat(code, resultType, reg)
}
code.FreeReg(source.Type, reg)
result = values.TempRegOperand(resultType, regResult, false)
case opers.Reinterpret:
if source.Type.Category() == types.Int {
MovSSE.opFromReg(code, source.Type, regResult, reg)
} else {
MovSSE.opToReg(code, source.Type, regResult, reg)
}
code.FreeReg(source.Type, reg)
result = values.TempRegOperand(resultType, regResult, true)
}
return
}
// OpSetGlobal must not update CPU's condition flags.
func (mach X86) OpSetGlobal(code gen.Coder, offset int32, x values.Operand) {
var reg regs.R
if x.Storage.IsReg() {
reg = x.Reg()
} else {
mach.OpMove(code, regScratch, x, true)
reg = regScratch
}
if x.Type.Category() == types.Int {
Mov.opToIndirect(code, x.Type, reg, 0, NoIndex, regMemoryBase, offset)
} else {
MovSSE.opToIndirect(code, x.Type, reg, 0, NoIndex, regMemoryBase, offset)
}
}
// OpStoreStack must not allocate registers.
func (mach X86) OpStoreStack(code gen.Coder, offset int32, x values.Operand) {
var reg regs.R
if x.Storage.IsReg() {
reg = x.Reg()
} else {
reg = regScratch
mach.OpMove(code, reg, x, true)
}
mach.OpStoreStackReg(code, x.Type, offset, reg)
if x.Storage == values.TempReg {
code.FreeReg(x.Type, reg)
}
}
// opMaybeResultReg returns either the register of the given operand, or the
// reserved result register with the value of the operand. The caller has
// exclusive ownership of the register.
func (mach X86) opMaybeResultReg(code gen.RegCoder, x values.Operand, preserveFlags bool) (reg regs.R, zeroExt bool) {
if x.Storage == values.TempReg {
reg = x.Reg()
zeroExt = x.RegZeroExt()
} else {
var ok bool
reg, ok = code.TryAllocReg(x.Type)
if !ok {
reg = regResult
}
if x.Storage != values.Nowhere {
mach.OpMove(code, reg, x, preserveFlags)
zeroExt = true
}
}
return
}
func (mach X86) binaryShiftOp(code gen.RegCoder, index uint8, a, b values.Operand) values.Operand {
insn := binaryShiftInsns[index]
var targetReg regs.R
switch b.Storage {
case values.Imm:
targetReg, _ = mach.opMaybeResultReg(code, a, true)
insn.imm.op(code, b.Type, targetReg, uint8(b.ImmValue()))
default:
if b.Storage.IsReg() && b.Reg() == regShiftCount {
targetReg, _ = mach.opMaybeResultReg(code, a, false)
defer code.Discard(b)
} else {
if code.RegAllocated(types.I32, regShiftCount) {
targetReg, _ = mach.opMaybeResultReg(code, a, true)
if targetReg == regShiftCount {
Mov.opFromReg(code, a.Type, regResult, regShiftCount)
targetReg = regResult
defer code.FreeReg(types.I32, regShiftCount)
} else {
// unknown operand in regShiftCount
Mov.opFromReg(code, types.I64, regScratch, regShiftCount)
defer Mov.opFromReg(code, types.I64, regShiftCount, regScratch)
}
} else {
code.AllocSpecificReg(types.I32, regShiftCount)
defer code.FreeReg(types.I32, regShiftCount)
targetReg, _ = mach.opMaybeResultReg(code, a, true)
}
b.Type = types.I32 // TODO: 8-bit mov
mach.OpMove(code, regShiftCount, b, false)
}
insn.opReg(code, a.Type, targetReg)
}
return values.TempRegOperand(a.Type, targetReg, true)
}
// OpPush must not allocate registers, and must not update CPU's condition
// flags unless the operand is the condition flags.
func (mach X86) OpPush(code gen.Coder, x values.Operand) {
var reg regs.R
switch {
case x.Storage.IsReg():
reg = x.Reg()
case x.Storage == values.Imm:
value := x.ImmValue()
switch {
case value >= -0x80 && value < 0x80:
PushImm8.op(code, imm{int8(value)})
return
case value >= -0x80000000 && value < 0x80000000:
PushImm32.op(code, imm{int32(value)})
return
}
fallthrough
default:
reg = regScratch
mach.OpMove(code, reg, x, true)
}
switch x.Type.Category() {
case types.Int:
Push.op(code, reg)
case types.Float:
pushFloatOp(code, x.Type, reg)
default:
panic(x)
}
if x.Storage == values.TempReg {
code.FreeReg(x.Type, reg)
}
}
func (mach X86) binaryIntCompareOp(code gen.RegCoder, cond uint8, a, b values.Operand) (result values.Operand) {
result = values.ConditionFlagsOperand(values.Condition(cond))
targetReg, _, own := mach.opBorrowMaybeResultReg(code, a, false)
if own {
defer code.FreeReg(a.Type, targetReg)
}
if b.Storage == values.VarMem {
Cmp.opFromStack(code, a.Type, targetReg, b.VarMemOffset())
return
}
var sourceReg regs.R
if b.Storage.IsReg() {
sourceReg = b.Reg()
} else {
if b.Storage == values.Imm {
if value := b.ImmValue(); value >= -0x80000000 && value < 0x80000000 {
Cmp.opImm(code, a.Type, targetReg, int32(value))
return
}
}
sourceReg = regScratch
mach.OpMove(code, sourceReg, b, false)
}
Cmp.opFromReg(code, a.Type, targetReg, sourceReg)
code.Consumed(b)
return
}
func (mach X86) OpBranchIf(code gen.Coder, x values.Operand, yes bool, addr int32) (sites []int32) {
var cond values.Condition
if x.Storage == values.ConditionFlags {
cond = x.Condition()
} else {
reg, _, own := mach.opBorrowMaybeScratchReg(code, x, false)
if own {
defer code.FreeReg(types.I32, reg)
}
Test.opFromReg(code, types.I32, reg, reg)
cond = values.Ne
}
if !yes {
cond = values.InvertedConditions[cond]
}
var end links.L
switch {
case cond >= values.MinUnorderedOrCondition:
Jp.op(code, addr)
sites = append(sites, code.Len())
case cond >= values.MinOrderedAndCondition:
Jp.rel8.opStub(code)
end.AddSite(code.Len())
}
conditionInsns[cond].jcc.op(code, addr)
sites = append(sites, code.Len())
end.Addr = code.Len()
mach.updateBranches8(code, &end)
return
}
func (mach X86) OpSelect(code gen.RegCoder, a, b, condOperand values.Operand) values.Operand {
defer code.Consumed(condOperand)
var cond values.Condition
switch condOperand.Storage {
case values.VarMem:
Cmp.opImmToStack(code, types.I32, condOperand.VarMemOffset(), 0)
cond = values.Ne
case values.VarReg, values.TempReg:
reg := condOperand.Reg()
Test.opFromReg(code, types.I32, reg, reg)
cond = values.Ne
case values.Stack:
mach.OpAddImmToStackPtr(code, 8) // do before cmp to avoid overwriting flags
Cmp.opImmToStack(code, types.I32, -8, 0)
cond = values.Ne
case values.ConditionFlags:
cond = condOperand.Condition()
case values.Imm:
if condOperand.ImmValue() != 0 {
code.Consumed(b)
return a
} else {
code.Consumed(a)
return b
}
default:
panic(condOperand)
}
t := a.Type
targetReg, _ := mach.opMaybeResultReg(code, b, true)
switch t.Category() {
case types.Int:
cmov := conditionInsns[cond].cmov
switch a.Storage {
case values.VarMem:
cmov.opFromStack(code, t, targetReg, a.VarMemOffset())
default:
aReg, _, own := mach.opBorrowMaybeScratchReg(code, a, true)
if own {
defer code.FreeReg(t, aReg)
}
cmov.opFromReg(code, t, targetReg, aReg)
}
case types.Float:
var moveIt links.L
var end links.L
cond = values.InvertedConditions[cond]
notCondJump := conditionInsns[cond].jcc
switch {
case cond >= values.MinUnorderedOrCondition:
Jp.rel8.opStub(code) // move it if unordered
moveIt.AddSite(code.Len())
notCondJump.rel8.opStub(code) // break if not cond
end.AddSite(code.Len())
case cond >= values.MinOrderedAndCondition:
Jp.rel8.opStub(code) // break if unordered
end.AddSite(code.Len())
notCondJump.rel8.opStub(code) // break if not cond
end.AddSite(code.Len())
default:
notCondJump.rel8.opStub(code) // break if not cond
end.AddSite(code.Len())
}
moveIt.Addr = code.Len()
mach.updateBranches8(code, &moveIt)
mach.OpMove(code, targetReg, a, false)
end.Addr = code.Len()
mach.updateBranches8(code, &end)
default:
panic(t)
}
// cmov zero-extends the target unconditionally
return values.TempRegOperand(t, targetReg, true)
}
// OpMove must not update CPU's condition flags if preserveFlags is set.
//
// X86 implementation note: must not blindly rely on regScratch or regResult in
// this function because we may be moving to one of them.
func (mach X86) OpMove(code gen.Coder, targetReg regs.R, x values.Operand, preserveFlags bool) (zeroExt bool) {
switch x.Type.Category() {
case types.Int:
switch x.Storage {
case values.Imm:
if value := x.ImmValue(); value == 0 && !preserveFlags {
Xor.opFromReg(code, types.I32, targetReg, targetReg)
} else {
MovImm64.op(code, x.Type, targetReg, value)
}
zeroExt = true
case values.VarMem:
Mov.opFromStack(code, x.Type, targetReg, x.VarMemOffset())
zeroExt = true
case values.VarReg:
if sourceReg := x.Reg(); sourceReg != targetReg {
Mov.opFromReg(code, x.Type, targetReg, sourceReg)
zeroExt = true
}
case values.TempReg:
if sourceReg := x.Reg(); sourceReg != targetReg {
Mov.opFromReg(code, x.Type, targetReg, sourceReg)
zeroExt = true
} else if targetReg == regResult {
zeroExt = x.RegZeroExt()
} else {
panic("moving temporary integer register to itself")
}
case values.Stack:
Pop.op(code, targetReg)
case values.ConditionFlags:
if x.Type != types.I32 {
panic(x)
}
var end links.L
cond := x.Condition()
setcc := conditionInsns[cond].setcc
switch {
case cond >= values.MinUnorderedOrCondition:
MovImm.opImm(code, x.Type, targetReg, 1) // true
Jp.rel8.opStub(code) // if unordered, else
end.AddSite(code.Len()) //
setcc.opReg(code, targetReg) // cond
case cond >= values.MinOrderedAndCondition:
MovImm.opImm(code, x.Type, targetReg, 0) // false
Jp.rel8.opStub(code) // if unordered, else
end.AddSite(code.Len()) //
setcc.opReg(code, targetReg) // cond
default:
setcc.opReg(code, targetReg)
Movzx8.opFromReg(code, x.Type, targetReg, targetReg)
}
end.Addr = code.Len()
mach.updateBranches8(code, &end)
zeroExt = true
default:
panic(x)
}
case types.Float:
switch x.Storage {
case values.Imm:
if value := x.ImmValue(); value == 0 {
PxorSSE.opFromReg(code, x.Type, targetReg, targetReg)
} else {
MovImm64.op(code, x.Type, regScratch, value) // integer scratch register
MovSSE.opFromReg(code, x.Type, targetReg, regScratch)
}
case values.VarMem:
MovsSSE.opFromStack(code, x.Type, targetReg, x.VarMemOffset())
case values.VarReg:
if sourceReg := x.Reg(); sourceReg != targetReg {
MovsSSE.opFromReg(code, x.Type, targetReg, sourceReg)
}
case values.TempReg:
if sourceReg := x.Reg(); sourceReg != targetReg {
MovsSSE.opFromReg(code, x.Type, targetReg, sourceReg)
} else if targetReg != regResult {
panic("moving temporary float register to itself")
}
case values.Stack:
popFloatOp(code, x.Type, targetReg)
default:
panic(x)
}
default:
panic(x)
}
code.Consumed(x)
return
}
func (mach X86) binaryIntOp(code gen.RegCoder, index uint8, a, b values.Operand) (result values.Operand) {
if a.Storage == values.Imm && a.ImmValue() == 0 && index == opers.IndexIntSub {
targetReg, _ := mach.opMaybeResultReg(code, b, false)
Neg.opReg(code, a.Type, targetReg)
return values.TempRegOperand(a.Type, targetReg, true)
}
switch b.Storage {
case values.Imm:
value := b.ImmValue()
switch {
case index == opers.IndexIntAdd && value == 1: // assume that we won't see sub -1
reg, _ := mach.opMaybeResultReg(code, a, false)
Inc.opReg(code, a.Type, reg)
return values.TempRegOperand(a.Type, reg, true)
case index == opers.IndexIntSub && value == 1: // assume that we won't see add -1
reg, _ := mach.opMaybeResultReg(code, a, false)
Dec.opReg(code, a.Type, reg)
return values.TempRegOperand(a.Type, reg, true)
case value < -0x80000000 || value >= 0x80000000:
// TODO: merge this with the next outer case
sourceReg, _, own := mach.opBorrowMaybeScratchReg(code, b, true)
b = values.RegOperand(own, a.Type, sourceReg)
}
case values.Stack, values.ConditionFlags:
sourceReg, _, own := mach.opBorrowMaybeScratchReg(code, b, true)
b = values.RegOperand(own, a.Type, sourceReg)
}
insn := binaryIntInsns[index]
targetReg, _ := mach.opMaybeResultReg(code, a, false)
result = values.TempRegOperand(a.Type, targetReg, true)
if b.Storage == values.VarMem {
insn.opFromStack(code, a.Type, targetReg, b.VarMemOffset())
return
}
var sourceReg regs.R
if b.Storage.IsReg() {
sourceReg = b.Reg()
} else {
if b.Storage == values.Imm {
if value := b.ImmValue(); value >= -0x80000000 && value < 0x80000000 {
insn.opImm(code, a.Type, targetReg, int32(b.ImmValue()))
return
}
}
sourceReg = regScratch
mach.OpMove(code, sourceReg, b, false)
}
insn.opFromReg(code, a.Type, targetReg, sourceReg)
code.Consumed(b)
return
}
func (mach X86) binaryDivmulOp(code gen.RegCoder, index uint8, a, b values.Operand) values.Operand {
insn := binaryDivmulInsns[index]
t := a.Type
if b.Storage == values.Imm {
value := b.ImmValue()
switch {
case value == -1:
reg, _ := mach.opMaybeResultReg(code, a, false)
Neg.opReg(code, t, reg)
return values.TempRegOperand(t, reg, true)
case insn.shiftImm.defined() && value > 0 && isPowerOfTwo(uint64(value)):
reg, _ := mach.opMaybeResultReg(code, a, false)
insn.shiftImm.op(code, t, reg, log2(uint64(value)))
return values.TempRegOperand(t, reg, true)
}
}
division := (index & opers.DivmulMul) == 0
checkZero := true
checkOverflow := true
if b.Storage.IsReg() {
if b.Reg() == regResult {
newReg := regScratch
if division {
var ok bool
// can't use scratch reg as divisor since it contains the dividend high bits
newReg, ok = code.TryAllocReg(t)
if !ok {
// borrow a register which we don't need in this function
MovMMX.opFromReg(code, types.I64, regScratchMMX, regTextBase)
defer MovMMX.opToReg(code, types.I64, regTextBase, regScratchMMX)
newReg = regTextBase
}
}
Mov.opFromReg(code, t, newReg, regResult)
b = values.RegOperand(true, t, newReg)
}
} else {
if division && b.Storage == values.Imm {
value := b.ImmValue()
if value != 0 {
checkZero = false
}
if value != -1 {
checkOverflow = false
}
}
reg, ok := code.TryAllocReg(t)
if !ok {
// borrow a register which we don't need in this function
MovMMX.opFromReg(code, types.I64, regScratchMMX, regTextBase)
defer MovMMX.opToReg(code, types.I64, regTextBase, regScratchMMX)
reg = regTextBase
}
mach.OpMove(code, reg, b, true)
b = values.RegOperand(true, t, reg)
}
mach.OpMove(code, regResult, a, false)
remainder := (index & opers.DivmulRem) != 0
var doNot links.L
if division {
if checkZero {
mach.opCheckDivideByZero(code, t, b.Reg())
}
if a.Storage == values.Imm {
value := a.ImmValue()
if t.Size() == types.Size32 {
if value != -0x80000000 {
checkOverflow = false
}
} else {
if value != -0x8000000000000000 {
checkOverflow = false
}
}
}
signed := (index & opers.DivmulSign) != 0
if signed && checkOverflow {
var do links.L
if remainder {
Xor.opFromReg(code, types.I32, regScratch, regScratch) // moved to result at the end
Cmp.opImm(code, t, b.Reg(), -1)
Je.rel8.opStub(code)
doNot.AddSite(code.Len())
} else {
switch t.Size() {
case types.Size32:
Cmp.opImm(code, t, regResult, -0x80000000)
case types.Size64:
MovImm64.op(code, t, regScratch, -0x8000000000000000)
Cmp.opFromReg(code, t, regResult, regScratch)
default:
panic(a)
}
Jne.rel8.opStub(code)
do.AddSite(code.Len())
Cmp.opImm(code, t, b.Reg(), -1)
Jne.rel8.opStub(code)
do.AddSite(code.Len())
code.OpTrapCall(traps.IntegerOverflow)
}
do.Addr = code.Len()
mach.updateBranches8(code, &do)
}
if signed {
// sign-extend dividend low bits to high bits
CdqCqo.op(code, t)
} else {
// zero-extend dividend high bits
Xor.opFromReg(code, types.I32, regScratch, regScratch)
}
}
insn.opReg(code, t, b.Reg())
code.Consumed(b)
doNot.Addr = code.Len()
mach.updateBranches8(code, &doNot)
if remainder {
Mov.opFromReg(code, t, regResult, regScratch)
}
return values.TempRegOperand(t, regResult, true)
}