func (_ SigmoidCECost) Cost(x linalg.Vector, a autofunc.Result) autofunc.Result { logsig := autofunc.LogSigmoid{} log := logsig.Apply(a) invLog := logsig.Apply(autofunc.Scale(a, -1)) xVar := &autofunc.Variable{x} oneMinusX := autofunc.AddScaler(autofunc.Scale(xVar, -1), 1) sums := autofunc.Add(autofunc.Mul(xVar, log), autofunc.Mul(oneMinusX, invLog)) return autofunc.Scale(autofunc.SumAll(sums), -1) }
func (_ CrossEntropyCost) Cost(x linalg.Vector, a autofunc.Result) autofunc.Result { return autofunc.Pool(a, func(a autofunc.Result) autofunc.Result { xVar := &autofunc.Variable{x} logA := autofunc.Log{}.Apply(a) oneMinusA := autofunc.AddScaler(autofunc.Scale(a, -1), 1) oneMinusX := autofunc.AddScaler(autofunc.Scale(xVar, -1), 1) log1A := autofunc.Log{}.Apply(oneMinusA) errorVec := autofunc.Add(autofunc.Mul(xVar, logA), autofunc.Mul(oneMinusX, log1A)) return autofunc.Scale(autofunc.SumAll(errorVec), -1) }) }
func (s *LogSoftmaxLayer) Apply(in autofunc.Result) autofunc.Result { return autofunc.Pool(in, func(in autofunc.Result) autofunc.Result { // Compute the log of the sum of the exponents by // factoring out the largest exponent so that all // the exponentials fit nicely inside floats. maxIdx := maxVecIdx(in.Output()) maxValue := autofunc.Slice(in, maxIdx, maxIdx+1) exponents := autofunc.AddFirst(in, autofunc.Scale(maxValue, -1)) expSum := autofunc.SumAll(autofunc.Exp{}.Apply(exponents)) expLog := autofunc.Log{}.Apply(expSum) denomLog := autofunc.Add(expLog, maxValue) return autofunc.AddFirst(in, autofunc.Scale(denomLog, -1)) }) }
func (d *DropoutLayer) Apply(in autofunc.Result) autofunc.Result { if d.Training { return autofunc.Mul(in, d.dropoutMask(len(in.Output()))) } else { return autofunc.Scale(in, d.KeepProbability) } }
func (r *RegularizingCost) Cost(a linalg.Vector, x autofunc.Result) autofunc.Result { regFunc := autofunc.SquaredNorm{} cost := r.CostFunc.Cost(a, x) for _, variable := range r.Variables { norm := regFunc.Apply(variable) cost = autofunc.Add(cost, autofunc.Scale(norm, r.Penalty)) } return cost }
// ApplyBlock applies the block to an input. func (g *GRU) ApplyBlock(s []State, in []autofunc.Result) BlockResult { stateVars, stateRes := PoolVecStates(s) var gateInputs []autofunc.Result for i, x := range stateRes { gateInputs = append(gateInputs, in[i], x) } n := len(in) gateInput := autofunc.Concat(gateInputs...) stateIn := autofunc.Concat(stateRes...) resetMask := g.resetGate.Batch(gateInput, n) updateMask := g.updateGate.Batch(gateInput, n) maskedByReset := autofunc.Mul(resetMask, stateIn) inputValue := autofunc.PoolSplit(n, maskedByReset, func(newStates []autofunc.Result) autofunc.Result { var newGateInputs []autofunc.Result for i, input := range in { newGateInputs = append(newGateInputs, input, newStates[i]) } newIn := autofunc.Concat(newGateInputs...) return g.inputValue.Batch(newIn, n) }) newState := autofunc.Pool(updateMask, func(umask autofunc.Result) autofunc.Result { updateComplement := autofunc.AddScaler(autofunc.Scale(umask, -1), 1) return autofunc.Add(autofunc.Mul(umask, stateIn), autofunc.Mul(updateComplement, inputValue)) }) return &gruResult{ InStates: stateVars, Output: newState, } }
func (r *RescaleLayer) Apply(in autofunc.Result) autofunc.Result { return autofunc.Scale(autofunc.AddScaler(in, r.Bias), r.Scale) }
func (_ HyperbolicTangent) Apply(r autofunc.Result) autofunc.Result { stretched := autofunc.Scale(autofunc.Sigmoid{}.Apply(autofunc.Scale(r, 2)), 2) return autofunc.AddScaler(stretched, -1) }
func (_ DotCost) Cost(x linalg.Vector, a autofunc.Result) autofunc.Result { xVar := &autofunc.Variable{x} return autofunc.Scale(autofunc.SumAll(autofunc.Mul(xVar, a)), -1) }