Beispiel #1
0
func testRBatcher(t *testing.T, rv autofunc.RVector, b batchFuncR, in autofunc.RResult,
	n int, params []*autofunc.Variable) {
	funcRBatcher := autofunc.RFuncBatcher{F: b}

	t.Run("Forward", func(t *testing.T) {
		expected := funcRBatcher.BatchR(rv, in, n)
		actual := b.BatchR(rv, in, n)
		diff := actual.Output().Copy().Scale(-1).Add(expected.Output()).MaxAbs()
		if diff > 1e-5 {
			t.Errorf("expected output %v but got %v", expected, actual)
		}
		diff = actual.ROutput().Copy().Scale(-1).Add(expected.ROutput()).MaxAbs()
		if diff > 1e-5 {
			t.Errorf("expected r-output %v but got %v", expected, actual)
		}
	})

	t.Run("Backward", func(t *testing.T) {
		expectedOut := funcRBatcher.BatchR(rv, in, n)
		actualOut := b.BatchR(rv, in, n)

		expected := autofunc.NewGradient(params)
		actual := autofunc.NewGradient(params)
		expectedR := autofunc.NewRGradient(params)
		actualR := autofunc.NewRGradient(params)

		outGrad := make(linalg.Vector, len(expectedOut.Output()))
		outGradR := make(linalg.Vector, len(expectedOut.Output()))
		for i := range outGrad {
			outGrad[i] = rand.NormFloat64()
			outGradR[i] = rand.NormFloat64()
		}

		expectedOut.PropagateRGradient(outGrad.Copy(), outGradR.Copy(), expectedR, expected)
		actualOut.PropagateRGradient(outGrad, outGradR, actualR, actual)

		for i, variable := range params {
			actualVec := actual[variable]
			expectedVec := expected[variable]
			diff := actualVec.Copy().Scale(-1).Add(expectedVec).MaxAbs()
			if diff > 1e-5 {
				t.Errorf("variable %d (grad): expected %v got %v", i, expectedVec, actualVec)
			}
			actualVec = actualR[variable]
			expectedVec = expectedR[variable]
			diff = actualVec.Copy().Scale(-1).Add(expectedVec).MaxAbs()
			if diff > 1e-5 {
				t.Errorf("variable %d (rgrad): expected %v got %v", i, expectedVec, actualVec)
			}
		}
	})
}
Beispiel #2
0
func (b *BlockChecker) testNilUpstreamR(t *testing.T) {
	t.Run("Nil Upstream R", func(t *testing.T) {
		out := b.B.ApplyBlockR(b.RV, []rnn.RState{b.B.StartRState(b.RV)},
			[]autofunc.RResult{autofunc.NewRVariable(b.Input[0][0], b.RV)})
		g1 := autofunc.NewGradient(b.Vars)
		rg1 := autofunc.NewRGradient(b.Vars)
		initLen1 := len(g1)
		c := len(out.PropagateRGradient(nil, nil, nil, rg1, g1))
		if c != 1 {
			t.Errorf("expected %d downstream states, got %d", 1, c)
		}
		g2 := autofunc.NewGradient(b.Vars)
		rg2 := autofunc.NewRGradient(b.Vars)
		initLen2 := len(g2)

		zeroUpstream := make([]linalg.Vector, len(out.Outputs()))
		for i, x := range out.Outputs() {
			zeroUpstream[i] = make(linalg.Vector, len(x))
		}
		nilStateUpstream := make([]rnn.RStateGrad, len(out.RStates()))
		c = len(out.PropagateRGradient(zeroUpstream, zeroUpstream, nilStateUpstream, rg2, g2))
		if c != 1 {
			t.Errorf("expected %d downstream states, got %d", 1, c)
		}

		if len(g1) != initLen1 {
			t.Errorf("all nil gradient length changed from %d to %d", initLen1, len(g1))
		}
		if len(rg1) != initLen1 {
			t.Errorf("all nil r-gradient length changed from %d to %d", initLen1, len(rg1))
		}
		if len(g2) != initLen2 {
			t.Errorf("non-nil gradient length changed from %d to %d", initLen2, len(g2))
		}
		if len(rg2) != initLen2 {
			t.Errorf("non-nil r-gradient length changed from %d to %d", initLen2, len(rg2))
		}

		for i, variable := range b.Vars {
			val1 := g1[variable]
			val2 := g2[variable]
			if !b.vecsEqual(val1, val2) {
				t.Errorf("gradients for var %d don't match: %v and %v", i, val1, val2)
			}
			val1 = rg1[variable]
			val2 = rg2[variable]
			if !b.vecsEqual(val1, val2) {
				t.Errorf("r-gradients for var %d don't match: %v and %v", i, val1, val2)
			}
		}
	})
}
func (b *SingleRGradienter) RGradient(rv autofunc.RVector, s sgd.SampleSet) (autofunc.Gradient,
	autofunc.RGradient) {
	if b.gradCache == nil {
		b.gradCache = autofunc.NewGradient(b.Learner.Parameters())
	} else {
		b.gradCache.Zero()
	}
	if b.rgradCache == nil {
		b.rgradCache = autofunc.NewRGradient(b.Learner.Parameters())
	} else {
		b.rgradCache.Zero()
	}

	for i := 0; i < s.Len(); i++ {
		sample := s.GetSample(i)
		vs := sample.(VectorSample)
		output := vs.Output
		inVar := &autofunc.Variable{vs.Input}
		rVar := autofunc.NewRVariable(inVar, rv)
		result := b.Learner.ApplyR(rv, rVar)
		cost := b.CostFunc.CostR(rv, output, result)
		cost.PropagateRGradient(linalg.Vector{1}, linalg.Vector{0},
			b.rgradCache, b.gradCache)
	}

	return b.gradCache, b.rgradCache
}
Beispiel #4
0
func (g *gradientCache) AllocR() autofunc.RGradient {
	if len(g.rGradients) == 0 {
		res := autofunc.NewRGradient(g.variables)
		return res
	}
	res := g.rGradients[len(g.rGradients)-1]
	g.rGradients = g.rGradients[:len(g.rGradients)-1]
	res.Zero()
	return res
}