diff --git a/GPy/mappings/__init__.py b/GPy/mappings/__init__.py index d331c678..b1cb194b 100644 --- a/GPy/mappings/__init__.py +++ b/GPy/mappings/__init__.py @@ -4,4 +4,5 @@ from kernel import Kernel from linear import Linear from mlp import MLP -#from rbf import RBF +from additive import Additive +from compound import Compound diff --git a/GPy/mappings/additive.py b/GPy/mappings/additive.py index 4e7c545d..1c86b680 100644 --- a/GPy/mappings/additive.py +++ b/GPy/mappings/additive.py @@ -2,8 +2,7 @@ # Licensed under the BSD 3-clause license (see LICENSE.txt) import numpy as np -from ..core.mapping import Mapping -import GPy +from ..core import Mapping class Additive(Mapping): """ @@ -27,8 +26,6 @@ class Additive(Mapping): Mapping.__init__(self, input_dim=input_dim, output_dim=output_dim) self.mapping1 = mapping1 self.mapping2 = mapping2 - self.num_params = self.mapping1.num_params + self.mapping2.num_params - self.name = self.mapping1.name + '+' + self.mapping2.name def f(self, X): return self.mapping1.f(X) + self.mapping2.f(X) diff --git a/GPy/mappings/compound.py b/GPy/mappings/compound.py new file mode 100644 index 00000000..5a1e8dd1 --- /dev/null +++ b/GPy/mappings/compound.py @@ -0,0 +1,39 @@ +# Copyright (c) 2015, James Hensman and Alan Saul +# Licensed under the BSD 3-clause license (see LICENSE.txt) + +from ..core import Mapping + +class Compound(Mapping): + """ + Mapping based on passing one mapping through another + + .. math:: + + f(\mathbf{x}) = f_2(f_1(\mathbf{x})) + + :param mapping1: first mapping + :type mapping1: GPy.mappings.Mapping + :param mapping2: second mapping + :type mapping2: GPy.mappings.Mapping + + """ + + def __init__(self, mapping1, mapping2): + assert(mapping1.output_dim==mapping2.input_dim) + input_dim, output_dim = mapping1.input_dim, mapping2.output_dim + Mapping.__init__(self, input_dim=input_dim, output_dim=output_dim) + self.mapping1 = mapping1 + self.mapping2 = mapping2 + self.link_parameters(self.mapping1, self.mapping2) + + def f(self, X): + return self.mapping2.f(self.mapping1.f(X)) + + def update_gradients(self, dL_dF, X): + hidden = self.mapping1.f(X) + self.mapping2.update_gradients(dL_dF, hidden) + self.mapping1.update_gradients(self.mapping2.gradients_X(dL_dF, hidden), X) + + def gradients_X(self, dL_dF, X): + hidden = self.mapping1.f(X) + return self.mapping1.gradients_X(self.mapping2.gradients_X(dL_dF, hidden), X) diff --git a/GPy/mappings/kernel.py b/GPy/mappings/kernel.py index 3bfcd388..ea1720db 100644 --- a/GPy/mappings/kernel.py +++ b/GPy/mappings/kernel.py @@ -36,16 +36,16 @@ class Kernel(Mapping): Mapping.__init__(self, input_dim=input_dim, output_dim=output_dim, name=name) self.kern = kernel self.Z = Z - self.num_bases, Zdim = X.shape + self.num_bases, Zdim = Z.shape assert Zdim == self.input_dim - self.A = GPy.core.Param('A', np.random.randn(self.num_bases, self.output_dim)) - self.add_parameter(self.A) + self.A = Param('A', np.random.randn(self.num_bases, self.output_dim)) + self.link_parameter(self.A) def f(self, X): return np.dot(self.kern.K(X, self.Z), self.A) def update_gradients(self, dL_dF, X): - self.kern.update_gradients_full(np.dot(dL_dF, self.A.T)) + self.kern.update_gradients_full(np.dot(dL_dF, self.A.T), X, self.Z) self.A.gradient = np.dot( self.kern.K(self.Z, X), dL_dF) def gradients_X(self, dL_dF, X): diff --git a/GPy/mappings/mlp.py b/GPy/mappings/mlp.py index f22fc07f..f0fe21e5 100644 --- a/GPy/mappings/mlp.py +++ b/GPy/mappings/mlp.py @@ -11,32 +11,45 @@ class MLP(Mapping): """ def __init__(self, input_dim=1, output_dim=1, hidden_dim=3, name='mlpmap'): - super(MLP).__init__(self, input_dim=input_dim, output_dim=output_dim, name=name) + super(MLP, self).__init__(input_dim=input_dim, output_dim=output_dim, name=name) self.hidden_dim = hidden_dim self.W1 = Param('W1', np.random.randn(self.input_dim, self.hidden_dim)) self.b1 = Param('b1', np.random.randn(self.hidden_dim)) self.W2 = Param('W2', np.random.randn(self.hidden_dim, self.output_dim)) self.b2 = Param('b2', np.random.randn(self.output_dim)) + self.link_parameters(self.W1, self.b1, self.W2, self.b2) def f(self, X): - N, D = X.shape - activations = np.tanh(np.dot(X,self.W1) + self.b1) - self.out = np.dot(self.activations,self.W2) + self.b2 - return self.output_fn(self.out) + layer1 = np.dot(X, self.W1) + self.b1 + activations = np.tanh(layer1) + return np.dot(activations, self.W2) + self.b2 def update_gradients(self, dL_dF, X): - activations = np.tanh(np.dot(X,self.W1) + self.b1) - + layer1 = np.dot(X,self.W1) + self.b1 + activations = np.tanh(layer1) #Evaluate second-layer gradients. self.W2.gradient = np.dot(activations.T, dL_dF) self.b2.gradient = np.sum(dL_dF, 0) # Backpropagation to hidden layer. - delta_hid = np.dot(dL_dF, self.W2.T) * (1.0 - activations**2) + dL_dact = np.dot(dL_dF, self.W2.T) + dL_dlayer1 = dL_dact / np.square(np.cosh(layer1)) # Finally, evaluate the first-layer gradients. - self.W1.gradients = np.dot(X.T,delta_hid) - self.b1.gradients = np.sum(delta_hid, 0) + self.W1.gradient = np.dot(X.T,dL_dlayer1) + self.b1.gradient = np.sum(dL_dlayer1, 0) + + def gradients_X(self, dL_dF, X): + layer1 = np.dot(X,self.W1) + self.b1 + activations = np.tanh(layer1) + + # Backpropagation to hidden layer. + dL_dact = np.dot(dL_dF, self.W2.T) + dL_dlayer1 = dL_dact / np.square(np.cosh(layer1)) + + return np.dot(dL_dlayer1, self.W1.T) + + diff --git a/GPy/testing/mapping_tests.py b/GPy/testing/mapping_tests.py new file mode 100644 index 00000000..2e32dad3 --- /dev/null +++ b/GPy/testing/mapping_tests.py @@ -0,0 +1,72 @@ +# Copyright (c) 2012, 2013 GPy authors (see AUTHORS.txt). +# Licensed under the BSD 3-clause license (see LICENSE.txt) + +import unittest +import numpy as np +import GPy + +class MappingGradChecker(GPy.core.Model): + """ + This class has everything we need to check the gradient of a mapping. It + implement a simple likelihood which is a weighted sum of the outputs of the + mapping. the gradients are checked against the parameters of the mapping + and the input. + """ + def __init__(self, mapping, X, name='map_grad_check'): + super(MappingGradChecker, self).__init__(name) + self.mapping = mapping + self.link_parameter(self.mapping) + self.X = GPy.core.Param('X',X) + self.link_parameter(self.X) + self.dL_dY = np.random.randn(self.X.shape[0], self.mapping.output_dim) + def log_likelihood(self): + return np.sum(self.mapping.f(self.X) * self.dL_dY) + def parameters_changed(self): + self.X.gradient = self.mapping.gradients_X(self.dL_dY, self.X) + self.mapping.update_gradients(self.dL_dY, self.X) + + + + + + + +class MappingTests(unittest.TestCase): + + def test_kernelmapping(self): + X = np.random.randn(100,3) + Z = np.random.randn(10,3) + mapping = GPy.mappings.Kernel(3, 2, Z, GPy.kern.RBF(3)) + self.assertTrue(MappingGradChecker(mapping, X).checkgrad()) + + def test_linearmapping(self): + mapping = GPy.mappings.Linear(3, 2) + X = np.random.randn(100,3) + self.assertTrue(MappingGradChecker(mapping, X).checkgrad()) + + def test_mlpmapping(self): + mapping = GPy.mappings.MLP(input_dim=3, hidden_dim=5, output_dim=2) + X = np.random.randn(100,3) + self.assertTrue(MappingGradChecker(mapping, X).checkgrad()) + + def test_addmapping(self): + m1 = GPy.mappings.MLP(input_dim=3, hidden_dim=5, output_dim=2) + m2 = GPy.mappings.Linear(input_dim=3, output_dim=2) + mapping = GPy.mappings.Additive(m1, m2) + X = np.random.randn(100,3) + self.assertTrue(MappingGradChecker(mapping, X).checkgrad()) + + def test_compoundmapping(self): + m1 = GPy.mappings.MLP(input_dim=3, hidden_dim=5, output_dim=2) + Z = np.random.randn(10,2) + m2 = GPy.mappings.Kernel(2, 4, Z, GPy.kern.RBF(2)) + mapping = GPy.mappings.Compound(m1, m2) + X = np.random.randn(100,3) + self.assertTrue(MappingGradChecker(mapping, X).checkgrad()) + + + + +if __name__ == "__main__": + print "Running unit tests, please be (very) patient..." + unittest.main()