From d71aabceb50bd0a50f9105c569fd499adcfd2069 Mon Sep 17 00:00:00 2001 From: Alan Saul Date: Mon, 11 Mar 2013 14:58:42 +0000 Subject: [PATCH 01/12] Changed example tests --- GPy/testing/examples_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPy/testing/examples_tests.py b/GPy/testing/examples_tests.py index 25cfad04..0e41a541 100644 --- a/GPy/testing/examples_tests.py +++ b/GPy/testing/examples_tests.py @@ -13,7 +13,7 @@ class ExamplesTests(unittest.TestCase): pass def test_all_examples(self): - pass + examples_module = __import__("GPy").examples #Load models #Loop through models From c39af496a633b44b2e42356a444dba81c6630b65 Mon Sep 17 00:00:00 2001 From: Alan Saul Date: Mon, 11 Mar 2013 17:05:18 +0000 Subject: [PATCH 02/12] Added test generator (not quite finished yet) --- GPy/examples/regression.py | 4 +-- GPy/examples/tutorials.py | 18 +++------- GPy/testing/examples_tests.py | 62 ++++++++++++++++++++++++++++------- 3 files changed, 58 insertions(+), 26 deletions(-) diff --git a/GPy/examples/regression.py b/GPy/examples/regression.py index 6c22b68e..f5d0d3b1 100644 --- a/GPy/examples/regression.py +++ b/GPy/examples/regression.py @@ -194,7 +194,7 @@ def multiple_optima(gene_number=937,resolution=80, model_restarts=10, seed=10000 # Remove the mean (no bias kernel to ensure signal/noise is in RBF/white) data['Y'] = data['Y'] - np.mean(data['Y']) - lls = GPy.examples.regression.contour_data(data, length_scales, log_SNRs, GPy.kern.rbf) + lls = GPy.examples.regression._contour_data(data, length_scales, log_SNRs, GPy.kern.rbf) pb.contour(length_scales, log_SNRs, np.exp(lls), 20) ax = pb.gca() pb.xlabel('length scale') @@ -229,7 +229,7 @@ def multiple_optima(gene_number=937,resolution=80, model_restarts=10, seed=10000 ax.set_ylim(ylim) return (models, lls) -def contour_data(data, length_scales, log_SNRs, signal_kernel_call=GPy.kern.rbf): +def _contour_data(data, length_scales, log_SNRs, signal_kernel_call=GPy.kern.rbf): """Evaluate the GP objective function for a given data set for a range of signal to noise ratios and a range of lengthscales. :data_set: A data set from the utils.datasets director. diff --git a/GPy/examples/tutorials.py b/GPy/examples/tutorials.py index 9d892b8e..a199aba9 100644 --- a/GPy/examples/tutorials.py +++ b/GPy/examples/tutorials.py @@ -6,14 +6,14 @@ Code of Tutorials """ +import pylab as pb +pb.ion() +import numpy as np +import GPy + def tuto_GP_regression(): """The detailed explanations of the commands used in this file can be found in the tutorial section""" - import pylab as pb - pb.ion() - import numpy as np - import GPy - X = np.random.uniform(-3.,3.,(20,1)) Y = np.sin(X) + np.random.randn(20,1)*0.05 @@ -39,11 +39,6 @@ def tuto_GP_regression(): # 2-dimensional example # ########################### - import pylab as pb - pb.ion() - import numpy as np - import GPy - # sample inputs and outputs X = np.random.uniform(-3.,3.,(50,2)) Y = np.sin(X[:,0:1]) * np.sin(X[:,1:2])+np.random.randn(50,1)*0.05 @@ -67,9 +62,6 @@ def tuto_GP_regression(): def tuto_kernel_overview(): """The detailed explanations of the commands used in this file can be found in the tutorial section""" - import pylab as pb - import numpy as np - import GPy pb.ion() ker1 = GPy.kern.rbf(1) # Equivalent to ker1 = GPy.kern.rbf(D=1, variance=1., lengthscale=1.) diff --git a/GPy/testing/examples_tests.py b/GPy/testing/examples_tests.py index 0e41a541..967d7a6a 100644 --- a/GPy/testing/examples_tests.py +++ b/GPy/testing/examples_tests.py @@ -4,23 +4,63 @@ import unittest import numpy as np import GPy +import inspect +import pkgutil +import os + class ExamplesTests(unittest.TestCase): - def test_check_model_returned(self): - pass + def _checkgrad(self, model): + self.assertTrue(model.checkgrad()) - def test_model_checkgrads(self): - pass + def _model_instance(self, model): + self.assertTrue(isinstance(model, GPy.models)) - def test_all_examples(self): - examples_module = __import__("GPy").examples - #Load models +""" +def model_instance_generator(model): + def check_model_returned(self): + self._model_instance(model) + return check_model_returned - #Loop through models - #for model in models: - #self.assertTrue(m.checkgrad()) +def checkgrads_generator(model): + def model_checkgrads(self): + self._checkgrad(model) + return model_checkgrads +""" +def model_checkgrads(model): + assert model.checkgrad() is True +def model_instance(model): + assert model.checkgrad() is True + +def test_models(): + examples_path = os.path.dirname(GPy.examples.__file__) + #Load modules + for loader, module_name, is_pkg in pkgutil.iter_modules([examples_path]): + #Load examples + module_examples = loader.find_module(module_name).load_module(module_name) + functions = [ func for func in [inspect.getmembers(module_examples, predicate=inspect.isfunction)[0]] if func[0].startswith('_') is False ] + for example in functions: + print "Testing example: ", example[0] + #Generate model + model = example[1]() + print model + + #Create tests for instance check + """ + test = model_instance_generator(model) + test.__name__ = 'test_instance_%s' % example[0] + setattr(ExamplesTests, test.__name__, test) + + #Create tests for checkgrads check + test = checkgrads_generator(model) + test.__name__ = 'test_checkgrads_%s' % example[0] + setattr(ExamplesTests, test.__name__, test) + """ + model_checkgrads.description = 'test_checkgrads_%s' % example[0] + yield model_checkgrads, model + model_instance.description = 'test_checkgrads_%s' % example[0] + yield model_instance, model if __name__ == "__main__": print "Running unit tests, please be (very) patient..." - unittest.main() From 5c768231eb94c538ecbe87dc70bc124ffb9fbf1e Mon Sep 17 00:00:00 2001 From: Alan Saul Date: Mon, 11 Mar 2013 17:35:01 +0000 Subject: [PATCH 03/12] Add example test generation --- GPy/testing/examples_tests.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/GPy/testing/examples_tests.py b/GPy/testing/examples_tests.py index 967d7a6a..9636b286 100644 --- a/GPy/testing/examples_tests.py +++ b/GPy/testing/examples_tests.py @@ -27,19 +27,27 @@ def checkgrads_generator(model): self._checkgrad(model) return model_checkgrads """ + def model_checkgrads(model): assert model.checkgrad() is True + def model_instance(model): assert model.checkgrad() is True + def test_models(): examples_path = os.path.dirname(GPy.examples.__file__) #Load modules for loader, module_name, is_pkg in pkgutil.iter_modules([examples_path]): #Load examples module_examples = loader.find_module(module_name).load_module(module_name) - functions = [ func for func in [inspect.getmembers(module_examples, predicate=inspect.isfunction)[0]] if func[0].startswith('_') is False ] + print "MODULE", module_examples + print "Before" + print inspect.getmembers(module_examples, predicate=inspect.isfunction) + functions = [ func for func in inspect.getmembers(module_examples, predicate=inspect.isfunction) if func[0].startswith('_') is False ] + print "After" + print functions for example in functions: print "Testing example: ", example[0] #Generate model @@ -59,7 +67,7 @@ def test_models(): """ model_checkgrads.description = 'test_checkgrads_%s' % example[0] yield model_checkgrads, model - model_instance.description = 'test_checkgrads_%s' % example[0] + model_instance.description = 'test_instance_%s' % example[0] yield model_instance, model if __name__ == "__main__": From e6acaf651e5798250c1f28265714e0b9edb850da Mon Sep 17 00:00:00 2001 From: Alan Saul Date: Mon, 11 Mar 2013 18:05:25 +0000 Subject: [PATCH 04/12] Should now test all (although upon error it stops trying to generate any more) --- GPy/testing/examples_tests.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/GPy/testing/examples_tests.py b/GPy/testing/examples_tests.py index 9636b286..5ae741ee 100644 --- a/GPy/testing/examples_tests.py +++ b/GPy/testing/examples_tests.py @@ -29,11 +29,11 @@ def checkgrads_generator(model): """ def model_checkgrads(model): - assert model.checkgrad() is True + assert model.checkgrad() def model_instance(model): - assert model.checkgrad() is True + assert isinstance(model, GPy.core.model) def test_models(): @@ -45,7 +45,7 @@ def test_models(): print "MODULE", module_examples print "Before" print inspect.getmembers(module_examples, predicate=inspect.isfunction) - functions = [ func for func in inspect.getmembers(module_examples, predicate=inspect.isfunction) if func[0].startswith('_') is False ] + functions = [ func for func in inspect.getmembers(module_examples, predicate=inspect.isfunction) if func[0].startswith('_') is False ][::-1] print "After" print functions for example in functions: @@ -72,3 +72,4 @@ def test_models(): if __name__ == "__main__": print "Running unit tests, please be (very) patient..." + unittest.main() From 62615f2ca891229bc9bb883bdfacecb00f17418b Mon Sep 17 00:00:00 2001 From: Alan Saul Date: Mon, 11 Mar 2013 18:09:47 +0000 Subject: [PATCH 05/12] Trying to shuffle --- GPy/testing/examples_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/GPy/testing/examples_tests.py b/GPy/testing/examples_tests.py index 5ae741ee..07ef4177 100644 --- a/GPy/testing/examples_tests.py +++ b/GPy/testing/examples_tests.py @@ -7,6 +7,7 @@ import GPy import inspect import pkgutil import os +import random class ExamplesTests(unittest.TestCase): @@ -45,7 +46,7 @@ def test_models(): print "MODULE", module_examples print "Before" print inspect.getmembers(module_examples, predicate=inspect.isfunction) - functions = [ func for func in inspect.getmembers(module_examples, predicate=inspect.isfunction) if func[0].startswith('_') is False ][::-1] + functions = [ func for func in inspect.getmembers(module_examples, predicate=inspect.isfunction) if func[0].startswith('_') is False ] print "After" print functions for example in functions: From 2a1b5f94c8a0f682ee23d42b9953b1ca8a783b2a Mon Sep 17 00:00:00 2001 From: Alan Saul Date: Mon, 11 Mar 2013 18:14:23 +0000 Subject: [PATCH 06/12] Got rid of foo --- GPy/kern/rbf.py | 1 - GPy/testing/examples_tests.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/GPy/kern/rbf.py b/GPy/kern/rbf.py index 133895ff..3c3d59e6 100644 --- a/GPy/kern/rbf.py +++ b/GPy/kern/rbf.py @@ -55,7 +55,6 @@ class rbf(kernpart): self._X, self._X2, self._params = np.empty(shape=(3,1)) def _get_params(self): - foo return np.hstack((self.variance,self.lengthscale)) def _set_params(self,x): diff --git a/GPy/testing/examples_tests.py b/GPy/testing/examples_tests.py index 07ef4177..141d3999 100644 --- a/GPy/testing/examples_tests.py +++ b/GPy/testing/examples_tests.py @@ -46,7 +46,7 @@ def test_models(): print "MODULE", module_examples print "Before" print inspect.getmembers(module_examples, predicate=inspect.isfunction) - functions = [ func for func in inspect.getmembers(module_examples, predicate=inspect.isfunction) if func[0].startswith('_') is False ] + functions = [ func for func in inspect.getmembers(module_examples, predicate=inspect.isfunction) if func[0].startswith('_') is False ][::-1] print "After" print functions for example in functions: From 0e187b69212e2efbf06b4ee07fddb2b9e9e05de3 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 11 Mar 2013 18:21:29 +0000 Subject: [PATCH 07/12] update in the rational quadratic kernel and new the tutorial on writting kernels --- GPy/kern/rational_quadratic.py | 5 +- doc/GPy.examples.rst | 68 +---------- doc/GPy.kern.rst | 8 ++ doc/GPy.models.rst | 6 +- doc/GPy.rst | 8 -- doc/GPy.testing.rst | 59 ++++++++++ doc/index.rst | 3 +- doc/tuto_creating_new_kernels.rst | 183 ++++++++++++++++++++++++++++++ 8 files changed, 263 insertions(+), 77 deletions(-) create mode 100644 doc/GPy.testing.rst create mode 100644 doc/tuto_creating_new_kernels.rst diff --git a/GPy/kern/rational_quadratic.py b/GPy/kern/rational_quadratic.py index b71d1354..15200fd3 100644 --- a/GPy/kern/rational_quadratic.py +++ b/GPy/kern/rational_quadratic.py @@ -11,7 +11,7 @@ class rational_quadratic(kernpart): .. math:: - k(r) = \sigma^2 \left(1 + \frac{r^2}{2 \ell^2})^{- \alpha} \ \ \ \ \ \\text{ where } r^2 = (x-y)^2 + k(r) = \sigma^2 \\bigg( 1 + \\frac{r^2}{2 \ell^2} \\bigg)^{- \\alpha} \ \ \ \ \ \\text{ where } r^2 = (x-y)^2 :param D: the number of input dimensions :type D: int (D=1 is the only value currently supported) @@ -19,6 +19,8 @@ class rational_quadratic(kernpart): :type variance: float :param lengthscale: the lengthscale :math:`\ell` :type lengthscale: float + :param power: the power :math:`\\alpha` + :type power: float :rtype: kernpart object """ @@ -76,4 +78,3 @@ class rational_quadratic(kernpart): def dKdiag_dX(self,dL_dKdiag,X,target): pass - diff --git a/doc/GPy.examples.rst b/doc/GPy.examples.rst index d369de41..f17cf826 100644 --- a/doc/GPy.examples.rst +++ b/doc/GPy.examples.rst @@ -9,14 +9,6 @@ examples Package :undoc-members: :show-inheritance: -:mod:`BGPLVM_demo` Module -------------------------- - -.. automodule:: GPy.examples.BGPLVM_demo - :members: - :undoc-members: - :show-inheritance: - :mod:`classification` Module ---------------------------- @@ -25,18 +17,18 @@ examples Package :undoc-members: :show-inheritance: -:mod:`oil_flow_demo` Module ---------------------------- +:mod:`dimensionality_reduction` Module +-------------------------------------- -.. automodule:: GPy.examples.oil_flow_demo +.. automodule:: GPy.examples.dimensionality_reduction :members: :undoc-members: :show-inheritance: -:mod:`poisson` Module ---------------------- +:mod:`non_gaussian` Module +-------------------------- -.. automodule:: GPy.examples.poisson +.. automodule:: GPy.examples.non_gaussian :members: :undoc-members: :show-inheritance: @@ -49,30 +41,6 @@ examples Package :undoc-members: :show-inheritance: -:mod:`sparse_GPLVM_demo` Module -------------------------------- - -.. automodule:: GPy.examples.sparse_GPLVM_demo - :members: - :undoc-members: - :show-inheritance: - -:mod:`sparse_GP_regression_demo` Module ---------------------------------------- - -.. automodule:: GPy.examples.sparse_GP_regression_demo - :members: - :undoc-members: - :show-inheritance: - -:mod:`sparse_ep_fix` Module ---------------------------- - -.. automodule:: GPy.examples.sparse_ep_fix - :members: - :undoc-members: - :show-inheritance: - :mod:`tutorials` Module ----------------------- @@ -81,27 +49,3 @@ examples Package :undoc-members: :show-inheritance: -:mod:`uncertain_input_GP_regression_demo` Module ------------------------------------------------- - -.. automodule:: GPy.examples.uncertain_input_GP_regression_demo - :members: - :undoc-members: - :show-inheritance: - -:mod:`uncollapsed_GP_demo` Module ---------------------------------- - -.. automodule:: GPy.examples.uncollapsed_GP_demo - :members: - :undoc-members: - :show-inheritance: - -:mod:`unsupervised` Module --------------------------- - -.. automodule:: GPy.examples.unsupervised - :members: - :undoc-members: - :show-inheritance: - diff --git a/doc/GPy.kern.rst b/doc/GPy.kern.rst index 3ebeda40..aef712dc 100644 --- a/doc/GPy.kern.rst +++ b/doc/GPy.kern.rst @@ -137,6 +137,14 @@ kern Package :undoc-members: :show-inheritance: +:mod:`rational_quadratic` Module +-------------------------------- + +.. automodule:: GPy.kern.rational_quadratic + :members: + :undoc-members: + :show-inheritance: + :mod:`rbf` Module ----------------- diff --git a/doc/GPy.models.rst b/doc/GPy.models.rst index 8837ac4e..85bd727a 100644 --- a/doc/GPy.models.rst +++ b/doc/GPy.models.rst @@ -9,10 +9,10 @@ models Package :undoc-members: :show-inheritance: -:mod:`BGPLVM` Module --------------------- +:mod:`Bayesian_GPLVM` Module +---------------------------- -.. automodule:: GPy.models.BGPLVM +.. automodule:: GPy.models.Bayesian_GPLVM :members: :undoc-members: :show-inheritance: diff --git a/doc/GPy.rst b/doc/GPy.rst index 242a22bc..e56e48e1 100644 --- a/doc/GPy.rst +++ b/doc/GPy.rst @@ -9,14 +9,6 @@ GPy Package :undoc-members: :show-inheritance: -:mod:`test_coreg` Module ------------------------- - -.. automodule:: GPy.test_coreg - :members: - :undoc-members: - :show-inheritance: - Subpackages ----------- diff --git a/doc/GPy.testing.rst b/doc/GPy.testing.rst new file mode 100644 index 00000000..5b32558b --- /dev/null +++ b/doc/GPy.testing.rst @@ -0,0 +1,59 @@ +testing Package +=============== + +:mod:`bgplvm_tests` Module +-------------------------- + +.. automodule:: GPy.testing.bgplvm_tests + :members: + :undoc-members: + :show-inheritance: + +:mod:`examples_tests` Module +---------------------------- + +.. automodule:: GPy.testing.examples_tests + :members: + :undoc-members: + :show-inheritance: + +:mod:`gplvm_tests` Module +------------------------- + +.. automodule:: GPy.testing.gplvm_tests + :members: + :undoc-members: + :show-inheritance: + +:mod:`kernel_tests` Module +-------------------------- + +.. automodule:: GPy.testing.kernel_tests + :members: + :undoc-members: + :show-inheritance: + +:mod:`prior_tests` Module +------------------------- + +.. automodule:: GPy.testing.prior_tests + :members: + :undoc-members: + :show-inheritance: + +:mod:`sparse_gplvm_tests` Module +-------------------------------- + +.. automodule:: GPy.testing.sparse_gplvm_tests + :members: + :undoc-members: + :show-inheritance: + +:mod:`unit_tests` Module +------------------------ + +.. automodule:: GPy.testing.unit_tests + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/index.rst b/doc/index.rst index 5066278f..a7b68c16 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -10,8 +10,7 @@ For a quick start, you can have a look at one of the tutorials: * `Basic Gaussian process regression `_ * `Interacting with models `_ * `A kernel overview `_ -* Advanced GP regression (Forthcoming) -* Writing kernels (Forthcoming) +* `Writing new kernels `_ You may also be interested by some examples in the GPy/examples folder. diff --git a/doc/tuto_creating_new_kernels.rst b/doc/tuto_creating_new_kernels.rst new file mode 100644 index 00000000..672bc1e7 --- /dev/null +++ b/doc/tuto_creating_new_kernels.rst @@ -0,0 +1,183 @@ +******************** +Creating new kernels +******************** + +We will see in this tutorial how to create new kernels in GPy. We will also give details on how to implement each function of the kernel and illustrate with a running example: the rational quadratic kernel. + +Structure of a kernel in GPy +============================ + +In GPy a kernel object is made of a list of kernpart objects, which correspond to symetric positive definite functions. More precisely, the kernel should be understood as the sum of the kernparts. In order to implement a new covariance, the following steps must be followed + + 1. implement the new covariance as a kernpart object + 2. update the constructors that allow to use the kernpart as a kern object + 3. update the __init__.py file + +Theses three steps are detailed below. + +Implementing a kernpart object +============================== + +We advise the reader to start with copy-pasting an existing kernel and to modify the new file. We will now give a description of the various functions that can be found in a kernpart object. + +**Header** + +The header is similar to all kernels:: + + from kernpart import kernpart + import numpy as np + + class rational_quadratic(kernpart): + +**__init__(self,D, param1, param2, ...)** + +The implementation of this function in mandatory. + +For all kernparts the first parameter ``D`` corresponds to the dimension of the input space, and the following parameters stand for the parameterization of the kernel. + +The following attributes are compulsory: ``self.D`` (the dimension, integer), ``self.name`` (name of the kernel, string), ``self.Nparam`` (number of parameters, integer).:: + + def __init__(self,D,variance=1.,lengthscale=1.,power=1.): + assert D == 1, "For this kernel we assume D=1" + self.D = D + self.Nparam = 3 + self.name = 'rat_quad' + self.variance = variance + self.lengthscale = lengthscale + self.power = power + +**_get_params(self)** + +The implementation of this function in mandatory. + +This function returns a one dimensional array of length ``self.Nparam`` containing the value of the parameters.:: + + def _get_params(self): + return np.hstack((self.variance,self.lengthscale,self.power)) + +**_set_params(self,x)** + +The implementation of this function in mandatory. + +The input is a one dimensional array of length ``self.Nparam`` containing the value of the parameters. The function has no output but it updates the values of the attribute associated to the parameters (such as ``self.variance``, ``self.lengthscale``, ...).:: + + def _set_params(self,x): + self.variance = x[0] + self.lengthscale = x[1] + self.power = x[2] + +**_get_param_names(self)** + +The implementation of this function in mandatory. + +It returns a list of strings of length ``self.Nparam`` corresponding to the parameter names.:: + + def _get_param_names(self): + return ['variance','lengthscale','power'] + +**K(self,X,X2,target)** + +The implementation of this function in mandatory. + +This function is used to compute the covariance matrix associated with the inputs X, X2 (np.arrays with arbitrary number of line (say :math:`n_1`, :math:`n_2`) and ``self.D`` columns). This function does not returns anything but it adds the :math:`n_1 \times n_2` covariance matrix to the kernpart to the object ``target`` (a :math:`n_1 \times n_2` np.array). This trick allows to compute the covariance matrix of a kernel containing many kernparts with a limited memory use.:: + + def K(self,X,X2,target): + if X2 is None: X2 = X + dist2 = np.square((X-X2.T)/self.lengthscale) + target += self.variance*(1 + dist2/2.)**(-self.power) + +**Kdiag(self,X,target)** + +The implementation of this function in mandatory. + +This function is similar to ``K`` but it computes only the values of the kernel on the diagonal. Thus, ``target`` is a 1-dimensional np.array of length :math:`n_1`.:: + + def Kdiag(self,X,target): + target += self.variance + + +**dK_dtheta(self,dL_dK,X,X2,target)** + +This function is required for the optimization of the parameters. + +Computes the derivative of the likelihood. As previously, the values are added to the object target which is a 1-dimensional np.array of length ``self.Nparam``. For example, if the kernel is parameterized by :math:`\sigma^2,\ \theta`, then :math:`\frac{dL}{d\sigma^2} = \frac{dL}{d K} \frac{dK}{d\sigma^2}` is added to the first element of target and :math:`\frac{dL}{d\theta} = \frac{dL}{d K} \frac{dK}{d\theta}` to the second.:: + + def dK_dtheta(self,dL_dK,X,X2,target): + if X2 is None: X2 = X + dist2 = np.square((X-X2.T)/self.lengthscale) + + dvar = (1 + dist2/2.)**(-self.power) + dl = self.power * self.variance * dist2 * self.lengthscale**(-3) * (1 + dist2/2./self.power)**(-self.power-1) + dp = - self.variance * np.log(1 + dist2/2.) * (1 + dist2/2.)**(-self.power) + + target[0] += np.sum(dvar*dL_dK) + target[1] += np.sum(dl*dL_dK) + target[2] += np.sum(dp*dL_dK) + + +**dKdiag_dtheta(self,dL_dKdiag,X,target)** + +This function is required for BGPLVM, sparse models and uncertain inputs. + +As previously, target is an ``self.Nparam`` array and :math:`\frac{dL}{d Kdiag} \frac{dKdiag}{dparam}` is added to each element.:: + + def dKdiag_dtheta(self,dL_dKdiag,X,target): + target[0] += np.sum(dL_dKdiag) + # here self.lengthscale and self.power have no influence on Kdiag so target[1:] are unchanged + +**dK_dX(self,dL_dK,X,X2,target)** + +This function is required for GPLVM, BGPLVM, sparse models and uncertain inputs. + +Computes the derivative of the likelihood with respect to the inputs ``X`` (a :math:`n \times D` np.array). The result is added to target which is a :math:`n \times D` np.array.:: + + def dK_dX(self,dL_dK,X,X2,target): + """derivative of the covariance matrix with respect to X.""" + if X2 is None: X2 = X + dist2 = np.square((X-X2.T)/self.lengthscale) + + dX = -self.variance*self.power * (X-X2.T)/self.lengthscale**2 * (1 + dist2/2./self.power)**(-self.power-1) + target += np.sum(dL_dK*dX) + +**dKdiag_dX(self,dL_dKdiag,X,target)** + +This function is required for BGPLVM, sparse models and uncertain inputs. As for ``dKdiag_dtheta``, :math:`\frac{dL}{d Kdiag} \frac{dKdiag}{dX}` is added to each element of target.:: + + def dKdiag_dX(self,dL_dKdiag,X,target): + pass + +**Psi statistics** + +The psi statistics and their derivatives are required for BGPLVM and GPS with uncertain inputs. + +The expressions of the psi statistics are: + +TODO + +For the rational quadratic we have: + +TODO + +Update the constructor +====================== + +Once the required functions have been implemented as a kernpart object, the file GPy/kern/constructors.py has to be updated to allow to build a kernel based on the kernpart object. + +The following line should be added in the preamble of the file:: + + from rational_quadratic import rational_quadratic as rational_quadratic_part + +as well as the following block:: + + def rational_quadratic(D,variance=1., lengthscale=1., power=1.): + part = rational_quadraticpart(D,variance, lengthscale, power) + return kern(D, [part]) + + +Update initialization +===================== + +The last step is to update the list of kernels imported from constructor in GPy/kern/__init__.py. + + + From b06b4ea8f1c8793540bec7d68340310489afdadd Mon Sep 17 00:00:00 2001 From: Alan Saul Date: Mon, 11 Mar 2013 18:25:11 +0000 Subject: [PATCH 08/12] Fixed checkgrad test to randomize before checking --- GPy/testing/examples_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/GPy/testing/examples_tests.py b/GPy/testing/examples_tests.py index 141d3999..feba2b50 100644 --- a/GPy/testing/examples_tests.py +++ b/GPy/testing/examples_tests.py @@ -30,6 +30,7 @@ def checkgrads_generator(model): """ def model_checkgrads(model): + model.randomize() assert model.checkgrad() From b336d914739f8965aeec322691d0b1638313e400 Mon Sep 17 00:00:00 2001 From: Nicolo Fusi Date: Mon, 11 Mar 2013 18:43:59 +0000 Subject: [PATCH 09/12] fixed bug in RBF, added inducing inputs to BGPLVM plots --- GPy/kern/rbf.py | 1 - GPy/models/Bayesian_GPLVM.py | 4 ++++ GPy/models/GPLVM.py | 4 +--- GPy/models/sparse_GPLVM.py | 4 ++++ 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/GPy/kern/rbf.py b/GPy/kern/rbf.py index 133895ff..3c3d59e6 100644 --- a/GPy/kern/rbf.py +++ b/GPy/kern/rbf.py @@ -55,7 +55,6 @@ class rbf(kernpart): self._X, self._X2, self._params = np.empty(shape=(3,1)) def _get_params(self): - foo return np.hstack((self.variance,self.lengthscale)) def _set_params(self,x): diff --git a/GPy/models/Bayesian_GPLVM.py b/GPy/models/Bayesian_GPLVM.py index 0eb957a9..430c2718 100644 --- a/GPy/models/Bayesian_GPLVM.py +++ b/GPy/models/Bayesian_GPLVM.py @@ -83,3 +83,7 @@ class Bayesian_GPLVM(sparse_GP, GPLVM): def _log_likelihood_gradients(self): return np.hstack((self.dL_dmuS().flatten(), sparse_GP._log_likelihood_gradients(self))) + + def plot_latent(self, *args, **kwargs): + input_1, input_2 = GPLVM.plot_latent(*args, **kwargs) + pb.plot(m.Z[:, input_1], m.Z[:, input_2], '^w') diff --git a/GPy/models/GPLVM.py b/GPy/models/GPLVM.py index d0dc766f..b44801fc 100644 --- a/GPy/models/GPLVM.py +++ b/GPy/models/GPLVM.py @@ -117,6 +117,4 @@ class GPLVM(GP): pb.xlim(xmin[0],xmax[0]) pb.ylim(xmin[1],xmax[1]) - - - + return input_1, input_2 diff --git a/GPy/models/sparse_GPLVM.py b/GPy/models/sparse_GPLVM.py index 542fbe0e..591c49b2 100644 --- a/GPy/models/sparse_GPLVM.py +++ b/GPy/models/sparse_GPLVM.py @@ -55,3 +55,7 @@ class sparse_GPLVM(sparse_GP_regression, GPLVM): #passing Z without a small amout of jitter will induce the white kernel where we don;t want it! mu, var, upper, lower = sparse_GP_regression.predict(self, self.Z+np.random.randn(*self.Z.shape)*0.0001) pb.plot(mu[:, 0] , mu[:, 1], 'ko') + + def plot_latent(self, *args, **kwargs): + input_1, input_2 = GPLVM.plot_latent(*args, **kwargs) + pb.plot(m.Z[:, input_1], m.Z[:, input_2], '^w') From 3dd62c8251740440c037aead3b4bd2f855f9805b Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 11 Mar 2013 18:45:04 +0000 Subject: [PATCH 10/12] Few bugs fixed in the documentation --- GPy/kern/rbf.py | 3 +-- doc/tuto_creating_new_kernels.rst | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/GPy/kern/rbf.py b/GPy/kern/rbf.py index 133895ff..ae587202 100644 --- a/GPy/kern/rbf.py +++ b/GPy/kern/rbf.py @@ -12,7 +12,7 @@ class rbf(kernpart): .. math:: - k(r) = \sigma^2 \exp(- \frac{1}{2}r^2) \ \ \ \ \ \\text{ where } r^2 = \sum_{i=1}^d \frac{ (x_i-x^\prime_i)^2}{\ell_i^2}} + k(r) = \sigma^2 \exp \\bigg(- \\frac{1}{2} r^2 \\bigg) \ \ \ \ \ \\text{ where } r^2 = \sum_{i=1}^d \\frac{ (x_i-x^\prime_i)^2}{\ell_i^2} where \ell_i is the lengthscale, \sigma^2 the variance and d the dimensionality of the input. @@ -55,7 +55,6 @@ class rbf(kernpart): self._X, self._X2, self._params = np.empty(shape=(3,1)) def _get_params(self): - foo return np.hstack((self.variance,self.lengthscale)) def _set_params(self,x): diff --git a/doc/tuto_creating_new_kernels.rst b/doc/tuto_creating_new_kernels.rst index 672bc1e7..8ebf8b8f 100644 --- a/doc/tuto_creating_new_kernels.rst +++ b/doc/tuto_creating_new_kernels.rst @@ -22,7 +22,7 @@ We advise the reader to start with copy-pasting an existing kernel and to modify **Header** -The header is similar to all kernels:: +The header is similar to all kernels: :: from kernpart import kernpart import numpy as np @@ -35,7 +35,7 @@ The implementation of this function in mandatory. For all kernparts the first parameter ``D`` corresponds to the dimension of the input space, and the following parameters stand for the parameterization of the kernel. -The following attributes are compulsory: ``self.D`` (the dimension, integer), ``self.name`` (name of the kernel, string), ``self.Nparam`` (number of parameters, integer).:: +The following attributes are compulsory: ``self.D`` (the dimension, integer), ``self.name`` (name of the kernel, string), ``self.Nparam`` (number of parameters, integer). :: def __init__(self,D,variance=1.,lengthscale=1.,power=1.): assert D == 1, "For this kernel we assume D=1" @@ -50,7 +50,7 @@ The following attributes are compulsory: ``self.D`` (the dimension, integer), `` The implementation of this function in mandatory. -This function returns a one dimensional array of length ``self.Nparam`` containing the value of the parameters.:: +This function returns a one dimensional array of length ``self.Nparam`` containing the value of the parameters. :: def _get_params(self): return np.hstack((self.variance,self.lengthscale,self.power)) @@ -59,7 +59,7 @@ This function returns a one dimensional array of length ``self.Nparam`` containi The implementation of this function in mandatory. -The input is a one dimensional array of length ``self.Nparam`` containing the value of the parameters. The function has no output but it updates the values of the attribute associated to the parameters (such as ``self.variance``, ``self.lengthscale``, ...).:: +The input is a one dimensional array of length ``self.Nparam`` containing the value of the parameters. The function has no output but it updates the values of the attribute associated to the parameters (such as ``self.variance``, ``self.lengthscale``, ...). :: def _set_params(self,x): self.variance = x[0] @@ -70,7 +70,7 @@ The input is a one dimensional array of length ``self.Nparam`` containing the va The implementation of this function in mandatory. -It returns a list of strings of length ``self.Nparam`` corresponding to the parameter names.:: +It returns a list of strings of length ``self.Nparam`` corresponding to the parameter names. :: def _get_param_names(self): return ['variance','lengthscale','power'] @@ -79,7 +79,7 @@ It returns a list of strings of length ``self.Nparam`` corresponding to the para The implementation of this function in mandatory. -This function is used to compute the covariance matrix associated with the inputs X, X2 (np.arrays with arbitrary number of line (say :math:`n_1`, :math:`n_2`) and ``self.D`` columns). This function does not returns anything but it adds the :math:`n_1 \times n_2` covariance matrix to the kernpart to the object ``target`` (a :math:`n_1 \times n_2` np.array). This trick allows to compute the covariance matrix of a kernel containing many kernparts with a limited memory use.:: +This function is used to compute the covariance matrix associated with the inputs X, X2 (np.arrays with arbitrary number of line (say :math:`n_1`, :math:`n_2`) and ``self.D`` columns). This function does not returns anything but it adds the :math:`n_1 \times n_2` covariance matrix to the kernpart to the object ``target`` (a :math:`n_1 \times n_2` np.array). This trick allows to compute the covariance matrix of a kernel containing many kernparts with a limited memory use. :: def K(self,X,X2,target): if X2 is None: X2 = X @@ -90,7 +90,7 @@ This function is used to compute the covariance matrix associated with the input The implementation of this function in mandatory. -This function is similar to ``K`` but it computes only the values of the kernel on the diagonal. Thus, ``target`` is a 1-dimensional np.array of length :math:`n_1`.:: +This function is similar to ``K`` but it computes only the values of the kernel on the diagonal. Thus, ``target`` is a 1-dimensional np.array of length :math:`n_1`. :: def Kdiag(self,X,target): target += self.variance @@ -100,7 +100,7 @@ This function is similar to ``K`` but it computes only the values of the kernel This function is required for the optimization of the parameters. -Computes the derivative of the likelihood. As previously, the values are added to the object target which is a 1-dimensional np.array of length ``self.Nparam``. For example, if the kernel is parameterized by :math:`\sigma^2,\ \theta`, then :math:`\frac{dL}{d\sigma^2} = \frac{dL}{d K} \frac{dK}{d\sigma^2}` is added to the first element of target and :math:`\frac{dL}{d\theta} = \frac{dL}{d K} \frac{dK}{d\theta}` to the second.:: +Computes the derivative of the likelihood. As previously, the values are added to the object target which is a 1-dimensional np.array of length ``self.Nparam``. For example, if the kernel is parameterized by :math:`\sigma^2,\ \theta`, then :math:`\frac{dL}{d\sigma^2} = \frac{dL}{d K} \frac{dK}{d\sigma^2}` is added to the first element of target and :math:`\frac{dL}{d\theta} = \frac{dL}{d K} \frac{dK}{d\theta}` to the second. :: def dK_dtheta(self,dL_dK,X,X2,target): if X2 is None: X2 = X @@ -119,7 +119,7 @@ Computes the derivative of the likelihood. As previously, the values are added t This function is required for BGPLVM, sparse models and uncertain inputs. -As previously, target is an ``self.Nparam`` array and :math:`\frac{dL}{d Kdiag} \frac{dKdiag}{dparam}` is added to each element.:: +As previously, target is an ``self.Nparam`` array and :math:`\frac{dL}{d Kdiag} \frac{dKdiag}{dparam}` is added to each element. :: def dKdiag_dtheta(self,dL_dKdiag,X,target): target[0] += np.sum(dL_dKdiag) @@ -129,7 +129,7 @@ As previously, target is an ``self.Nparam`` array and :math:`\frac{dL}{d Kdiag} This function is required for GPLVM, BGPLVM, sparse models and uncertain inputs. -Computes the derivative of the likelihood with respect to the inputs ``X`` (a :math:`n \times D` np.array). The result is added to target which is a :math:`n \times D` np.array.:: +Computes the derivative of the likelihood with respect to the inputs ``X`` (a :math:`n \times D` np.array). The result is added to target which is a :math:`n \times D` np.array. :: def dK_dX(self,dL_dK,X,X2,target): """derivative of the covariance matrix with respect to X.""" @@ -141,7 +141,7 @@ Computes the derivative of the likelihood with respect to the inputs ``X`` (a :m **dKdiag_dX(self,dL_dKdiag,X,target)** -This function is required for BGPLVM, sparse models and uncertain inputs. As for ``dKdiag_dtheta``, :math:`\frac{dL}{d Kdiag} \frac{dKdiag}{dX}` is added to each element of target.:: +This function is required for BGPLVM, sparse models and uncertain inputs. As for ``dKdiag_dtheta``, :math:`\frac{dL}{d Kdiag} \frac{dKdiag}{dX}` is added to each element of target. :: def dKdiag_dX(self,dL_dKdiag,X,target): pass @@ -167,7 +167,7 @@ The following line should be added in the preamble of the file:: from rational_quadratic import rational_quadratic as rational_quadratic_part -as well as the following block:: +as well as the following block :: def rational_quadratic(D,variance=1., lengthscale=1., power=1.): part = rational_quadraticpart(D,variance, lengthscale, power) From 6febbed36fe2ed6043c2d37cda36304e36cc83e6 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 11 Mar 2013 19:13:19 +0000 Subject: [PATCH 11/12] tie_param changed to tie_params in tutorials --- doc/tuto_kernel_overview.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/tuto_kernel_overview.rst b/doc/tuto_kernel_overview.rst index dfb7fb3f..da19803b 100644 --- a/doc/tuto_kernel_overview.rst +++ b/doc/tuto_kernel_overview.rst @@ -133,7 +133,7 @@ Various constrains can be applied to the parameters of a kernel * ``constrain_fixed`` to fix the value of a parameter (the value will not be modified during optimisation) * ``constrain_positive`` to make sure the parameter is greater than 0. * ``constrain_bounded`` to impose the parameter to be in a given range. - * ``tie_param`` to impose the value of two (or more) parameters to be equal. + * ``tie_params`` to impose the value of two (or more) parameters to be equal. When calling one of these functions, the parameters to constrain can either by specified by a regular expression that matches its name or by a number that corresponds to the rank of the parameter. Here is an example :: @@ -146,7 +146,7 @@ When calling one of these functions, the parameters to constrain can either by s k.constrain_positive('var') k.constrain_fixed(np.array([1]),1.75) - k.tie_param('len') + k.tie_params('len') k.unconstrain('white') k.constrain_bounded('white',lower=1e-5,upper=.5) print k From 4525ddd75a66b99ea1514ff3266e75ca985da135 Mon Sep 17 00:00:00 2001 From: andreas Date: Mon, 11 Mar 2013 19:19:36 +0000 Subject: [PATCH 12/12] fixed plots for BGPLVM --- GPy/models/Bayesian_GPLVM.py | 4 ++-- GPy/models/GPLVM.py | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/GPy/models/Bayesian_GPLVM.py b/GPy/models/Bayesian_GPLVM.py index 430c2718..a18ec9bb 100644 --- a/GPy/models/Bayesian_GPLVM.py +++ b/GPy/models/Bayesian_GPLVM.py @@ -85,5 +85,5 @@ class Bayesian_GPLVM(sparse_GP, GPLVM): return np.hstack((self.dL_dmuS().flatten(), sparse_GP._log_likelihood_gradients(self))) def plot_latent(self, *args, **kwargs): - input_1, input_2 = GPLVM.plot_latent(*args, **kwargs) - pb.plot(m.Z[:, input_1], m.Z[:, input_2], '^w') + input_1, input_2 = GPLVM.plot_latent(self, *args, **kwargs) + pb.plot(self.Z[:, input_1], self.Z[:, input_2], '^w') diff --git a/GPy/models/GPLVM.py b/GPy/models/GPLVM.py index b44801fc..5be54049 100644 --- a/GPy/models/GPLVM.py +++ b/GPy/models/GPLVM.py @@ -81,13 +81,16 @@ class GPLVM(GP): raise ValueError, "cannot Atomatically determine which dimensions to plot, please pass 'which_indices'" k = k[0] if k.name=='rbf': - input_1, input_2 = np.argsort(k.lengthscales)[:2] + input_1, input_2 = np.argsort(k.lengthscale)[:2] elif k.name=='linear': input_1, input_2 = np.argsort(k.variances)[::-1][:2] #first, plot the output variance as a function of the latent space Xtest, xx,yy,xmin,xmax = util.plot.x_frame2D(self.X[:,[input_1, input_2]],resolution=resolution) - mu, var, low, up = self.predict(Xtest) + Xtest_full = np.zeros((Xtest.shape[0], self.X.shape[1])) + Xtest_full[:, :2] = Xtest + mu, var, low, up = self.predict(Xtest_full) + var = var[:, :2] pb.imshow(var.reshape(resolution,resolution).T[::-1,:],extent=[xmin[0],xmax[0],xmin[1],xmax[1]],cmap=pb.cm.binary,interpolation='bilinear')