very weird merge conflict, including in files that I did not change

GPy/core/gp.py

@@ -7,10 +7,10 @@ import warnings
 from .. import kern
 from ..util.linalg import dtrtrs
 from model import Model
-from parameterization import ObservableArray
+from parameterization import ObsAr
 from .. import likelihoods
 from ..likelihoods.gaussian import Gaussian
-from ..inference.latent_function_inference import exact_gaussian_inference
+from ..inference.latent_function_inference import exact_gaussian_inference, expectation_propagation
 from parameterization.variational import VariationalPosterior
 
 class GP(Model):
@@ -27,28 +27,26 @@ class GP(Model):
 
 
     """
-    def __init__(self, X, Y, kernel, likelihood, inference_method=None, Y_metadata=None, name='gp'):
+    def __init__(self, X, Y, kernel, likelihood, inference_method=None, name='gp', Y_metadata=None):
         super(GP, self).__init__(name)
 
         assert X.ndim == 2
-        if isinstance(X, (ObservableArray, VariationalPosterior)):
+        if isinstance(X, (ObsAr, VariationalPosterior)):
             self.X = X
-        else: self.X = ObservableArray(X)
+        else: self.X = ObsAr(X)
 
         self.num_data, self.input_dim = self.X.shape
 
         assert Y.ndim == 2
-        self.Y = ObservableArray(Y)
+        self.Y = ObsAr(Y)
         assert Y.shape[0] == self.num_data
         _, self.output_dim = self.Y.shape
 
-        if Y_metadata is not None:
-            self.Y_metadata = ObservableArray(Y_metadata)
-        else:
-            self.Y_metadata = None
+        #TODO: check the type of this is okay?
+        self.Y_metadata = Y_metadata
 
         assert isinstance(kernel, kern.Kern)
-        assert self.input_dim == kernel.input_dim
+        #assert self.input_dim == kernel.input_dim
         self.kern = kernel
 
         assert isinstance(likelihood, likelihoods.Likelihood)
@@ -56,10 +54,10 @@ class GP(Model):
 
         #find a sensible inference method
         if inference_method is None:
-            if isinstance(likelihood, likelihoods.Gaussian):
+            if isinstance(likelihood, likelihoods.Gaussian) or isinstance(likelihood, likelihoods.MixedNoise):
                 inference_method = exact_gaussian_inference.ExactGaussianInference()
             else:
-                inference_method = expectation_propagation
+                inference_method = expectation_propagation.EP()
                 print "defaulting to ", inference_method, "for latent function inference"
         self.inference_method = inference_method
 
@@ -67,8 +65,9 @@ class GP(Model):
         self.add_parameter(self.likelihood)
 
     def parameters_changed(self):
-        self.posterior, self._log_marginal_likelihood, grad_dict = self.inference_method.inference(self.kern, self.X, self.likelihood, self.Y, Y_metadata=self.Y_metadata)
-        self.kern.update_gradients_full(grad_dict['dL_dK'], self.X)
+        self.posterior, self._log_marginal_likelihood, self.grad_dict = self.inference_method.inference(self.kern, self.X, self.likelihood, self.Y, self.Y_metadata)
+        self.likelihood.update_gradients(self.grad_dict['dL_dthetaL'])
+        self.kern.update_gradients_full(self.grad_dict['dL_dK'], self.X)
 
     def log_likelihood(self):
         return self._log_marginal_likelihood
@@ -96,9 +95,12 @@ class GP(Model):
             #var = Kxx - np.sum(LiKx*LiKx, 0)
             var = Kxx - np.sum(WiKx*Kx, 0)
             var = var.reshape(-1, 1)
+
+        #force mu to be a column vector
+        if len(mu.shape)==1: mu = mu[:,None]
         return mu, var
 
-    def predict(self, Xnew, full_cov=False, **likelihood_args):
+    def predict(self, Xnew, full_cov=False, Y_metadata=None):
         """
         Predict the function(s) at the new point(s) Xnew.
 
@@ -122,8 +124,12 @@ class GP(Model):
         mu, var = self._raw_predict(Xnew, full_cov=full_cov)
 
         # now push through likelihood
-        mean, var, _025pm, _975pm = self.likelihood.predictive_values(mu, var, full_cov, **likelihood_args)
-        return mean, var, _025pm, _975pm
+        mean, var = self.likelihood.predictive_values(mu, var, full_cov, Y_metadata)
+        return mean, var
+
+    def predict_quantiles(self, X, quantiles=(2.5, 97.5), Y_metadata=None):
+        m, v = self._raw_predict(X,  full_cov=False)
+        return self.likelihood.predictive_quantiles(m, v, quantiles, Y_metadata)
 
     def posterior_samples_f(self,X,size=10, full_cov=True):
         """
@@ -146,7 +152,7 @@ class GP(Model):
 
         return Ysim
 
-    def posterior_samples(self,X,size=10, full_cov=True,noise_model=None):
+    def posterior_samples(self, X, size=10, full_cov=False, Y_metadata=None):
         """
         Samples the posterior GP at the points X.
 
@@ -161,15 +167,7 @@ class GP(Model):
         :returns: Ysim: set of simulations, a Numpy array (N x samples).
         """
         Ysim = self.posterior_samples_f(X, size, full_cov=full_cov)
-        if isinstance(self.likelihood, Gaussian):
-            noise_std = np.sqrt(self.likelihood._get_params())
-            Ysim += np.random.normal(0,noise_std,Ysim.shape)
-        elif isinstance(self.likelihood, Gaussian_Mixed_Noise):
-            assert noise_model is not None, "A noise model must be specified."
-            noise_std = np.sqrt(self.likelihood._get_params()[noise_model])
-            Ysim += np.random.normal(0,noise_std,Ysim.shape)
-        else:
-            Ysim = self.likelihood.noise_model.samples(Ysim)
+        Ysim = self.likelihood.samples(Ysim, Y_metadata)
 
         return Ysim
 
@@ -185,7 +183,7 @@ class GP(Model):
         """
         assert "matplotlib" in sys.modules, "matplotlib package has not been imported."
         from ..plotting.matplot_dep import models_plots
-        models_plots.plot_fit_f(self,*args,**kwargs)
+        return models_plots.plot_fit_f(self,*args,**kwargs)
 
     def plot(self, *args, **kwargs):
         """
@@ -206,7 +204,7 @@ class GP(Model):
         """
         assert "matplotlib" in sys.modules, "matplotlib package has not been imported."
         from ..plotting.matplot_dep import models_plots
-        models_plots.plot_fit(self,*args,**kwargs)
+        return models_plots.plot_fit(self,*args,**kwargs)
 
     def _getstate(self):
         """

GPy/core/model.py

@@ -15,7 +15,7 @@ import itertools
 class Model(Parameterized):
     _fail_count = 0  # Count of failed optimization steps (see objective)
     _allowed_failures = 10  # number of allowed failures
-        
+
     def __init__(self, name):
         super(Model, self).__init__(name)  # Parameterized.__init__(self)
         self.optimization_runs = []
@@ -27,7 +27,7 @@ class Model(Parameterized):
 
     def _log_likelihood_gradients(self):
         return self.gradient
-        
+
     def _getstate(self):
         """
         Get the current state of the class.
@@ -231,7 +231,7 @@ class Model(Parameterized):
             raise RuntimeError, "Cannot optimize, when everything is fixed"
         if self.size == 0:
             raise RuntimeError, "Model without parameters cannot be minimized"
-        
+
         if optimizer is None:
             optimizer = self.preferred_optimizer
 
@@ -271,7 +271,7 @@ class Model(Parameterized):
            and numerical gradients is within <tolerance> of unity.
         """
         x = self._get_params_transformed().copy()
-        
+
         if not verbose:
             # make sure only to test the selected parameters
             if target_param is None:
@@ -298,12 +298,10 @@ class Model(Parameterized):
 
             dx = dx[transformed_index]
             gradient = gradient[transformed_index]
-            
+
             denominator = (2 * np.dot(dx, gradient))
             global_ratio = (f1 - f2) / np.where(denominator==0., 1e-32, denominator)
-            gloabl_diff = (f1 - f2) - denominator
-        
-            return (np.abs(1. - global_ratio) < tolerance) or (np.abs(gloabl_diff) < tolerance) 
+            return np.abs(1. - global_ratio) < tolerance or np.abs(f1-f2).sum() + np.abs((2 * np.dot(dx, gradient))).sum() < tolerance
         else:
             # check the gradient of each parameter individually, and do some pretty printing
             try:
@@ -339,7 +337,7 @@ class Model(Parameterized):
                     print "No free parameters to check"
                     return
 
-            gradient = self.objective_function_gradients(x)
+            gradient = self.objective_function_gradients(x).copy()
             np.where(gradient == 0, 1e-312, gradient)
             ret = True
             for nind, xind in itertools.izip(param_index, transformed_index):
@@ -349,7 +347,7 @@ class Model(Parameterized):
                 xx[xind] -= 2.*step
                 f2 = self.objective_function(xx)
                 numerical_gradient = (f1 - f2) / (2 * step)
-                if np.all(gradient[xind]==0): ratio = (f1-f2) == gradient[xind] 
+                if np.all(gradient[xind]==0): ratio = (f1-f2) == gradient[xind]
                 else: ratio = (f1 - f2) / (2 * step * gradient[xind])
                 difference = np.abs((f1 - f2) / 2 / step - gradient[xind])
 
@@ -366,7 +364,7 @@ class Model(Parameterized):
                 ng = '%.6f' % float(numerical_gradient)
                 grad_string = "{0:<{c0}}|{1:^{c1}}|{2:^{c2}}|{3:^{c3}}|{4:^{c4}}".format(formatted_name, r, d, g, ng, c0=cols[0] + 9, c1=cols[1], c2=cols[2], c3=cols[3], c4=cols[4])
                 print grad_string
-                
+
             self._set_params_transformed(x)
             return ret
 

GPy/core/parameterization/__init__.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2012, GPy authors (see AUTHORS.txt).
 # Licensed under the BSD 3-clause license (see LICENSE.txt)
 
-from param import Param, ObservableArray
+from param import Param, ObsAr
 from parameterized import Parameterized

GPy/core/parameterization/array_core.py

@@ -1,37 +1,43 @@
 # Copyright (c) 2012, GPy authors (see AUTHORS.txt).
 # Licensed under the BSD 3-clause license (see LICENSE.txt)
 
-__updated__ = '2013-12-16'
+__updated__ = '2014-03-17'
 
 import numpy as np
 from parameter_core import Observable
 
-class ObservableArray(np.ndarray, Observable):
+class ObsAr(np.ndarray, Observable):
     """
     An ndarray which reports changes to its observers.
     The observers can add themselves with a callable, which
     will be called every time this array changes. The callable
     takes exactly one argument, which is this array itself.
     """
-    __array_priority__ = -1 # Never give back ObservableArray
-    def __new__(cls, input_array):
-        if not isinstance(input_array, ObservableArray):
-            obj = np.atleast_1d(input_array).view(cls)
+    __array_priority__ = -1 # Never give back ObsAr
+    def __new__(cls, input_array, *a, **kw):
+        if not isinstance(input_array, ObsAr):
+            obj = np.atleast_1d(np.require(input_array, dtype=np.float64, requirements=['W', 'C'])).view(cls)
         else: obj = input_array
-        cls.__name__ = "ObservableArray\n     "
+        #cls.__name__ = "ObsAr" # because of fixed printing of `array` in np printing
+        super(ObsAr, obj).__init__(*a, **kw)
         return obj
-    
-    def __init__(self, *a, **kw):
-        super(ObservableArray, self).__init__(*a, **kw)
-    
+
     def __array_finalize__(self, obj):
         # see InfoArray.__array_finalize__ for comments
         if obj is None: return
         self._observer_callables_ = getattr(obj, '_observer_callables_', None)
-        
+
     def __array_wrap__(self, out_arr, context=None):
         return out_arr.view(np.ndarray)
 
+    def __reduce__(self):
+        func, args, state = np.ndarray.__reduce__(self)
+        return func, args, (state, Observable._getstate(self))
+
+    def __setstate__(self, state):
+        np.ndarray.__setstate__(self, state[0])
+        Observable._setstate(self, state[1])
+
     def _s_not_empty(self, s):
         # this checks whether there is something picked by this slice.
         return True
@@ -48,17 +54,17 @@ class ObservableArray(np.ndarray, Observable):
 
     def __setitem__(self, s, val):
         if self._s_not_empty(s):
-            super(ObservableArray, self).__setitem__(s, val)
+            super(ObsAr, self).__setitem__(s, val)
             self.notify_observers(self[s])
-                
+
     def __getslice__(self, start, stop):
         return self.__getitem__(slice(start, stop))
-    
+
     def __setslice__(self, start, stop, val):
         return self.__setitem__(slice(start, stop), val)
 
     def __copy__(self, *args):
-        return ObservableArray(self.view(np.ndarray).copy())
+        return ObsAr(self.view(np.ndarray).copy())
 
     def copy(self, *args):
         return self.__copy__(*args)
@@ -85,7 +91,7 @@ class ObservableArray(np.ndarray, Observable):
         self.notify_observers()
         return r
 
-    
+
     def __ifloordiv__(self, *args, **kwargs):
         r = np.ndarray.__ifloordiv__(self, *args, **kwargs)
         self.notify_observers()

GPy/core/parameterization/index_operations.py

@@ -23,17 +23,16 @@ class ParameterIndexOperations(object):
         if constraints is not None:
             for t, i in constraints.iteritems():
                 self.add(t, i)
-        
+
     def __getstate__(self):
-        return self._properties#, self._reverse
-        
+        return self._properties
+
     def __setstate__(self, state):
-        self._properties = state[0]
-        # self._reverse = state[1]
+        self._properties = state
 
     def iteritems(self):
         return self._properties.iteritems()
-    
+
     def items(self):
         return self._properties.items()
 
@@ -42,7 +41,7 @@ class ParameterIndexOperations(object):
 
     def iterproperties(self):
         return self._properties.iterkeys()
-    
+
     def shift_right(self, start, size):
         for ind in self.iterindices():
             toshift = ind>=start
@@ -58,29 +57,26 @@ class ParameterIndexOperations(object):
                 ind[toshift] -= size
             if ind.size != 0: self._properties[v] = ind
             else: del self._properties[v]
-    
+
     def clear(self):
         self._properties.clear()
-    
+
     @property
     def size(self):
-        return reduce(lambda a,b: a+b.size, self.iterindices(), 0)    
-    
+        return reduce(lambda a,b: a+b.size, self.iterindices(), 0)
+
     def iterindices(self):
         return self._properties.itervalues()
-    
+
     def indices(self):
         return self._properties.values()
 
     def properties_for(self, index):
         return vectorize(lambda i: [prop for prop in self.iterproperties() if i in self[prop]], otypes=[list])(index)
-        
+
     def add(self, prop, indices):
-        try:
-            self._properties[prop] = combine_indices(self._properties[prop], indices)
-        except KeyError:
-            self._properties[prop] = indices
-    
+        self._properties[prop] = combine_indices(self._properties[prop], indices)
+
     def remove(self, prop, indices):
         if prop in self._properties:
             diff = remove_indices(self[prop], indices)
@@ -91,22 +87,22 @@ class ParameterIndexOperations(object):
                 del self._properties[prop]
             return removed.astype(int)
         return numpy.array([]).astype(int)
-    
+
     def update(self, parameter_index_view, offset=0):
         for i, v in parameter_index_view.iteritems():
             self.add(i, v+offset)
 
-    
+
     def copy(self):
         return ParameterIndexOperations(dict(self.iteritems()))
-    
+
     def __getitem__(self, prop):
         return self._properties[prop]
-    
+
     def __str__(self, *args, **kwargs):
         import pprint
         return pprint.pformat(dict(self._properties))
-       
+
 def combine_indices(arr1, arr2):
     return numpy.union1d(arr1, arr2)
 
@@ -114,24 +110,22 @@ def remove_indices(arr, to_remove):
     return numpy.setdiff1d(arr, to_remove, True)
 
 def index_empty(index):
-    return numpy.size(index) == 0 
+    return numpy.size(index) == 0
 
 class ParameterIndexOperationsView(object):
     def __init__(self, param_index_operations, offset, size):
         self._param_index_ops = param_index_operations
         self._offset = offset
         self._size = size
-    
+
     def __getstate__(self):
         return [self._param_index_ops, self._offset, self._size]
 
-
     def __setstate__(self, state):
         self._param_index_ops = state[0]
         self._offset = state[1]
         self._size = state[2]
 
-
     def _filter_index(self, ind):
         return ind[(ind >= self._offset) * (ind < (self._offset + self._size))] - self._offset
 
@@ -140,7 +134,7 @@ class ParameterIndexOperationsView(object):
         for i, ind in self._param_index_ops.iteritems():
             ind2 = self._filter_index(ind)
             if ind2.size > 0:
-                yield i, ind2 
+                yield i, ind2
 
     def items(self):
         return [[i,v] for i,v in self.iteritems()]
@@ -151,7 +145,7 @@ class ParameterIndexOperationsView(object):
 
     def iterproperties(self):
         for i, _ in self.iteritems():
-            yield i 
+            yield i
 
 
     def shift_right(self, start, size):
@@ -161,7 +155,7 @@ class ParameterIndexOperationsView(object):
         self._param_index_ops.shift_left(start+self._offset, size)
         self._offset -= size
         self._size -= size
-            
+
     def clear(self):
         for i, ind in self.items():
             self._param_index_ops.remove(i, ind+self._offset)
@@ -198,7 +192,7 @@ class ParameterIndexOperationsView(object):
     def __getitem__(self, prop):
         ind = self._filter_index(self._param_index_ops[prop])
         return ind
-    
+
     def __str__(self, *args, **kwargs):
         import pprint
         return pprint.pformat(dict(self.iteritems()))
@@ -206,8 +200,8 @@ class ParameterIndexOperationsView(object):
     def update(self, parameter_index_view, offset=0):
         for i, v in parameter_index_view.iteritems():
             self.add(i, v+offset)
-    
-    
+
+
     def copy(self):
         return ParameterIndexOperations(dict(self.iteritems()))
     pass

GPy/core/parameterization/lists_and_dicts.py

@@ -5,21 +5,17 @@ Created on 27 Feb 2014
 '''
 
 from collections import defaultdict
-class DefaultArrayDict(defaultdict):
-    def __init__(self):
+
+def intarray_default_factory():
+    import numpy as np
+    return np.int_([])
+
+class IntArrayDict(defaultdict):
+    def __init__(self, default_factory=None):
         """
         Default will be self._default, if not set otherwise
         """
-        defaultdict.__init__(self, self.default_factory)
-
-class SetDict(DefaultArrayDict):
-    def default_factory(self):
-        return set()
-
-class IntArrayDict(DefaultArrayDict):
-    def default_factory(self):
-        import numpy as np
-        return np.int_([])
+        defaultdict.__init__(self, intarray_default_factory)
 
 class ArrayList(list):
     """

GPy/core/parameterization/param.py

@@ -3,8 +3,8 @@
 
 import itertools
 import numpy
-from parameter_core import OptimizationHandlable, Gradcheckable, adjust_name_for_printing
-from array_core import ObservableArray
+from parameter_core import OptimizationHandlable, adjust_name_for_printing
+from array_core import ObsAr
 
 ###### printing
 __constraints_name__ = "Constraint"
@@ -15,7 +15,7 @@ __precision__ = numpy.get_printoptions()['precision'] # numpy printing precision
 __print_threshold__ = 5
 ######
 
-class Param(OptimizationHandlable, ObservableArray):
+class Param(OptimizationHandlable, ObsAr):
     """
     Parameter object for GPy models.
 
@@ -43,16 +43,12 @@ class Param(OptimizationHandlable, ObservableArray):
     _fixes_ = None
     _parameters_ = []
     def __new__(cls, name, input_array, default_constraint=None):
-        obj = numpy.atleast_1d(super(Param, cls).__new__(cls, input_array=input_array))
+        obj = numpy.atleast_1d(super(Param, cls).__new__(cls, input_array=input_array, name=name, default_constraint=default_constraint))
         cls.__name__ = "Param"
         obj._current_slice_ = (slice(obj.shape[0]),)
         obj._realshape_ = obj.shape
         obj._realsize_ = obj.size
         obj._realndim_ = obj.ndim
-        obj._updated_ = False
-        from lists_and_dicts import SetDict
-        obj._tied_to_me_ = SetDict()
-        obj._tied_to_ = []
         obj._original_ = True
         obj._gradient_array_ = numpy.zeros(obj.shape, dtype=numpy.float64)
         return obj
@@ -81,14 +77,11 @@ class Param(OptimizationHandlable, ObservableArray):
         self._parent_index_ = getattr(obj, '_parent_index_', None)
         self._default_constraint_ = getattr(obj, '_default_constraint_', None)
         self._current_slice_ = getattr(obj, '_current_slice_', None)
-        self._tied_to_me_ = getattr(obj, '_tied_to_me_', None)
-        self._tied_to_ = getattr(obj, '_tied_to_', None)
         self._realshape_ = getattr(obj, '_realshape_', None)
         self._realsize_ = getattr(obj, '_realsize_', None)
         self._realndim_ = getattr(obj, '_realndim_', None)
-        self._updated_ = getattr(obj, '_updated_', None)
         self._original_ = getattr(obj, '_original_', None)
-        self._name = getattr(obj, 'name', None)
+        self._name = getattr(obj, '_name', None)
         self._gradient_array_ = getattr(obj, '_gradient_array_', None)
         self.constraints = getattr(obj, 'constraints', None)
         self.priors = getattr(obj, 'priors', None)
@@ -96,22 +89,22 @@ class Param(OptimizationHandlable, ObservableArray):
     @property
     def _param_array_(self):
         return self
-    
+
     @property
     def gradient(self):
         return self._gradient_array_[self._current_slice_]
-     
+
     @gradient.setter
     def gradient(self, val):
         self.gradient[:] = val
-        
+
     #===========================================================================
     # Pickling operations
     #===========================================================================
-    def __reduce_ex__(self):
+    def __reduce__(self):
         func, args, state = super(Param, self).__reduce__()
         return func, args, (state,
-                            (self.name,
+                            (self._name,
                              self._parent_,
                              self._parent_index_,
                              self._default_constraint_,
@@ -119,18 +112,16 @@ class Param(OptimizationHandlable, ObservableArray):
                              self._realshape_,
                              self._realsize_,
                              self._realndim_,
-                             self._tied_to_me_,
-                             self._tied_to_,
-                             self._updated_,
+                             self.constraints,
+                             self.priors
                             )
                             )
 
     def __setstate__(self, state):
         super(Param, self).__setstate__(state[0])
         state = list(state[1])
-        self._updated_ = state.pop()
-        self._tied_to_ = state.pop()
-        self._tied_to_me_ = state.pop()
+        self.priors = state.pop()
+        self.constraints = state.pop()
         self._realndim_ = state.pop()
         self._realsize_ = state.pop()
         self._realshape_ = state.pop()
@@ -138,8 +129,8 @@ class Param(OptimizationHandlable, ObservableArray):
         self._default_constraint_ = state.pop()
         self._parent_index_ = state.pop()
         self._parent_ = state.pop()
-        self.name = state.pop()
-    
+        self._name = state.pop()
+
     def copy(self, *args):
         constr = self.constraints.copy()
         priors = self.priors.copy()
@@ -155,13 +146,13 @@ class Param(OptimizationHandlable, ObservableArray):
 #         if trigger_parent: min_priority = None
 #         else: min_priority = -numpy.inf
 #         self.notify_observers(None, min_priority)
-# 
+#
 #     def _get_params(self):
 #         return self.flat
-# 
+#
 #     def _collect_gradient(self, target):
 #         target += self.gradient.flat
-# 
+#
 #     def _set_gradient(self, g):
 #         self.gradient = g.reshape(self._realshape_)
 
@@ -177,29 +168,29 @@ class Param(OptimizationHandlable, ObservableArray):
         try: new_arr._current_slice_ = s; new_arr._original_ = self.base is new_arr.base
         except AttributeError: pass  # returning 0d array or float, double etc
         return new_arr
-    
+
     def __setitem__(self, s, val):
         super(Param, self).__setitem__(s, val)
-        
+
     #===========================================================================
     # Index Operations:
     #===========================================================================
-    def _internal_offset(self):
-        internal_offset = 0
-        extended_realshape = numpy.cumprod((1,) + self._realshape_[:0:-1])[::-1]
-        for i, si in enumerate(self._current_slice_[:self._realndim_]):
-            if numpy.all(si == Ellipsis):
-                continue
-            if isinstance(si, slice):
-                a = si.indices(self._realshape_[i])[0]
-            elif isinstance(si, (list,numpy.ndarray,tuple)):
-                a = si[0]
-            else: a = si
-            if a < 0:
-                a = self._realshape_[i] + a
-            internal_offset += a * extended_realshape[i]
-        return internal_offset
-    
+    #def _internal_offset(self):
+    #    internal_offset = 0
+    #    extended_realshape = numpy.cumprod((1,) + self._realshape_[:0:-1])[::-1]
+    #    for i, si in enumerate(self._current_slice_[:self._realndim_]):
+    #        if numpy.all(si == Ellipsis):
+    #            continue
+    #        if isinstance(si, slice):
+    #            a = si.indices(self._realshape_[i])[0]
+    #        elif isinstance(si, (list,numpy.ndarray,tuple)):
+    #            a = si[0]
+    #        else: a = si
+    #        if a < 0:
+    #            a = self._realshape_[i] + a
+    #        internal_offset += a * extended_realshape[i]
+    #    return internal_offset
+
     def _raveled_index(self, slice_index=None):
         # return an index array on the raveled array, which is formed by the current_slice
         # of this object
@@ -207,7 +198,10 @@ class Param(OptimizationHandlable, ObservableArray):
         ind = self._indices(slice_index)
         if ind.ndim < 2: ind = ind[:, None]
         return numpy.asarray(numpy.apply_along_axis(lambda x: numpy.sum(extended_realshape * x), 1, ind), dtype=int)
-    
+
+    def _raveled_index_for(self, obj):
+        return self._raveled_index()
+
     def _expand_index(self, slice_index=None):
         # this calculates the full indexing arrays from the slicing objects given by get_item for _real..._ attributes
         # it basically translates slices to their respective index arrays and turns negative indices around
@@ -228,6 +222,11 @@ class Param(OptimizationHandlable, ObservableArray):
                 return numpy.r_[a]
             return numpy.r_[:b]
         return itertools.imap(f, itertools.izip_longest(slice_index[:self._realndim_], self._realshape_, fillvalue=slice(self.size)))
+    #===========================================================================
+    # Constrainable
+    #===========================================================================
+    def _ensure_fixes(self):
+        if not self._has_fixes(): self._fixes_ = numpy.ones(self._realsize_, dtype=bool)
 
     #===========================================================================
     # Convenience
@@ -243,13 +242,12 @@ class Param(OptimizationHandlable, ObservableArray):
     #round.__doc__ = numpy.round.__doc__
     def _get_original(self, param):
         return self
-
     #===========================================================================
     # Printing -> done
     #===========================================================================
     @property
     def _description_str(self):
-        if self.size <= 1: 
+        if self.size <= 1:
             return [str(self.view(numpy.ndarray)[0])]
         else: return [str(self.shape)]
     def parameter_names(self, add_self=False, adjust_for_printing=False):
@@ -270,23 +268,13 @@ class Param(OptimizationHandlable, ObservableArray):
         return [' '.join(map(lambda c: str(c[0]) if c[1].size == self._realsize_ else "{" + str(c[0]) + "}", self.priors.iteritems()))]
     @property
     def _ties_str(self):
-        return [t._short() for t in self._tied_to_] or ['']
+        return ['']
+    def _ties_for(self, ravi):
+        return [['N/A']]*ravi.size
     def __repr__(self, *args, **kwargs):
         name = "\033[1m{x:s}\033[0;0m:\n".format(
                             x=self.hierarchy_name())
         return name + super(Param, self).__repr__(*args, **kwargs)
-    def _ties_for(self, rav_index):
-        # size = sum(p.size for p in self._tied_to_)
-        ties = numpy.empty(shape=(len(self._tied_to_), numpy.size(rav_index)), dtype=Param)
-        for i, tied_to in enumerate(self._tied_to_):
-            for t, ind in tied_to._tied_to_me_.iteritems():
-                if t._parent_index_ == self._parent_index_:
-                    matches = numpy.where(rav_index[:, None] == t._raveled_index()[None, :])
-                    tt_rav_index = tied_to._raveled_index()
-                    ind_rav_matches = numpy.where(tt_rav_index == numpy.array(list(ind)))[0]
-                    if len(ind) != 1: ties[i, matches[0][ind_rav_matches]] = numpy.take(tt_rav_index, matches[1], mode='wrap')[ind_rav_matches]
-                    else: ties[i, matches[0]] = numpy.take(tt_rav_index, matches[1], mode='wrap')
-        return map(lambda a: sum(a, []), zip(*[[[tie.flatten()] if tx != None else [] for tx in t] for t, tie in zip(ties, self._tied_to_)]))
     def _indices(self, slice_index=None):
         # get a int-array containing all indices in the first axis.
         if slice_index is None:
@@ -327,7 +315,7 @@ class Param(OptimizationHandlable, ObservableArray):
         if constr_matrix is None: constr_matrix = self.constraints.properties_for(ravi)
         if prirs is None: prirs = self.priors.properties_for(ravi)
         if ties is None: ties = self._ties_for(ravi)
-        ties = [' '.join(map(lambda x: x._short(), t)) for t in ties]
+        ties = [' '.join(map(lambda x: x, t)) for t in ties]
         if lc is None: lc = self._max_len_names(constr_matrix, __constraints_name__)
         if lx is None: lx = self._max_len_values()
         if li is None: li = self._max_len_index(indices)
@@ -360,7 +348,7 @@ class ParamConcatenation(object):
         self._param_sizes = [p.size for p in self.params]
         startstops = numpy.cumsum([0] + self._param_sizes)
         self._param_slices_ = [slice(start, stop) for start,stop in zip(startstops, startstops[1:])]
-        
+
         parents = dict()
         for p in self.params:
             if p.has_parent():
@@ -400,7 +388,7 @@ class ParamConcatenation(object):
     def update_all_params(self):
         for par in self.parents:
             par.notify_observers(-numpy.inf)
-        
+
     def constrain(self, constraint, warning=True):
         [param.constrain(constraint, trigger_parent=False) for param in self.params]
         self.update_all_params()

GPy/core/parameterization/parameter_core.py

@@ -1,7 +1,7 @@
 # Copyright (c) 2012, GPy authors (see AUTHORS.txt).
 # Licensed under the BSD 3-clause license (see LICENSE.txt)
 """
-Core module for parameterization. 
+Core module for parameterization.
 This module implements all parameterization techniques, split up in modular bits.
 
 HierarchyError:
@@ -15,9 +15,8 @@ Observable Pattern for patameterization
 
 from transformations import Transformation, Logexp, NegativeLogexp, Logistic, __fixed__, FIXED, UNFIXED
 import numpy as np
-import itertools
 
-__updated__ = '2013-12-16'
+__updated__ = '2014-03-18'
 
 class HierarchyError(Exception):
     """
@@ -32,101 +31,10 @@ def adjust_name_for_printing(name):
         return name.replace(" ", "_").replace(".", "_").replace("-", "_m_").replace("+", "_p_").replace("!", "_I_").replace("**", "_xx_").replace("*", "_x_").replace("/", "_l_").replace("@",'_at_')
     return ''
 
-class Observable(object):
-    """
-    Observable pattern for parameterization.
-    
-    This Object allows for observers to register with self and a (bound!) function
-    as an observer. Every time the observable changes, it sends a notification with
-    self as only argument to all its observers.
-    """
-    _updated = True
-    def __init__(self, *args, **kwargs):
-        self._observer_callables_ = []
-    def __del__(self, *args, **kwargs):
-        del self._observer_callables_
-        
-    def add_observer(self, observer, callble, priority=0):
-        self._insert_sorted(priority, observer, callble)
-    
-    def remove_observer(self, observer, callble=None):
-        to_remove = []
-        for p, obs, clble in self._observer_callables_:
-            if callble is not None:
-                if (obs == observer) and (callble == clble):
-                    to_remove.append((p, obs, clble))
-            else:
-                if obs is observer:
-                    to_remove.append((p, obs, clble))
-        for r in to_remove:
-            self._observer_callables_.remove(r)
-       
-    def notify_observers(self, which=None, min_priority=None):
-        """
-        Notifies all observers. Which is the element, which kicked off this 
-        notification loop.
-        
-        NOTE: notifies only observers with priority p > min_priority!
-                                                    ^^^^^^^^^^^^^^^^
-        
-        :param which: object, which started this notification loop
-        :param min_priority: only notify observers with priority > min_priority
-                             if min_priority is None, notify all observers in order
-        """
-        if which is None:
-            which = self
-        if min_priority is None:
-            [callble(which) for _, _, callble in self._observer_callables_]
-        else:
-            for p, _, callble in self._observer_callables_:
-                if p <= min_priority:
-                    break
-                callble(which)
+class InterfacePickleFunctions(object):
+    def __init__(self, *a, **kw):
+        super(InterfacePickleFunctions, self).__init__()
 
-    def _insert_sorted(self, p, o, c):
-        ins = 0
-        for pr, _, _ in self._observer_callables_:
-            if p > pr:
-                break
-            ins += 1
-        self._observer_callables_.insert(ins, (p, o, c))
-        
-class Pickleable(object):
-    """
-    Make an object pickleable (See python doc 'pickling'). 
-    
-    This class allows for pickling support by Memento pattern.
-    _getstate returns a memento of the class, which gets pickled.
-    _setstate(<memento>) (re-)sets the state of the class to the memento 
-    """
-    #===========================================================================
-    # Pickling operations
-    #===========================================================================
-    def pickle(self, f, protocol=-1):
-        """
-        :param f: either filename or open file object to write to.
-                  if it is an open buffer, you have to make sure to close
-                  it properly.
-        :param protocol: pickling protocol to use, python-pickle for details.
-        """
-        import cPickle
-        if isinstance(f, str):
-            with open(f, 'w') as f:
-                cPickle.dump(self, f, protocol)
-        else:
-            cPickle.dump(self, f, protocol)    
-    def __getstate__(self):
-        if self._has_get_set_state():
-            return self._getstate()
-        return self.__dict__
-    def __setstate__(self, state):
-        if self._has_get_set_state():
-            self._setstate(state)  
-            # TODO: maybe parameters_changed() here?
-            return
-        self.__dict__ = state
-    def _has_get_set_state(self):
-        return '_getstate' in vars(self.__class__) and '_setstate' in vars(self.__class__)
     def _getstate(self):
         """
         Returns the state of this class in a memento pattern.
@@ -148,20 +56,125 @@ class Pickleable(object):
         """
         raise NotImplementedError, "To be able to use pickling you need to implement this method"
 
+class Pickleable(InterfacePickleFunctions):
+    """
+    Make an object pickleable (See python doc 'pickling').
+
+    This class allows for pickling support by Memento pattern.
+    _getstate returns a memento of the class, which gets pickled.
+    _setstate(<memento>) (re-)sets the state of the class to the memento
+    """
+    def __init__(self, *a, **kw):
+        super(Pickleable, self).__init__()
+    #===========================================================================
+    # Pickling operations
+    #===========================================================================
+    def pickle(self, f, protocol=-1):
+        """
+        :param f: either filename or open file object to write to.
+                  if it is an open buffer, you have to make sure to close
+                  it properly.
+        :param protocol: pickling protocol to use, python-pickle for details.
+        """
+        import cPickle
+        if isinstance(f, str):
+            with open(f, 'w') as f:
+                cPickle.dump(self, f, protocol)
+        else:
+            cPickle.dump(self, f, protocol)
+    def __getstate__(self):
+        if self._has_get_set_state():
+            return self._getstate()
+        return self.__dict__
+    def __setstate__(self, state):
+        if self._has_get_set_state():
+            self._setstate(state)
+            # TODO: maybe parameters_changed() here?
+            return
+        self.__dict__ = state
+    def _has_get_set_state(self):
+        return '_getstate' in vars(self.__class__) and '_setstate' in vars(self.__class__)
+
+class Observable(Pickleable):
+    """
+    Observable pattern for parameterization.
+
+    This Object allows for observers to register with self and a (bound!) function
+    as an observer. Every time the observable changes, it sends a notification with
+    self as only argument to all its observers.
+    """
+    _updated = True
+    def __init__(self, *args, **kwargs):
+        super(Observable, self).__init__(*args, **kwargs)
+        self._observer_callables_ = []
+
+    def add_observer(self, observer, callble, priority=0):
+        self._insert_sorted(priority, observer, callble)
+
+    def remove_observer(self, observer, callble=None):
+        to_remove = []
+        for p, obs, clble in self._observer_callables_:
+            if callble is not None:
+                if (obs == observer) and (callble == clble):
+                    to_remove.append((p, obs, clble))
+            else:
+                if obs is observer:
+                    to_remove.append((p, obs, clble))
+        for r in to_remove:
+            self._observer_callables_.remove(r)
+
+    def notify_observers(self, which=None, min_priority=None):
+        """
+        Notifies all observers. Which is the element, which kicked off this
+        notification loop.
+
+        NOTE: notifies only observers with priority p > min_priority!
+                                                    ^^^^^^^^^^^^^^^^
+
+        :param which: object, which started this notification loop
+        :param min_priority: only notify observers with priority > min_priority
+                             if min_priority is None, notify all observers in order
+        """
+        if which is None:
+            which = self
+        if min_priority is None:
+            [callble(which) for _, _, callble in self._observer_callables_]
+        else:
+            for p, _, callble in self._observer_callables_:
+                if p <= min_priority:
+                    break
+                callble(which)
+
+    def _insert_sorted(self, p, o, c):
+        ins = 0
+        for pr, _, _ in self._observer_callables_:
+            if p > pr:
+                break
+            ins += 1
+        self._observer_callables_.insert(ins, (p, o, c))
+
+    def _getstate(self):
+        return [self._observer_callables_]
+
+    def _setstate(self, state):
+        self._observer_callables_ = state.pop()
+
 #===============================================================================
 # Foundation framework for parameterized and param objects:
 #===============================================================================
 
-class Parentable(object):
+class Parentable(Observable):
     """
     Enable an Object to have a parent.
-    
+
     Additionally this adds the parent_index, which is the index for the parent
     to look for in its parameter list.
     """
     _parent_ = None
     _parent_index_ = None
-        
+    def __init__(self, *args, **kwargs):
+        super(Parentable, self).__init__(*args, **kwargs)
+
     def has_parent(self):
         """
         Return whether this parentable object currently has a parent.
@@ -201,19 +214,20 @@ class Gradcheckable(Parentable):
     Adds the functionality for an object to be gradcheckable.
     It is just a thin wrapper of a call to the highest parent for now.
     TODO: Can be done better, by only changing parameters of the current parameter handle,
-    such that object hierarchy only has to change for those. 
+    such that object hierarchy only has to change for those.
     """
     def __init__(self, *a, **kw):
         super(Gradcheckable, self).__init__(*a, **kw)
+
     def checkgrad(self, verbose=0, step=1e-6, tolerance=1e-3):
         """
-        Check the gradient of this parameter with respect to the highest parent's 
+        Check the gradient of this parameter with respect to the highest parent's
         objective function.
         This is a three point estimate of the gradient, wiggling at the parameters
         with a stepsize step.
-        The check passes if either the ratio or the difference between numerical and 
+        The check passes if either the ratio or the difference between numerical and
         analytical gradient is smaller then tolerance.
-        
+
         :param bool verbose: whether each parameter shall be checked individually.
         :param float step: the stepsize for the numerical three point gradient estimate.
         :param flaot tolerance: the tolerance for the gradient ratio or difference.
@@ -221,12 +235,13 @@ class Gradcheckable(Parentable):
         if self.has_parent():
             return self._highest_parent_._checkgrad(self, verbose=verbose, step=step, tolerance=tolerance)
         return self._checkgrad(self[''], verbose=verbose, step=step, tolerance=tolerance)
-    def _checkgrad(self, param):
+
+    def _checkgrad(self, param, verbose=0, step=1e-6, tolerance=1e-3):
         """
         Perform the checkgrad on the model.
         TODO: this can be done more efficiently, when doing it inside here
         """
-        raise NotImplementedError, "Need log likelihood to check gradient against"
+        raise HierarchyError, "This parameter is not in a model with a likelihood, and, therefore, cannot be gradient checked!"
 
 
 class Nameable(Gradcheckable):
@@ -257,7 +272,7 @@ class Nameable(Gradcheckable):
     def hierarchy_name(self, adjust_for_printing=True):
         """
         return the name for this object with the parents names attached by dots.
-        
+
         :param bool adjust_for_printing: whether to call :func:`~adjust_for_printing()`
         on the names, recursively
         """
@@ -272,20 +287,16 @@ class Indexable(object):
     Enable enraveled indexes and offsets for this object.
     The raveled index of an object is the index for its parameters in a flattened int array.
     """
+    def __init__(self, *a, **kw):
+        super(Indexable, self).__init__()
+
     def _raveled_index(self):
         """
         Flattened array of ints, specifying the index of this object.
         This has to account for shaped parameters!
         """
         raise NotImplementedError, "Need to be able to get the raveled Index"
-        
-    def _internal_offset(self):
-        """
-        The offset for this parameter inside its parent. 
-        This has to account for shaped parameters!
-        """
-        return 0
-    
+
     def _offset_for(self, param):
         """
         Return the offset of the param inside this parameterized object.
@@ -293,15 +304,15 @@ class Indexable(object):
         basically just sums up the parameter sizes which come before param.
         """
         raise NotImplementedError, "shouldnt happen, offset required from non parameterization object?"
-    
+
     def _raveled_index_for(self, param):
         """
         get the raveled index for a param
         that is an int array, containing the indexes for the flattened
         param inside this parameterized logic.
         """
-        raise NotImplementedError, "shouldnt happen, raveld index transformation required from non parameterization object?"        
-        
+        raise NotImplementedError, "shouldnt happen, raveld index transformation required from non parameterization object?"
+
 
 class Constrainable(Nameable, Indexable):
     """
@@ -310,7 +321,7 @@ class Constrainable(Nameable, Indexable):
     Adding a constraint to a Parameter means to tell the highest parent that
     the constraint was added and making sure that all parameters covered
     by this object are indeed conforming to the constraint.
-    
+
     :func:`constrain()` and :func:`unconstrain()` are main methods here
     """
     def __init__(self, name, default_constraint=None, *a, **kw):
@@ -321,7 +332,7 @@ class Constrainable(Nameable, Indexable):
         self.priors = ParameterIndexOperations()
         if self._default_constraint_ is not None:
             self.constrain(self._default_constraint_)
-    
+
     def _disconnect_parent(self, constr=None, *args, **kw):
         """
         From Parentable:
@@ -335,7 +346,7 @@ class Constrainable(Nameable, Indexable):
         self._parent_index_ = None
         self._connect_fixes()
         self._notify_parent_change()
-        
+
     #===========================================================================
     # Fixing Parameters:
     #===========================================================================
@@ -347,40 +358,47 @@ class Constrainable(Nameable, Indexable):
         """
         if value is not None:
             self[:] = value
-        self.constrain(__fixed__, warning=warning, trigger_parent=trigger_parent)
+        reconstrained = self.unconstrain()
+        self._add_to_index_operations(self.constraints, reconstrained, __fixed__, warning)
         rav_i = self._highest_parent_._raveled_index_for(self)
         self._highest_parent_._set_fixed(rav_i)
+        self.notify_observers(self, None if trigger_parent else -np.inf)
     fix = constrain_fixed
-    
+
     def unconstrain_fixed(self):
         """
         This parameter will no longer be fixed.
         """
         unconstrained = self.unconstrain(__fixed__)
-        self._highest_parent_._set_unfixed(unconstrained)    
+        self._highest_parent_._set_unfixed(unconstrained)
     unfix = unconstrain_fixed
-    
-    def _set_fixed(self, index):
+
+    def _ensure_fixes(self):
+        # Ensure that the fixes array is set:
+        # Parameterized: ones(self.size)
+        # Param: ones(self._realsize_
         if not self._has_fixes(): self._fixes_ = np.ones(self.size, dtype=bool)
+
+    def _set_fixed(self, index):
+        self._ensure_fixes()
         self._fixes_[index] = FIXED
         if np.all(self._fixes_): self._fixes_ = None  # ==UNFIXED
-    
+
     def _set_unfixed(self, index):
-        if not self._has_fixes(): self._fixes_ = np.ones(self.size, dtype=bool)
-        # rav_i = self._raveled_index_for(param)[index]
+        self._ensure_fixes()
         self._fixes_[index] = UNFIXED
         if np.all(self._fixes_): self._fixes_ = None  # ==UNFIXED
 
     def _connect_fixes(self):
         fixed_indices = self.constraints[__fixed__]
         if fixed_indices.size > 0:
-            self._fixes_ = np.ones(self.size, dtype=bool) * UNFIXED
+            self._ensure_fixes()
             self._fixes_[fixed_indices] = FIXED
         else:
             self._fixes_ = None
-    
+
     def _has_fixes(self):
-        return hasattr(self, "_fixes_") and self._fixes_ is not None
+        return hasattr(self, "_fixes_") and self._fixes_ is not None and self._fixes_.size == self.size
 
     #===========================================================================
     # Prior Operations
@@ -393,30 +411,39 @@ class Constrainable(Nameable, Indexable):
         """
         repriorized = self.unset_priors()
         self._add_to_index_operations(self.priors, repriorized, prior, warning)
-    
+
+        from domains import _REAL, _POSITIVE, _NEGATIVE
+        if prior.domain is _POSITIVE:
+            self.constrain_positive(warning)
+        elif prior.domain is _NEGATIVE:
+            self.constrain_negative(warning)
+        elif prior.domain is _REAL:
+            rav_i = self._raveled_index()
+            assert all(all(c.domain is _REAL for c in con) for con in self.constraints.properties_for(rav_i))
+
     def unset_priors(self, *priors):
         """
         Un-set all priors given from this parameter handle.
-         
+
         """
         return self._remove_from_index_operations(self.priors, priors)
-    
+
     def log_prior(self):
         """evaluate the prior"""
         if self.priors.size > 0:
-            x = self._get_params()
-            return reduce(lambda a, b: a + b, [p.lnpdf(x[ind]).sum() for p, ind in self.priors.iteritems()], 0)
+            x = self._param_array_
+            return reduce(lambda a, b: a + b, (p.lnpdf(x[ind]).sum() for p, ind in self.priors.iteritems()), 0)
         return 0.
-    
+
     def _log_prior_gradients(self):
         """evaluate the gradients of the priors"""
         if self.priors.size > 0:
-            x = self._get_params()
+            x = self._param_array_
             ret = np.zeros(x.size)
             [np.put(ret, ind, p.lnpdf_grad(x[ind])) for p, ind in self.priors.iteritems()]
             return ret
         return 0.
-        
+
     #===========================================================================
     # Constrain operations -> done
     #===========================================================================
@@ -430,10 +457,10 @@ class Constrainable(Nameable, Indexable):
         Constrain the parameter to the given
         :py:class:`GPy.core.transformations.Transformation`.
         """
-        if isinstance(transform, Transformation):
-            self._param_array_[:] = transform.initialize(self._param_array_)
+        self._param_array_[:] = transform.initialize(self._param_array_)
         reconstrained = self.unconstrain()
         self._add_to_index_operations(self.constraints, reconstrained, transform, warning)
+        self.notify_observers(self, None if trigger_parent else -np.inf)
 
     def unconstrain(self, *transforms):
         """
@@ -443,7 +470,7 @@ class Constrainable(Nameable, Indexable):
         transformats of this parameter object.
         """
         return self._remove_from_index_operations(self.constraints, transforms)
-    
+
     def constrain_positive(self, warning=True, trigger_parent=True):
         """
         :param warning: print a warning if re-constraining parameters.
@@ -488,7 +515,7 @@ class Constrainable(Nameable, Indexable):
         Remove (lower, upper) bounded constrain from this parameter/
         """
         self.unconstrain(Logistic(lower, upper))
-    
+
     def _parent_changed(self, parent):
         """
         From Parentable:
@@ -517,7 +544,7 @@ class Constrainable(Nameable, Indexable):
     def _remove_from_index_operations(self, which, what):
         """
         Helper preventing copy code.
-        Remove given what (transform prior etc) from which param index ops. 
+        Remove given what (transform prior etc) from which param index ops.
         """
         if len(what) == 0:
             transforms = which.properties()
@@ -527,53 +554,62 @@ class Constrainable(Nameable, Indexable):
             removed = np.union1d(removed, unconstrained)
             if t is __fixed__:
                 self._highest_parent_._set_unfixed(unconstrained)
-        
+
         return removed
 
-class OptimizationHandlable(Constrainable, Observable):
+class OptimizationHandlable(Constrainable):
     """
     This enables optimization handles on an Object as done in GPy 0.4.
 
-    transformed: make sure the transformations and constraints etc are handled
+    `..._transformed`: make sure the transformations and constraints etc are handled
     """
+    def __init__(self, name, default_constraint=None, *a, **kw):
+        super(OptimizationHandlable, self).__init__(name, default_constraint=default_constraint, *a, **kw)
+
     def transform(self):
         [np.put(self._param_array_, ind, c.finv(self._param_array_[ind])) for c, ind in self.constraints.iteritems() if c != __fixed__]
-    
+
     def untransform(self):
         [np.put(self._param_array_, ind, c.f(self._param_array_[ind])) for c, ind in self.constraints.iteritems() if c != __fixed__]
-        
+
     def _get_params_transformed(self):
         # transformed parameters (apply transformation rules)
         p = self._param_array_.copy()
         [np.put(p, ind, c.finv(p[ind])) for c, ind in self.constraints.iteritems() if c != __fixed__]
-        if self._has_fixes():
+        if self.has_parent() and self.constraints[__fixed__].size != 0:
+            fixes = np.ones(self.size).astype(bool)
+            fixes[self.constraints[__fixed__]] = FIXED
+            return p[fixes]
+        elif self._has_fixes():
             return p[self._fixes_]
         return p
 
     def _set_params_transformed(self, p):
         if p is self._param_array_:
             p = p.copy()
-        if self._has_fixes(): self._param_array_[self._fixes_] = p
+        if self.has_parent() and self.constraints[__fixed__].size != 0:
+            fixes = np.ones(self.size).astype(bool)
+            fixes[self.constraints[__fixed__]] = FIXED
+            self._param_array_[fixes] = p
+        elif self._has_fixes(): self._param_array_[self._fixes_] = p
         else: self._param_array_[:] = p
         self.untransform()
         self._trigger_params_changed()
-        
+
     def _trigger_params_changed(self, trigger_parent=True):
         [p._trigger_params_changed(trigger_parent=False) for p in self._parameters_]
-        if trigger_parent: min_priority = None
-        else: min_priority = -np.inf
-        self.notify_observers(None, min_priority)
-    
+        self.notify_observers(None, None if trigger_parent else -np.inf)
+
     def _size_transformed(self):
         return self.size - self.constraints[__fixed__].size
-#     
+#
 #     def _untransform_params(self, p):
 #         # inverse apply transformations for parameters
 #         #p = p.copy()
 #         if self._has_fixes(): tmp = self._get_params(); tmp[self._fixes_] = p; p = tmp; del tmp
 #         [np.put(p, ind, c.f(p[ind])) for c, ind in self.constraints.iteritems() if c != __fixed__]
 #         return p
-#     
+#
 #     def _get_params(self):
 #         """
 #         get all parameters
@@ -584,7 +620,7 @@ class OptimizationHandlable(Constrainable, Observable):
 #             return p
 #         [np.put(p, ind, par._get_params()) for ind, par in itertools.izip(self._param)]
 #         return p
-        
+
 #     def _set_params(self, params, trigger_parent=True):
 #         self._param_array_.flat = params
 #         if trigger_parent: min_priority = None
@@ -592,14 +628,14 @@ class OptimizationHandlable(Constrainable, Observable):
 #         self.notify_observers(None, min_priority)
         # don't overwrite this anymore!
         #raise NotImplementedError, "Abstract superclass: This needs to be implemented in Param and Parameterizable"
-    
+
     #===========================================================================
     # Optimization handles:
     #===========================================================================
     def _get_param_names(self):
         n = np.array([p.hierarchy_name() + '[' + str(i) + ']' for p in self.flattened_parameters for i in p._indices()])
         return n
-    
+
     def _get_param_names_transformed(self):
         n = self._get_param_names()
         if self._has_fixes():
@@ -613,7 +649,7 @@ class OptimizationHandlable(Constrainable, Observable):
         """
         Randomize the model.
         Make this draw from the prior if one exists, else draw from given random generator
-        
+
         :param rand_gen: numpy random number generator which takes args and kwargs
         :param flaot loc: loc parameter for random number generator
         :param float scale: scale parameter for random number generator
@@ -625,6 +661,24 @@ class OptimizationHandlable(Constrainable, Observable):
         [np.put(x, ind, p.rvs(ind.size)) for p, ind in self.priors.iteritems() if not p is None]
         self._set_params_transformed(x) # makes sure all of the tied parameters get the same init (since there's only one prior object...)
 
+    #===========================================================================
+    # For shared memory arrays. This does nothing in Param, but sets the memory
+    # for all parameterized objects
+    #===========================================================================
+    def _propagate_param_grad(self, parray, garray):
+        pi_old_size = 0
+        for pi in self._parameters_:
+            pislice = slice(pi_old_size, pi_old_size+pi.size)
+
+            self._param_array_[pislice] = pi._param_array_.ravel()#, requirements=['C', 'W']).flat
+            self._gradient_array_[pislice] = pi._gradient_array_.ravel()#, requirements=['C', 'W']).flat
+
+            pi._param_array_.data = parray[pislice].data
+            pi._gradient_array_.data = garray[pislice].data
+
+            pi._propagate_param_grad(parray[pislice], garray[pislice])
+            pi_old_size += pi.size
+
 class Parameterizable(OptimizationHandlable):
     def __init__(self, *args, **kwargs):
         super(Parameterizable, self).__init__(*args, **kwargs)
@@ -634,11 +688,11 @@ class Parameterizable(OptimizationHandlable):
         self._param_array_ = np.empty(self.size, dtype=np.float64)
         self._gradient_array_ = np.empty(self.size, dtype=np.float64)
         self._added_names_ = set()
-    
+
     def parameter_names(self, add_self=False, adjust_for_printing=False, recursive=True):
         """
-        Get the names of all parameters of this model. 
-        
+        Get the names of all parameters of this model.
+
         :param bool add_self: whether to add the own name in front of names
         :param bool adjust_for_printing: whether to call `adjust_name_for_printing` on names
         :param bool recursive: whether to traverse through hierarchy and append leaf node names
@@ -649,11 +703,11 @@ class Parameterizable(OptimizationHandlable):
         else: names = [adjust(x.name) for x in self._parameters_]
         if add_self: names = map(lambda x: adjust(self.name) + "." + x, names)
         return names
-    
+
     @property
     def num_params(self):
         return len(self._parameters_)
-    
+
     def _add_parameter_name(self, param, ignore_added_names=False):
         pname = adjust_name_for_printing(param.name)
         if ignore_added_names:
@@ -668,7 +722,11 @@ class Parameterizable(OptimizationHandlable):
         elif pname not in dir(self):
             self.__dict__[pname] = param
             self._added_names_.add(pname)
-            
+        else:
+            print "WARNING: added a parameter with formatted name {}, which is already a member of {} object. Trying to change the parameter name to\n   {}".format(pname, self.__class__, param.name+"_")
+            param.name += "_"
+            self._add_parameter_name(param, ignore_added_names)
+
     def _remove_parameter_name(self, param=None, pname=None):
         assert param is None or pname is None, "can only delete either param by name, or the name of a param"
         pname = adjust_name_for_printing(pname) or adjust_name_for_printing(param.name)
@@ -680,14 +738,14 @@ class Parameterizable(OptimizationHandlable):
     def _name_changed(self, param, old_name):
         self._remove_parameter_name(None, old_name)
         self._add_parameter_name(param)
-    
+
     #=========================================================================
     # Gradient handling
     #=========================================================================
     @property
     def gradient(self):
-        return self._gradient_array_ 
-    
+        return self._gradient_array_
+
     @gradient.setter
     def gradient(self, val):
         self._gradient_array_[:] = val
@@ -708,23 +766,23 @@ class Parameterizable(OptimizationHandlable):
     # def _set_gradient(self, g):
     #     [p._set_gradient(g[s]) for p, s in itertools.izip(self._parameters_, self._param_slices_)]
     #===========================================================================
-        
+
     def add_parameter(self, param, index=None, _ignore_added_names=False):
         """
         :param parameters:  the parameters to add
         :type parameters:   list of or one :py:class:`GPy.core.param.Param`
         :param [index]:     index of where to put parameters
-        
+
         :param bool _ignore_added_names: whether the name of the parameter overrides a possibly existing field
 
         Add all parameters to this param class, you can insert parameters
         at any given index using the :func:`list.insert` syntax
         """
-        # if param.has_parent():
-        #    raise AttributeError, "parameter {} already in another model, create new object (or copy) for adding".format(param._short())
         if param in self._parameters_ and index is not None:
             self.remove_parameter(param)
             self.add_parameter(param, index)
+        elif param.has_parent():
+            raise HierarchyError, "parameter {} already in another model ({}), create new object (or copy) for adding".format(param._short(), param._highest_parent_._short())
         elif param not in self._parameters_:
             if param.has_parent():
                 parent = param._parent_
@@ -745,16 +803,22 @@ class Parameterizable(OptimizationHandlable):
                 self.constraints.update(param.constraints, start)
                 self.priors.update(param.priors, start)
                 self._parameters_.insert(index, param)
-            
-            param.add_observer(self, self._pass_through_notify_observers, -np.inf)
-            
-            self.size += param.size
 
-            self._connect_parameters(ignore_added_names=_ignore_added_names)
-            self._notify_parent_change()
-            self._connect_fixes()
+            param.add_observer(self, self._pass_through_notify_observers, -np.inf)
+
+            parent = self
+            while parent is not None:
+                parent.size += param.size
+                parent = parent._parent_
+
+            self._connect_parameters()
+
+            self._highest_parent_._connect_parameters(ignore_added_names=_ignore_added_names)
+            self._highest_parent_._notify_parent_change()
+            self._highest_parent_._connect_fixes()
+
         else:
-            raise RuntimeError, """Parameter exists already added and no copy made"""
+            raise HierarchyError, """Parameter exists already and no copy made"""
 
 
     def add_parameters(self, *parameters):
@@ -770,63 +834,60 @@ class Parameterizable(OptimizationHandlable):
         """
         if not param in self._parameters_:
             raise RuntimeError, "Parameter {} does not belong to this object, remove parameters directly from their respective parents".format(param._short())
-        
+
         start = sum([p.size for p in self._parameters_[:param._parent_index_]])
         self._remove_parameter_name(param)
         self.size -= param.size
         del self._parameters_[param._parent_index_]
-        
+
         param._disconnect_parent()
         param.remove_observer(self, self._pass_through_notify_observers)
         self.constraints.shift_left(start, param.size)
-        
-        self._connect_fixes()
+
         self._connect_parameters()
         self._notify_parent_change()
-        
+
         parent = self._parent_
         while parent is not None:
-            parent._connect_fixes()
-            parent._connect_parameters()
-            parent._notify_parent_change()
+            parent.size -= param.size
             parent = parent._parent_
-        
+
+        self._highest_parent_._connect_parameters()
+        self._highest_parent_._connect_fixes()
+        self._highest_parent_._notify_parent_change()
+
     def _connect_parameters(self, ignore_added_names=False):
         # connect parameterlist to this parameterized object
         # This just sets up the right connection for the params objects
         # to be used as parameters
-        # it also sets the constraints for each parameter to the constraints 
-        # of their respective parents 
+        # it also sets the constraints for each parameter to the constraints
+        # of their respective parents
         if not hasattr(self, "_parameters_") or len(self._parameters_) < 1:
             # no parameters for this class
             return
         old_size = 0
         self._param_array_ = np.empty(self.size, dtype=np.float64)
         self._gradient_array_ = np.empty(self.size, dtype=np.float64)
-        
+
         self._param_slices_ = []
-        
         for i, p in enumerate(self._parameters_):
             p._parent_ = self
             p._parent_index_ = i
-            
+
             pslice = slice(old_size, old_size+p.size)
-            pi_old_size = old_size
-            for pi in p.flattened_parameters:
-                pislice = slice(pi_old_size, pi_old_size+pi.size)
-                
-                self._param_array_[pislice] = pi._param_array_.flat
-                self._gradient_array_[pislice] = pi._gradient_array_.flat
-                
-                pi._param_array_.data = self._param_array_[pislice].data
-                pi._gradient_array_.data = self._gradient_array_[pislice].data
-                
-                pi_old_size += pi.size
-            
+            # first connect all children
+            p._propagate_param_grad(self._param_array_[pslice], self._gradient_array_[pslice])
+            # then connect children to self
+            self._param_array_[pslice] = p._param_array_.ravel()#, requirements=['C', 'W']).ravel(order='C')
+            self._gradient_array_[pslice] = p._gradient_array_.ravel()#, requirements=['C', 'W']).ravel(order='C')
+
+            if not p._param_array_.flags['C_CONTIGUOUS']:
+                import ipdb;ipdb.set_trace()
             p._param_array_.data = self._param_array_[pslice].data
             p._gradient_array_.data = self._gradient_array_[pslice].data
-            
+
             self._param_slices_.append(pslice)
+
             self._add_parameter_name(p, ignore_added_names=ignore_added_names)
             old_size += p.size
 
@@ -837,12 +898,13 @@ class Parameterizable(OptimizationHandlable):
         self.parameters_changed()
     def _pass_through_notify_observers(self, which):
         self.notify_observers(which)
-    
+
     #===========================================================================
     # TODO: not working yet
     #===========================================================================
     def copy(self):
         """Returns a (deep) copy of the current model"""
+        raise NotImplementedError, "Copy is not yet implemented, TODO: Observable hierarchy"
         import copy
         from .index_operations import ParameterIndexOperations, ParameterIndexOperationsView
         from .lists_and_dicts import ArrayList
@@ -856,7 +918,7 @@ class Parameterizable(OptimizationHandlable):
                     dc[k] = copy.deepcopy(v)
             if k == '_parameters_':
                 params = [p.copy() for p in v]
-            
+
         dc['_parent_'] = None
         dc['_parent_index_'] = None
         dc['_observer_callables_'] = []
@@ -867,12 +929,12 @@ class Parameterizable(OptimizationHandlable):
 
         s = self.__new__(self.__class__)
         s.__dict__ = dc
-        
+
         for p in params:
             s.add_parameter(p, _ignore_added_names=True)
-        
+
         return s
-    
+
     #===========================================================================
     # From being parentable, we have to define the parent_change notification
     #===========================================================================

GPy/core/parameterization/parameterized.py

@@ -65,8 +65,8 @@ class Parameterized(Parameterizable, Pickleable):
     # **Never** call parameters_changed() yourself 
     __metaclass__ = ParametersChangedMeta    
     #===========================================================================
-    def __init__(self, name=None, *a, **kw):
-        super(Parameterized, self).__init__(name=name, parent=None, parent_index=None, *a, **kw)
+    def __init__(self, name=None, parameters=[], *a, **kw):
+        super(Parameterized, self).__init__(name=name, *a, **kw)
         self._in_init_ = True
         self._parameters_ = ArrayList()
         self.size = sum(p.size for p in self._parameters_)
@@ -76,6 +76,7 @@ class Parameterized(Parameterizable, Pickleable):
         self._param_slices_ = []
         self._connect_parameters()
         del self._in_init_
+        self.add_parameters(*parameters)
 
     def build_pydot(self, G=None):
         import pydot  # @UnresolvedImport
@@ -100,7 +101,6 @@ class Parameterized(Parameterizable, Pickleable):
             return G
         return node
 
-    
     def _getstate(self):
         """
         Get the current state of the class,
@@ -205,25 +205,29 @@ class Parameterized(Parameterizable, Pickleable):
         return found_params
 
     def __getitem__(self, name, paramlist=None):
-        if paramlist is None:
-            paramlist = self.grep_param_names(name)
-        if len(paramlist) < 1: raise AttributeError, name
-        if len(paramlist) == 1:
-            if isinstance(paramlist[-1], Parameterized):
-                paramlist = paramlist[-1].flattened_parameters
-                if len(paramlist) != 1:
-                    return ParamConcatenation(paramlist)
-            return paramlist[-1]
-        return ParamConcatenation(paramlist)
+        if isinstance(name, (int, slice, tuple, np.ndarray)):
+            return self._param_array_[name]
+        else:
+            if paramlist is None:
+                paramlist = self.grep_param_names(name)
+            if len(paramlist) < 1: raise AttributeError, name
+            if len(paramlist) == 1:
+                if isinstance(paramlist[-1], Parameterized):
+                    paramlist = paramlist[-1].flattened_parameters
+                    if len(paramlist) != 1:
+                        return ParamConcatenation(paramlist)
+                return paramlist[-1]
+            return ParamConcatenation(paramlist)
     
     def __setitem__(self, name, value, paramlist=None):
         if isinstance(name, (slice, tuple, np.ndarray)):
             self._param_array_[name] = value
+            self.notify_observers()
         else:
             try: param = self.__getitem__(name, paramlist)
             except AttributeError as a: raise a
             param[:] = value
-        
+
     def __setattr__(self, name, val):
         # override the default behaviour, if setting a param, so broadcasting can by used        
         if hasattr(self, '_parameters_'):

GPy/core/parameterization/variational.py

@@ -63,12 +63,13 @@ class SpikeAndSlabPrior(VariationalPrior):
 
 
 class VariationalPosterior(Parameterized):
-    def __init__(self, means=None, variances=None, name=None, **kw):
-        super(VariationalPosterior, self).__init__(name=name, **kw)
+    def __init__(self, means=None, variances=None, name=None, *a, **kw):
+        super(VariationalPosterior, self).__init__(name=name, *a, **kw)
         self.mean = Param("mean", means)
+        self.variance = Param("variance", variances, Logexp())
         self.ndim = self.mean.ndim
         self.shape = self.mean.shape
-        self.variance = Param("variance", variances, Logexp())
+        self.num_data, self.input_dim = self.mean.shape
         self.add_parameters(self.mean, self.variance)
         self.num_data, self.input_dim = self.mean.shape
         if self.has_uncertain_inputs():
@@ -77,6 +78,24 @@ class VariationalPosterior(Parameterized):
     def has_uncertain_inputs(self):
         return not self.variance is None
 
+    def __getitem__(self, s):
+        if isinstance(s, (int, slice, tuple, list, np.ndarray)):
+            import copy
+            n = self.__new__(self.__class__, self.name)
+            dc = self.__dict__.copy()
+            dc['mean'] = self.mean[s]
+            dc['variance'] = self.variance[s]
+            dc['_parameters_'] = copy.copy(self._parameters_)
+            n.__dict__.update(dc)
+            n._parameters_[dc['mean']._parent_index_] = dc['mean']
+            n._parameters_[dc['variance']._parent_index_] = dc['variance']
+            n.ndim = n.mean.ndim
+            n.shape = n.mean.shape
+            n.num_data = n.mean.shape[0]
+            n.input_dim = n.mean.shape[1] if n.ndim != 1 else 1
+            return n
+        else:
+            return super(VariationalPrior, self).__getitem__(s)
 
 class NormalPosterior(VariationalPosterior):
     '''
@@ -107,6 +126,27 @@ class SpikeAndSlabPosterior(VariationalPosterior):
         super(SpikeAndSlabPosterior, self).__init__(means, variances, name)
         self.gamma = Param("binary_prob",binary_prob, Logistic(1e-10,1.-1e-10))
         self.add_parameter(self.gamma)
+        
+    def __getitem__(self, s):
+        if isinstance(s, (int, slice, tuple, list, np.ndarray)):
+            import copy
+            n = self.__new__(self.__class__, self.name)
+            dc = self.__dict__.copy()
+            dc['mean'] = self.mean[s]
+            dc['variance'] = self.variance[s]
+            dc['binary_prob'] = self.binary_prob[s]
+            dc['_parameters_'] = copy.copy(self._parameters_)
+            n.__dict__.update(dc)
+            n._parameters_[dc['mean']._parent_index_] = dc['mean']
+            n._parameters_[dc['variance']._parent_index_] = dc['variance']
+            n._parameters_[dc['binary_prob']._parent_index_] = dc['binary_prob']
+            n.ndim = n.mean.ndim
+            n.shape = n.mean.shape
+            n.num_data = n.mean.shape[0]
+            n.input_dim = n.mean.shape[1] if n.ndim != 1 else 1
+            return n
+        else:
+            return super(VariationalPrior, self).__getitem__(s)
 
     def plot(self, *args):
         """

GPy/core/sparse_gp.py

@@ -31,7 +31,7 @@ class SparseGP(GP):
 
     """
 
-    def __init__(self, X, Y, Z, kernel, likelihood, inference_method=None, name='sparse gp'):
+    def __init__(self, X, Y, Z, kernel, likelihood, inference_method=None, name='sparse gp', Y_metadata=None):
 
         #pick a sensible inference method
         if inference_method is None:
@@ -45,7 +45,7 @@ class SparseGP(GP):
         self.Z = Param('inducing inputs', Z)
         self.num_inducing = Z.shape[0]
 
-        GP.__init__(self, X, Y, kernel, likelihood, inference_method=inference_method, name=name)
+        GP.__init__(self, X, Y, kernel, likelihood, inference_method=inference_method, name=name, Y_metadata=Y_metadata)
 
         self.add_parameter(self.Z, index=0)
 
@@ -53,19 +53,19 @@ class SparseGP(GP):
         return isinstance(self.X, VariationalPosterior)
 
     def parameters_changed(self):
-        self.posterior, self._log_marginal_likelihood, self.grad_dict = self.inference_method.inference(self.kern, self.X, self.Z, self.likelihood, self.Y)
-        self.likelihood.update_gradients(self.grad_dict.pop('partial_for_likelihood'))
+        self.posterior, self._log_marginal_likelihood, self.grad_dict = self.inference_method.inference(self.kern, self.X, self.Z, self.likelihood, self.Y, self.Y_metadata)
+        self.likelihood.update_gradients(self.grad_dict['dL_dthetaL'])
         if isinstance(self.X, VariationalPosterior):
             #gradients wrt kernel
             dL_dKmm = self.grad_dict.pop('dL_dKmm')
             self.kern.update_gradients_full(dL_dKmm, self.Z, None)
             target = self.kern.gradient.copy()
-            self.kern.update_gradients_expectations(variational_posterior=self.X, Z=self.Z, **self.grad_dict)
+            self.kern.update_gradients_expectations(variational_posterior=self.X, Z=self.Z, dL_dpsi0=self.grad_dict['dL_dpsi0'], dL_dpsi1=self.grad_dict['dL_dpsi1'], dL_dpsi2=self.grad_dict['dL_dpsi2'])
             self.kern.gradient += target
 
             #gradients wrt Z
-            self.Z.gradient = self.kern.gradients_X(dL_dKmm, self.Z)
-            self.Z.gradient += self.kern.gradients_Z_expectations(
+            self.Z.gradient[:,self.kern.active_dims] = self.kern.gradients_X(dL_dKmm, self.Z)
+            self.Z.gradient[:,self.kern.active_dims] += self.kern.gradients_Z_expectations(
                                self.grad_dict['dL_dpsi1'], self.grad_dict['dL_dpsi2'], Z=self.Z, variational_posterior=self.X)
         else:
             #gradients wrt kernel
@@ -75,10 +75,9 @@ class SparseGP(GP):
             target += self.kern.gradient
             self.kern.update_gradients_full(self.grad_dict['dL_dKmm'], self.Z, None)
             self.kern.gradient += target
-
             #gradients wrt Z
-            self.Z.gradient = self.kern.gradients_X(self.grad_dict['dL_dKmm'], self.Z)
-            self.Z.gradient += self.kern.gradients_X(self.grad_dict['dL_dKnm'].T, self.Z, self.X)
+            self.Z.gradient[:,self.kern.active_dims] = self.kern.gradients_X(self.grad_dict['dL_dKmm'], self.Z)
+            self.Z.gradient[:,self.kern.active_dims] += self.kern.gradients_X(self.grad_dict['dL_dKnm'].T, self.Z, self.X)
 
     def _raw_predict(self, Xnew, full_cov=False):
         """

GPy/examples/classification.py

@@ -89,7 +89,7 @@ def toy_linear_1d_classification_laplace(seed=default_seed, optimize=True, plot=
 
     likelihood = GPy.likelihoods.Bernoulli()
     laplace_inf = GPy.inference.latent_function_inference.Laplace()
-    kernel = GPy.kern.rbf(1)
+    kernel = GPy.kern.RBF(1)
 
     # Model definition
     m = GPy.core.GP(data['X'], Y, kernel=kernel, likelihood=likelihood, inference_method=laplace_inf)

GPy/examples/coreg_example.py

@@ -0,0 +1,80 @@
+import numpy as np
+import pylab as pb
+import GPy
+pb.ion()
+pb.close('all')
+
+X1 = np.arange(3)[:,None]
+X2 = np.arange(4)[:,None]
+I1 = np.zeros_like(X1)
+I2 = np.ones_like(X2)
+
+_X = np.vstack([ X1, X2 ])
+_I = np.vstack([ I1, I2 ])
+
+X = np.hstack([ _X, _I ])
+
+Y1 = np.sin(X1/8.)
+Y2 = np.cos(X2/8.)
+
+Bias = GPy.kern.Bias(1,active_dims=[0])
+Coreg = GPy.kern.Coregionalize(1,2,active_dims=[1])
+K = Bias.prod(Coreg,name='X')
+
+#K.coregion.W = 0
+#print K.coregion.W
+
+#print Bias.K(_X,_X)
+#print K.K(X,X)
+
+#pb.matshow(K.K(X,X))
+
+
+Mlist = [GPy.kern.Matern32(1,lengthscale=20.,name="Mat")]
+kern = GPy.util.multioutput.LCM(input_dim=1,num_outputs=2,kernels_list=Mlist,name='H')
+kern.B.W = 0
+kern.B.kappa = 1.
+#kern.B.W.fix()
+#kern.B.kappa.fix()
+#m = GPy.models.GPCoregionalizedRegression(X_list=[X1,X2], Y_list=[Y1,Y2], kernel=kern)
+m = GPy.models.SparseGPCoregionalizedRegression(X_list=[X1], Y_list=[Y1], kernel=kern)
+#m.optimize()
+m.checkgrad(verbose=1)
+
+fig = pb.figure()
+ax0 = fig.add_subplot(211)
+ax1 = fig.add_subplot(212)
+slices = GPy.util.multioutput.get_slices([Y1,Y2])
+m.plot(fixed_inputs=[(1,0)],which_data_rows=slices[0],ax=ax0)
+#m.plot(fixed_inputs=[(1,1)],which_data_rows=slices[1],ax=ax1)
+
+
+
+"""
+
+X1 = 100 * np.random.rand(100)[:,None]
+X2 = 100 * np.random.rand(100)[:,None]
+#X1.sort()
+#X2.sort()
+
+Y1 = np.sin(X1/10.) + np.random.rand(100)[:,None]
+Y2 = np.cos(X2/10.) + np.random.rand(100)[:,None]
+
+
+
+
+Mlist = [GPy.kern.Matern32(1,lengthscale=20.,name="Mat")]
+kern = GPy.util.multioutput.LCM(input_dim=1,num_outputs=12,kernels_list=Mlist,name='H')
+
+
+m = GPy.models.GPCoregionalizedRegression(X_list=[X1,X2], Y_list=[Y1,Y2], kernel=kern)
+m.optimize()
+
+fig = pb.figure()
+ax0 = fig.add_subplot(211)
+ax1 = fig.add_subplot(212)
+slices = GPy.util.multioutput.get_slices([Y1,Y2])
+m.plot(fixed_inputs=[(1,0)],which_data_rows=slices[0],ax=ax0)
+m.plot(fixed_inputs=[(1,1)],which_data_rows=slices[1],ax=ax1)
+
+"""

GPy/examples/dimensionality_reduction.py

@@ -324,14 +324,14 @@ def mrd_simulation(optimize=True, verbose=True, plot=True, plot_sim=True, **kw):
 
     D1, D2, D3, N, num_inducing, Q = 60, 20, 36, 60, 6, 5
     _, _, Ylist = _simulate_sincos(D1, D2, D3, N, num_inducing, Q, plot_sim)
-    likelihood_list = [Gaussian(x, normalize=True) for x in Ylist]
-
-    k = kern.Linear(Q, ARD=True) + kern.Bias(Q, _np.exp(-2)) + kern.White(Q, _np.exp(-2))
-    m = MRD(likelihood_list, input_dim=Q, num_inducing=num_inducing, kernels=k, initx="", initz='permute', **kw)
-    m.ensure_default_constraints()
-
-    for i, bgplvm in enumerate(m.bgplvms):
-        m['{}_noise'.format(i)] = bgplvm.likelihood.Y.var() / 500.
+    
+    #Ylist = [Ylist[0]]
+    k = [kern.Linear(Q, ARD=True) + kern.White(Q, 1e-4) for _ in range(len(Ylist))]
+    m = MRD(Ylist, input_dim=Q, num_inducing=num_inducing, kernel=k, initx="", initz='permute', **kw)
+    
+    m['.*noise'] = [Y.var()/500. for Y in Ylist]
+    #for i, Y in enumerate(Ylist):
+    #    m['.*Y_{}.*Gaussian.*noise'.format(i)] = Y.var(1) / 500.
 
     if optimize:
         print "Optimizing Model:"

GPy/examples/regression.py

@@ -318,7 +318,7 @@ def toy_ARD(max_iters=1000, kernel_type='linear', num_samples=300, D=4, optimize
     Y /= Y.std()
 
     if kernel_type == 'linear':
-        kernel = GPy.kern.linear(X.shape[1], ARD=1)
+        kernel = GPy.kern.Linear(X.shape[1], ARD=1)
     elif kernel_type == 'rbf_inv':
         kernel = GPy.kern.RBF_inv(X.shape[1], ARD=1)
     else:
@@ -357,7 +357,7 @@ def toy_ARD_sparse(max_iters=1000, kernel_type='linear', num_samples=300, D=4, o
     Y /= Y.std()
 
     if kernel_type == 'linear':
-        kernel = GPy.kern.linear(X.shape[1], ARD=1)
+        kernel = GPy.kern.Linear(X.shape[1], ARD=1)
     elif kernel_type == 'rbf_inv':
         kernel = GPy.kern.RBF_inv(X.shape[1], ARD=1)
     else:
@@ -468,7 +468,7 @@ def sparse_GP_regression_2D(num_samples=400, num_inducing=50, max_iters=100, opt
 
 def uncertain_inputs_sparse_regression(max_iters=200, optimize=True, plot=True):
     """Run a 1D example of a sparse GP regression with uncertain inputs."""
-    fig, axes = pb.subplots(1, 2, figsize=(12, 5))
+    fig, axes = pb.subplots(1, 2, figsize=(12, 5), sharex=True, sharey=True)
 
     # sample inputs and outputs
     S = np.ones((20, 1))

GPy/inference/latent_function_inference/__init__.py

@@ -16,8 +16,8 @@ If the likelihood object is something other than Gaussian, then exact inference
 is not tractable. We then resort to a Laplace approximation (laplace.py) or
 expectation propagation (ep.py).
 
-The inference methods return a 
-:class:`~GPy.inference.latent_function_inference.posterior.Posterior` 
+The inference methods return a
+:class:`~GPy.inference.latent_function_inference.posterior.Posterior`
 instance, which is a simple
 structure which contains a summary of the posterior. The model classes can then
 use this posterior object for making predictions, optimizing hyper-parameters,
@@ -27,19 +27,19 @@ etc.
 
 from exact_gaussian_inference import ExactGaussianInference
 from laplace import Laplace
-expectation_propagation = 'foo' # TODO
 from GPy.inference.latent_function_inference.var_dtc import VarDTC
+from expectation_propagation import EP
 from dtc import DTC
 from fitc import FITC
 
 # class FullLatentFunctionData(object):
-#     
-# 
+#
+#
 # class LatentFunctionInference(object):
 #     def inference(self, kern, X, likelihood, Y, Y_metadata=None):
 #         """
 #         Do inference on the latent functions given a covariance function `kern`,
-#         inputs and outputs `X` and `Y`, and a likelihood `likelihood`. 
+#         inputs and outputs `X` and `Y`, and a likelihood `likelihood`.
 #         Additional metadata for the outputs `Y` can be given in `Y_metadata`.
 #         """
-#         raise NotImplementedError, "Abstract base class for full inference"
+#         raise NotImplementedError, "Abstract base class for full inference"

GPy/inference/latent_function_inference/dtc.py

@@ -19,7 +19,7 @@ class DTC(object):
     def __init__(self):
         self.const_jitter = 1e-6
 
-    def inference(self, kern, X, X_variance, Z, likelihood, Y):
+    def inference(self, kern, X, Z, likelihood, Y):
         assert X_variance is None, "cannot use X_variance with DTC. Try varDTC."
 
         #TODO: MAX! fix this!
@@ -40,7 +40,7 @@ class DTC(object):
         U = Knm
         Uy = np.dot(U.T,Y)
 
-        #factor Kmm 
+        #factor Kmm
         Kmmi, L, Li, _ = pdinv(Kmm)
 
         # Compute A
@@ -78,11 +78,9 @@ class DTC(object):
         Uv = np.dot(U, v)
         dL_dR = 0.5*(np.sum(U*np.dot(U,P), 1) - 1./beta + np.sum(np.square(Y), 1) - 2.*np.sum(Uv*Y, 1) + np.sum(np.square(Uv), 1))*beta**2
 
-        grad_dict = {'dL_dKmm': dL_dK, 'dL_dKdiag':np.zeros_like(Knn), 'dL_dKnm':dL_dU.T}
+        dL_dthetaL = likelihood.exact_inference_gradients(dL_dR)
 
-        #update gradients
-        kern.update_gradients_sparse(X=X, Z=Z, **grad_dict)
-        likelihood.update_gradients(dL_dR)
+        grad_dict = {'dL_dKmm': dL_dK, 'dL_dKdiag':np.zeros_like(Knn), 'dL_dKnm':dL_dU.T, 'dL_dthetaL':dL_dthetaL}
 
         #construct a posterior object
         post = Posterior(woodbury_inv=Kmmi-P, woodbury_vector=v, K=Kmm, mean=None, cov=None, K_chol=L)
@@ -158,11 +156,8 @@ class vDTC(object):
         dL_dR = 0.5*(np.sum(U*np.dot(U,P), 1) - 1./beta + np.sum(np.square(Y), 1) - 2.*np.sum(Uv*Y, 1) + np.sum(np.square(Uv), 1) )*beta**2
         dL_dR -=beta*trace_term/num_data
 
-        grad_dict = {'dL_dKmm': dL_dK, 'dL_dKdiag':np.zeros_like(Knn) + -0.5*beta, 'dL_dKnm':dL_dU.T}
-
-        #update gradients
-        kern.update_gradients_sparse(X=X, Z=Z, **grad_dict)
-        likelihood.update_gradients(dL_dR)
+        dL_dthetaL = likelihood.exact_inference_gradients(dL_dR)
+        grad_dict = {'dL_dKmm': dL_dK, 'dL_dKdiag':np.zeros_like(Knn) + -0.5*beta, 'dL_dKnm':dL_dU.T, 'dL_dthetaL':dL_dthetaL}
 
         #construct a posterior object
         post = Posterior(woodbury_inv=Kmmi-P, woodbury_vector=v, K=Kmm, mean=None, cov=None, K_chol=L)

GPy/inference/latent_function_inference/exact_gaussian_inference.py

@@ -3,6 +3,7 @@
 
 from posterior import Posterior
 from ...util.linalg import pdinv, dpotrs, tdot
+from ...util import diag
 import numpy as np
 log_2_pi = np.log(2*np.pi)
 
@@ -41,7 +42,9 @@ class ExactGaussianInference(object):
 
         K = kern.K(X)
 
-        Wi, LW, LWi, W_logdet = pdinv(K + likelihood.covariance_matrix(Y, Y_metadata))
+        Ky = K.copy()
+        diag.add(Ky, likelihood.gaussian_variance(Y, Y_metadata))
+        Wi, LW, LWi, W_logdet = pdinv(Ky)
 
         alpha, _ = dpotrs(LW, YYT_factor, lower=1)
 
@@ -49,9 +52,6 @@ class ExactGaussianInference(object):
 
         dL_dK = 0.5 * (tdot(alpha) - Y.shape[1] * Wi)
 
-        #TODO: does this really live here?
-        likelihood.update_gradients(np.diag(dL_dK))
-
-        return Posterior(woodbury_chol=LW, woodbury_vector=alpha, K=K), log_marginal, {'dL_dK':dL_dK}
-
+        dL_dthetaL = likelihood.exact_inference_gradients(np.diag(dL_dK),Y_metadata)
 
+        return Posterior(woodbury_chol=LW, woodbury_vector=alpha, K=K), log_marginal, {'dL_dK':dL_dK, 'dL_dthetaL':dL_dthetaL}

GPy/inference/latent_function_inference/expectation_propagation.py

@@ -1,7 +1,7 @@
 import numpy as np
-from scipy import stats
-from ..util.linalg import pdinv,mdot,jitchol,chol_inv,DSYR,tdot,dtrtrs
-from likelihood import likelihood
+from ...util.linalg import pdinv,jitchol,DSYR,tdot,dtrtrs, dpotrs
+from posterior import Posterior
+log_2_pi = np.log(2*np.pi)
 
 class EP(object):
     def __init__(self, epsilon=1e-6, eta=1., delta=1.):
@@ -28,30 +28,30 @@ class EP(object):
 
         K = kern.K(X)
 
-        mu_tilde, tau_tilde = self.expectation_propagation()
+        mu, Sigma, mu_tilde, tau_tilde, Z_hat = self.expectation_propagation(K, Y, likelihood, Y_metadata)
 
-        Wi, LW, LWi, W_logdet = pdinv(K + np.diag(1./tau_tilde)
+        Wi, LW, LWi, W_logdet = pdinv(K + np.diag(1./tau_tilde))
 
         alpha, _ = dpotrs(LW, mu_tilde, lower=1)
 
-        log_marginal =  0.5*(-num_data * log_2_pi - W_logdet - np.sum(alpha * mu_tilde))
+        log_marginal =  0.5*(-num_data * log_2_pi - W_logdet - np.sum(alpha * mu_tilde)) # TODO: add log Z_hat??
 
         dL_dK = 0.5 * (tdot(alpha[:,None]) - Wi)
 
-        #TODO: what abot derivatives of the likelihood parameters?
+        dL_dthetaL = np.zeros(likelihood.size)#TODO: derivatives of the likelihood parameters
 
-        return Posterior(woodbury_inv=Wi, woodbury_vector=alpha, K=K), log_marginal, {'dL_dK':dL_dK}
+        return Posterior(woodbury_inv=Wi, woodbury_vector=alpha, K=K), log_marginal, {'dL_dK':dL_dK, 'dL_dthetaL':dL_dthetaL}
 
 
 
-    def expectation_propagation(self, K, Y, Y_metadata, likelihood)
+    def expectation_propagation(self, K, Y, likelihood, Y_metadata):
 
         num_data, data_dim = Y.shape
         assert data_dim == 1, "This EP methods only works for 1D outputs"
 
 
         #Initial values - Posterior distribution parameters: q(f|X,Y) = N(f|mu,Sigma)
-        mu = np.zeros(self.num_data)
+        mu = np.zeros(num_data)
         Sigma = K.copy()
 
         #Initial values - Marginal moments
@@ -61,33 +61,32 @@ class EP(object):
 
         #initial values - Gaussian factors
         if self.old_mutilde is None:
-            tau_tilde, mu_tilde, v_tilde = np.zeros((3, num_data, num_data))
+            tau_tilde, mu_tilde, v_tilde = np.zeros((3, num_data))
         else:
             assert old_mutilde.size == num_data, "data size mis-match: did you change the data? try resetting!"
             mu_tilde, v_tilde = self.old_mutilde, self.old_vtilde
             tau_tilde = v_tilde/mu_tilde
 
         #Approximation
-        epsilon_np1 = self.epsilon + 1.
-        epsilon_np2 = self.epsilon + 1.
+        tau_diff = self.epsilon + 1.
+        v_diff = self.epsilon + 1.
        	iterations = 0
-        while (epsilon_np1 > self.epsilon) or (epsilon_np2 > self.epsilon):
+        while (tau_diff > self.epsilon) or (v_diff > self.epsilon):
             update_order = np.random.permutation(num_data)
             for i in update_order:
                 #Cavity distribution parameters
                 tau_cav = 1./Sigma[i,i] - self.eta*tau_tilde[i]
                 v_cav = mu[i]/Sigma[i,i] - self.eta*v_tilde[i]
                 #Marginal moments
-                Z_hat[i], mu_hat[i], sigma2_hat[i] = likelihood.moments_match(Y[i], tau_cav, v_cav, Y_metadata=(None if Y_metadata is None else Y_metadata[i]))
+                Z_hat[i], mu_hat[i], sigma2_hat[i] = likelihood.moments_match_ep(Y[i], tau_cav, v_cav)#, Y_metadata=None)#=(None if Y_metadata is None else Y_metadata[i]))
                 #Site parameters update
                 delta_tau = self.delta/self.eta*(1./sigma2_hat[i] - 1./Sigma[i,i])
                 delta_v = self.delta/self.eta*(mu_hat[i]/sigma2_hat[i] - mu[i]/Sigma[i,i])
                 tau_tilde[i] += delta_tau
                 v_tilde[i] += delta_v
                 #Posterior distribution parameters update
-                DSYR(Sigma, Sigma[:,i].copy(), -Delta_tau/(1.+ Delta_tau*Sigma[i,i]))
+                DSYR(Sigma, Sigma[:,i].copy(), -delta_tau/(1.+ delta_tau*Sigma[i,i]))
                 mu = np.dot(Sigma, v_tilde)
-                iterations += 1
 
             #(re) compute Sigma and mu using full Cholesky decompy
             tau_tilde_root = np.sqrt(tau_tilde)
@@ -99,10 +98,14 @@ class EP(object):
             mu = np.dot(Sigma,v_tilde)
 
             #monitor convergence
-            epsilon_np1 = np.mean(np.square(tau_tilde-tau_tilde_old))
-            epsilon_np2 = np.mean(np.square(v_tilde-v_tilde_old))
+            if iterations>0:
+                tau_diff = np.mean(np.square(tau_tilde-tau_tilde_old))
+                v_diff = np.mean(np.square(v_tilde-v_tilde_old))
             tau_tilde_old = tau_tilde.copy()
             v_tilde_old = v_tilde.copy()
 
-        return mu, Sigma, mu_tilde, tau_tilde
+            iterations += 1
+
+        mu_tilde = v_tilde/tau_tilde
+        return mu, Sigma, mu_tilde, tau_tilde, Z_hat
 

GPy/inference/latent_function_inference/fitc.py

@@ -3,6 +3,7 @@
 
 from posterior import Posterior
 from ...util.linalg import jitchol, tdot, dtrtrs, dpotri, pdinv
+from ...util import diag
 import numpy as np
 log_2_pi = np.log(2*np.pi)
 
@@ -14,15 +15,9 @@ class FITC(object):
     the posterior.
 
     """
-    def __init__(self):
-        self.const_jitter = 1e-6
+    const_jitter = 1e-6
 
-    def inference(self, kern, X, X_variance, Z, likelihood, Y):
-        assert X_variance is None, "cannot use X_variance with FITC. Try varDTC."
-        
-        #TODO: MAX! fix this!
-        from ...util.misc import param_to_array
-        Y = param_to_array(Y)
+    def inference(self, kern, X, Z, likelihood, Y):
 
         num_inducing, _ = Z.shape
         num_data, output_dim = Y.shape
@@ -37,7 +32,8 @@ class FITC(object):
         Knm = kern.K(X, Z)
         U = Knm
 
-        #factor Kmm 
+        #factor Kmm
+        diag.add(Kmm, self.const_jitter)
         Kmmi, L, Li, _ = pdinv(Kmm)
 
         #compute beta_star, the effective noise precision
@@ -73,7 +69,7 @@ class FITC(object):
         vvT_P = tdot(v.reshape(-1,1)) + P
         dL_dK = 0.5*(Kmmi - vvT_P)
         KiU = np.dot(Kmmi, U.T)
-        dL_dK += np.dot(KiU*dL_dR, KiU.T) 
+        dL_dK += np.dot(KiU*dL_dR, KiU.T)
 
         # Compute dL_dU
         vY = np.dot(v.reshape(-1,1),Y.T)
@@ -81,11 +77,8 @@ class FITC(object):
         dL_dU *= beta_star
         dL_dU -= 2.*KiU*dL_dR
 
-        grad_dict = {'dL_dKmm': dL_dK, 'dL_dKdiag':dL_dR, 'dL_dKnm':dL_dU.T}
-
-        #update gradients
-        kern.update_gradients_sparse(X=X, Z=Z, **grad_dict)
-        likelihood.update_gradients(dL_dR)
+        dL_dthetaL = likelihood.exact_inference_gradients(dL_dR)
+        grad_dict = {'dL_dKmm': dL_dK, 'dL_dKdiag':dL_dR, 'dL_dKnm':dL_dU.T, 'dL_dthetaL':dL_dthetaL}
 
         #construct a posterior object
         post = Posterior(woodbury_inv=Kmmi-P, woodbury_vector=v, K=Kmm, mean=None, cov=None, K_chol=L)

GPy/inference/latent_function_inference/laplace.py

@@ -52,15 +52,13 @@ class Laplace(object):
 
         f_hat, Ki_fhat = self.rasm_mode(K, Y, likelihood, Ki_f_init, Y_metadata=Y_metadata)
         self.f_hat = f_hat
-
+        self.Ki_fhat =  Ki_fhat
+        self.K = K.copy()
         #Compute hessian and other variables at mode
         log_marginal, woodbury_inv, dL_dK, dL_dthetaL = self.mode_computations(f_hat, Ki_fhat, K, Y, likelihood, kern, Y_metadata)
 
-        kern.update_gradients_full(dL_dK, X)
-        likelihood.update_gradients(dL_dthetaL)
-
         self._previous_Ki_fhat = Ki_fhat.copy()
-        return Posterior(woodbury_vector=Ki_fhat, woodbury_inv=woodbury_inv, K=K), log_marginal, {'dL_dK':dL_dK}
+        return Posterior(woodbury_vector=Ki_fhat, woodbury_inv=woodbury_inv, K=K), log_marginal, {'dL_dK':dL_dK, 'dL_dthetaL':dL_dthetaL}
 
     def rasm_mode(self, K, Y, likelihood, Ki_f_init, Y_metadata=None):
         """

GPy/inference/latent_function_inference/var_dtc.py

@@ -2,7 +2,8 @@
 # Licensed under the BSD 3-clause license (see LICENSE.txt)
 
 from posterior import Posterior
-from ...util.linalg import jitchol, backsub_both_sides, tdot, dtrtrs, dtrtri, dpotri, dpotrs, symmetrify
+from ...util.linalg import mdot, jitchol, backsub_both_sides, tdot, dtrtrs, dtrtri, dpotri, dpotrs, symmetrify
+from ...util import diag
 from ...core.parameterization.variational import VariationalPosterior
 import numpy as np
 from ...util.misc import param_to_array
@@ -28,7 +29,7 @@ class VarDTC(object):
     def set_limit(self, limit):
         self.get_trYYT.limit = limit
         self.get_YYTfactor.limit = limit
-        
+
     def _get_trYYT(self, Y):
         return param_to_array(np.sum(np.square(Y)))
 
@@ -47,7 +48,7 @@ class VarDTC(object):
     def get_VVTfactor(self, Y, prec):
         return Y * prec # TODO chache this, and make it effective
 
-    def inference(self, kern, X, Z, likelihood, Y):
+    def inference(self, kern, X, Z, likelihood, Y, Y_metadata=None):
         if isinstance(X, VariationalPosterior):
             uncertain_inputs = True
             psi0 = kern.psi0(Z, X)
@@ -64,7 +65,7 @@ class VarDTC(object):
         _, output_dim = Y.shape
 
         #see whether we've got a different noise variance for each datum
-        beta = 1./np.fmax(likelihood.variance, 1e-6)
+        beta = 1./np.fmax(likelihood.gaussian_variance(Y, Y_metadata), 1e-6)
         # VVT_factor is a matrix such that tdot(VVT_factor) = VVT...this is for efficiency!
         #self.YYTfactor = self.get_YYTfactor(Y)
         #VVT_factor = self.get_VVTfactor(self.YYTfactor, beta)
@@ -73,13 +74,14 @@ class VarDTC(object):
         trYYT = self.get_trYYT(Y)
 
         # do the inference:
-        het_noise = beta.size < 1
+        het_noise = beta.size > 1
         num_inducing = Z.shape[0]
         num_data = Y.shape[0]
         # kernel computations, using BGPLVM notation
-        Kmm = kern.K(Z)
 
-        Lm = jitchol(Kmm+np.eye(Z.shape[0])*self.const_jitter)
+        Kmm = kern.K(Z).copy()
+        diag.add(Kmm, self.const_jitter)
+        Lm = jitchol(Kmm)
 
         # The rather complex computations of A
         if uncertain_inputs:
@@ -132,28 +134,28 @@ class VarDTC(object):
 
         # log marginal likelihood
         log_marginal = _compute_log_marginal_likelihood(likelihood, num_data, output_dim, beta, het_noise,
-            psi0, A, LB, trYYT, data_fit)
+            psi0, A, LB, trYYT, data_fit, VVT_factor)
 
         #put the gradients in the right places
-        partial_for_likelihood = _compute_partial_for_likelihood(likelihood,
+        dL_dR = _compute_dL_dR(likelihood,
             het_noise, uncertain_inputs, LB,
             _LBi_Lmi_psi1Vf, DBi_plus_BiPBi, Lm, A,
             psi0, psi1, beta,
-            data_fit, num_data, output_dim, trYYT)
+            data_fit, num_data, output_dim, trYYT, Y)
 
-        #likelihood.update_gradients(partial_for_likelihood)
+        dL_dthetaL = likelihood.exact_inference_gradients(dL_dR,Y_metadata)
 
         if uncertain_inputs:
             grad_dict = {'dL_dKmm': dL_dKmm,
                          'dL_dpsi0':dL_dpsi0,
                          'dL_dpsi1':dL_dpsi1,
                          'dL_dpsi2':dL_dpsi2,
-                         'partial_for_likelihood':partial_for_likelihood}
+                         'dL_dthetaL':dL_dthetaL}
         else:
             grad_dict = {'dL_dKmm': dL_dKmm,
                          'dL_dKdiag':dL_dpsi0,
                          'dL_dKnm':dL_dpsi1,
-                         'partial_for_likelihood':partial_for_likelihood}
+                         'dL_dthetaL':dL_dthetaL}
 
         #get sufficient things for posterior prediction
         #TODO: do we really want to do this in  the loop?
@@ -168,7 +170,6 @@ class VarDTC(object):
         Bi, _ = dpotri(LB, lower=1)
         symmetrify(Bi)
         Bi = -dpotri(LB, lower=1)[0]
-        from ...util import diag
         diag.add(Bi, 1)
 
         woodbury_inv = backsub_both_sides(Lm, Bi)
@@ -207,7 +208,7 @@ class VarDTCMissingData(object):
             self._subarray_indices = [[slice(None),slice(None)]]
             return [Y], [(Y**2).sum()]
 
-    def inference(self, kern, X, Z, likelihood, Y):
+    def inference(self, kern, X, Z, likelihood, Y, Y_metadata=None):
         if isinstance(X, VariationalPosterior):
             uncertain_inputs = True
             psi0_all = kern.psi0(Z, X)
@@ -220,7 +221,7 @@ class VarDTCMissingData(object):
             psi2_all = None
 
         Ys, traces = self._Y(Y)
-        beta_all = 1./np.fmax(likelihood.variance, 1e-6)
+        beta_all = 1./np.fmax(likelihood.gaussian_variance(Y_metadata), 1e-6)
         het_noise = beta_all.size != 1
 
         import itertools
@@ -231,13 +232,14 @@ class VarDTCMissingData(object):
         if uncertain_inputs:
             dL_dpsi2_all = np.zeros((Y.shape[0], num_inducing, num_inducing))
 
-        partial_for_likelihood = 0
+        dL_dR = 0
         woodbury_vector = np.zeros((num_inducing, Y.shape[1]))
         woodbury_inv_all = np.zeros((num_inducing, num_inducing, Y.shape[1]))
         dL_dKmm = 0
         log_marginal = 0
 
-        Kmm = kern.K(Z)
+        Kmm = kern.K(Z).copy()
+        diag.add(Kmm, self.const_jitter)
         #factor Kmm
         Lm = jitchol(Kmm)
         if uncertain_inputs: LmInv = dtrtri(Lm)
@@ -303,10 +305,10 @@ class VarDTCMissingData(object):
 
             # log marginal likelihood
             log_marginal += _compute_log_marginal_likelihood(likelihood, num_data, output_dim, beta, het_noise,
-                psi0, A, LB, trYYT, data_fit)
+                psi0, A, LB, trYYT, data_fit,VVT_factor)
 
             #put the gradients in the right places
-            partial_for_likelihood += _compute_partial_for_likelihood(likelihood,
+            dL_dR += _compute_dL_dR(likelihood,
                 het_noise, uncertain_inputs, LB,
                 _LBi_Lmi_psi1Vf, DBi_plus_BiPBi, Lm, A,
                 psi0, psi1, beta,
@@ -323,22 +325,23 @@ class VarDTCMissingData(object):
             Bi, _ = dpotri(LB, lower=1)
             symmetrify(Bi)
             Bi = -dpotri(LB, lower=1)[0]
-            from ...util import diag
             diag.add(Bi, 1)
             woodbury_inv_all[:, :, ind] = backsub_both_sides(Lm, Bi)[:,:,None]
 
+        dL_dthetaL = likelihood.exact_inference_gradients(dL_dR)
+
         # gradients:
         if uncertain_inputs:
             grad_dict = {'dL_dKmm': dL_dKmm,
                          'dL_dpsi0':dL_dpsi0_all,
                          'dL_dpsi1':dL_dpsi1_all,
                          'dL_dpsi2':dL_dpsi2_all,
-                         'partial_for_likelihood':partial_for_likelihood}
+                         'dL_dthetaL':dL_dthetaL}
         else:
             grad_dict = {'dL_dKmm': dL_dKmm,
                          'dL_dKdiag':dL_dpsi0_all,
                          'dL_dKnm':dL_dpsi1_all,
-                         'partial_for_likelihood':partial_for_likelihood}
+                         'dL_dthetaL':dL_dthetaL}
 
         #get sufficient things for posterior prediction
         #TODO: do we really want to do this in  the loop?
@@ -384,40 +387,41 @@ def _compute_dL_dpsi(num_inducing, num_data, output_dim, beta, Lm, VVT_factor, C
     return dL_dpsi0, dL_dpsi1, dL_dpsi2
 
 
-def _compute_partial_for_likelihood(likelihood, het_noise, uncertain_inputs, LB, _LBi_Lmi_psi1Vf, DBi_plus_BiPBi, Lm, A, psi0, psi1, beta, data_fit, num_data, output_dim, trYYT):
+def _compute_dL_dR(likelihood, het_noise, uncertain_inputs, LB, _LBi_Lmi_psi1Vf, DBi_plus_BiPBi, Lm, A, psi0, psi1, beta, data_fit, num_data, output_dim, trYYT, Y):
     # the partial derivative vector for the likelihood
     if likelihood.size == 0:
         # save computation here.
-        partial_for_likelihood = None
+        dL_dR = None
     elif het_noise:
         if uncertain_inputs:
             raise NotImplementedError, "heteroscedatic derivates with uncertain inputs not implemented"
         else:
-            from ...util.linalg import chol_inv
-            LBi = chol_inv(LB)
+            #from ...util.linalg import chol_inv
+            #LBi = chol_inv(LB)
+            LBi, _ = dtrtrs(LB,np.eye(LB.shape[0]))
+
             Lmi_psi1, nil = dtrtrs(Lm, psi1.T, lower=1, trans=0)
             _LBi_Lmi_psi1, _ = dtrtrs(LB, Lmi_psi1, lower=1, trans=0)
 
-            partial_for_likelihood = -0.5 * beta + 0.5 * likelihood.V**2
-            partial_for_likelihood += 0.5 * output_dim * (psi0 - np.sum(Lmi_psi1**2,0))[:,None] * beta**2
+            dL_dR = -0.5 * beta + 0.5 * (beta*Y)**2
+            dL_dR += 0.5 * output_dim * (psi0 - np.sum(Lmi_psi1**2,0))[:,None] * beta**2
 
-            partial_for_likelihood += 0.5*np.sum(mdot(LBi.T,LBi,Lmi_psi1)*Lmi_psi1,0)[:,None]*beta**2
-
-            partial_for_likelihood += -np.dot(_LBi_Lmi_psi1Vf.T,_LBi_Lmi_psi1).T * likelihood.Y * beta**2
-            partial_for_likelihood += 0.5*np.dot(_LBi_Lmi_psi1Vf.T,_LBi_Lmi_psi1).T**2 * beta**2
+            dL_dR += 0.5*np.sum(mdot(LBi.T,LBi,Lmi_psi1)*Lmi_psi1,0)[:,None]*beta**2
 
+            dL_dR += -np.dot(_LBi_Lmi_psi1Vf.T,_LBi_Lmi_psi1).T * Y * beta**2
+            dL_dR += 0.5*np.dot(_LBi_Lmi_psi1Vf.T,_LBi_Lmi_psi1).T**2 * beta**2
     else:
         # likelihood is not heteroscedatic
-        partial_for_likelihood = -0.5 * num_data * output_dim * beta + 0.5 * trYYT * beta ** 2
-        partial_for_likelihood += 0.5 * output_dim * (psi0.sum() * beta ** 2 - np.trace(A) * beta)
-        partial_for_likelihood += beta * (0.5 * np.sum(A * DBi_plus_BiPBi) - data_fit)
-    return partial_for_likelihood
+        dL_dR = -0.5 * num_data * output_dim * beta + 0.5 * trYYT * beta ** 2
+        dL_dR += 0.5 * output_dim * (psi0.sum() * beta ** 2 - np.trace(A) * beta)
+        dL_dR += beta * (0.5 * np.sum(A * DBi_plus_BiPBi) - data_fit)
+    return dL_dR
 
-def _compute_log_marginal_likelihood(likelihood, num_data, output_dim, beta, het_noise, psi0, A, LB, trYYT, data_fit):
-#compute log marginal likelihood
+def _compute_log_marginal_likelihood(likelihood, num_data, output_dim, beta, het_noise, psi0, A, LB, trYYT, data_fit,Y):
+    #compute log marginal likelihood
     if het_noise:
-        lik_1 = -0.5 * num_data * output_dim * np.log(2. * np.pi) + 0.5 * np.sum(np.log(beta)) - 0.5 * np.sum(likelihood.V * likelihood.Y)
-        lik_2 = -0.5 * output_dim * (np.sum(beta * psi0) - np.trace(A))
+        lik_1 = -0.5 * num_data * output_dim * np.log(2. * np.pi) + 0.5 * np.sum(np.log(beta)) - 0.5 * np.sum(beta * np.square(Y).sum(axis=-1))
+        lik_2 = -0.5 * output_dim * (np.sum(beta.flatten() * psi0) - np.trace(A))
     else:
         lik_1 = -0.5 * num_data * output_dim * (np.log(2. * np.pi) - np.log(beta)) - 0.5 * beta * trYYT
         lik_2 = -0.5 * output_dim * (np.sum(beta * psi0) - np.trace(A))

GPy/kern/_src/add.py

@@ -1,49 +1,51 @@
 # Copyright (c) 2012, GPy authors (see AUTHORS.txt).
 # Licensed under the BSD 3-clause license (see LICENSE.txt)
 
-import sys
 import numpy as np
 import itertools
-from linear import Linear
-from ...core.parameterization import Parameterized
-from ...core.parameterization.param import Param
-from kern import Kern
+from ...util.caching import Cache_this
+from kern import CombinationKernel
 
-class Add(Kern):
-    def __init__(self, subkerns, tensor):
-        assert all([isinstance(k, Kern) for k in subkerns])
-        if tensor:
-            input_dim  = sum([k.input_dim for k in subkerns])
-            self.input_slices = []
-            n = 0
-            for k in subkerns:
-                self.input_slices.append(slice(n, n+k.input_dim))
-                n += k.input_dim
-        else:
-            assert all([k.input_dim == subkerns[0].input_dim for k in subkerns])
-            input_dim = subkerns[0].input_dim
-            self.input_slices = [slice(None) for k in subkerns]
-        super(Add, self).__init__(input_dim, 'add')
-        self.add_parameters(*subkerns)
+class Add(CombinationKernel):
+    """
+    Add given list of kernels together.
+    propagates gradients through.
+    
+    This kernel will take over the active dims of it's subkernels passed in.
+    """
+    def __init__(self, subkerns, name='add'):
+        super(Add, self).__init__(subkerns, name)
 
-
-    def K(self, X, X2=None):
+    @Cache_this(limit=2, force_kwargs=['which_parts'])
+    def K(self, X, X2=None, which_parts=None):
         """
-        Compute the kernel function.
-
-        :param X: the first set of inputs to the kernel
-        :param X2: (optional) the second set of arguments to the kernel. If X2
-                   is None, this is passed throgh to the 'part' object, which
-                   handLes this as X2 == X.
+        Add all kernels together.
+        If a list of parts (of this kernel!) `which_parts` is given, only
+        the parts of the list are taken to compute the covariance.
         """
         assert X.shape[1] == self.input_dim
-        if X2 is None:
-            return sum([p.K(X[:, i_s], None) for p, i_s in zip(self._parameters_, self.input_slices)])
-        else:
-            return sum([p.K(X[:, i_s], X2[:, i_s]) for p, i_s in zip(self._parameters_, self.input_slices)])
+        if which_parts is None:
+            which_parts = self.parts
+        elif not isinstance(which_parts, (list, tuple)):
+            # if only one part is given
+            which_parts = [which_parts]
+        return reduce(np.add, (p.K(X, X2) for p in which_parts))
 
-    def update_gradients_full(self, dL_dK, X):
-        [p.update_gradients_full(dL_dK, X[:,i_s]) for p, i_s in zip(self._parameters_, self.input_slices)]
+    @Cache_this(limit=2, force_kwargs=['which_parts'])
+    def Kdiag(self, X, which_parts=None):
+        assert X.shape[1] == self.input_dim
+        if which_parts is None:
+            which_parts = self.parts
+        elif not isinstance(which_parts, (list, tuple)):
+            # if only one part is given
+            which_parts = [which_parts]
+        return reduce(np.add, (p.Kdiag(X) for p in which_parts))
+
+    def update_gradients_full(self, dL_dK, X, X2=None):
+        [p.update_gradients_full(dL_dK, X, X2) for p in self.parts]
+
+    def update_gradients_diag(self, dL_dK, X):
+        [p.update_gradients_diag(dL_dK, X) for p in self.parts]
 
     def gradients_X(self, dL_dK, X, X2=None):
         """Compute the gradient of the objective function with respect to X.
@@ -55,92 +57,77 @@ class Add(Kern):
         :param X2: Observed data inputs (optional, defaults to X)
         :type X2: np.ndarray (num_inducing x input_dim)"""
 
-        target = np.zeros_like(X)
-        if X2 is None:
-            [np.add(target[:,i_s], p.gradients_X(dL_dK, X[:, i_s], None), target[:, i_s]) for p, i_s in zip(self._parameters_, self.input_slices)]
-        else:
-            [np.add(target[:,i_s], p.gradients_X(dL_dK, X[:, i_s], X2[:,i_s]), target[:, i_s]) for p, i_s in zip(self._parameters_, self.input_slices)]
+        target = np.zeros(X.shape)
+        [target.__iadd__(p.gradients_X(dL_dK, X, X2)) for p in self.parts]
         return target
 
-    def Kdiag(self, X):
-        assert X.shape[1] == self.input_dim
-        return sum([p.Kdiag(X[:, i_s]) for p, i_s in zip(self._parameters_, self.input_slices)])
-
+    def gradients_X_diag(self, dL_dKdiag, X):
+        target = np.zeros(X.shape)
+        [target.__iadd__(p.gradients_X_diag(dL_dKdiag, X)) for p in self.parts]
+        return target
 
     def psi0(self, Z, variational_posterior):
-        return np.sum([p.psi0(Z[:, i_s], mu[:, i_s], S[:, i_s]) for p, i_s in zip(self._parameters_, self.input_slices)],0)
+        return reduce(np.add, (p.psi0(Z, variational_posterior) for p in self.parts))
 
     def psi1(self, Z, variational_posterior):
-        return np.sum([p.psi1(Z[:, i_s], mu[:, i_s], S[:, i_s]) for p, i_s in zip(self._parameters_, self.input_slices)], 0)
+        return reduce(np.add, (p.psi1(Z, variational_posterior) for p in self.parts))
 
     def psi2(self, Z, variational_posterior):
-        psi2 = np.sum([p.psi2(Z[:, i_s], mu[:, i_s], S[:, i_s]) for p, i_s in zip(self._parameters_, self.input_slices)], 0)
-
+        psi2 = reduce(np.add, (p.psi2(Z, variational_posterior) for p in self.parts))
+        #return psi2
         # compute the "cross" terms
-        from white import White
+        from static import White, Bias
         from rbf import RBF
         #from rbf_inv import RBFInv
-        from bias import Bias
         from linear import Linear
         #ffrom fixed import Fixed
 
-        for (p1, i1), (p2, i2) in itertools.combinations(itertools.izip(self._parameters_, self.input_slices), 2):
+        for p1, p2 in itertools.combinations(self.parts, 2):
+            # i1, i2 = p1.active_dims, p2.active_dims
             # white doesn;t combine with anything
             if isinstance(p1, White) or isinstance(p2, White):
                 pass
             # rbf X bias
             #elif isinstance(p1, (Bias, Fixed)) and isinstance(p2, (RBF, RBFInv)):
             elif isinstance(p1,  Bias) and isinstance(p2, (RBF, Linear)):
-                tmp = p2.psi1(Z[:,i2], mu[:,i2], S[:,i2])
+                tmp = p2.psi1(Z, variational_posterior)
                 psi2 += p1.variance * (tmp[:, :, None] + tmp[:, None, :])
             #elif isinstance(p2, (Bias, Fixed)) and isinstance(p1, (RBF, RBFInv)):
             elif isinstance(p2, Bias) and isinstance(p1, (RBF, Linear)):
-                tmp = p1.psi1(Z[:,i1], mu[:,i1], S[:,i1])
+                tmp = p1.psi1(Z, variational_posterior)
                 psi2 += p2.variance * (tmp[:, :, None] + tmp[:, None, :])
+            elif isinstance(p2, (RBF, Linear)) and isinstance(p1, (RBF, Linear)):
+                assert np.intersect1d(p1.active_dims, p2.active_dims).size == 0, "only non overlapping kernel dimensions allowed so far"
+                tmp1 = p1.psi1(Z, variational_posterior)
+                tmp2 = p2.psi1(Z, variational_posterior)
+                psi2 += (tmp1[:, :, None] * tmp2[:, None, :]) + (tmp2[:, :, None] * tmp1[:, None, :])
             else:
                 raise NotImplementedError, "psi2 cannot be computed for this kernel"
         return psi2
 
     def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
-        from white import White
-        from rbf import RBF
-        #from rbf_inv import RBFInv
-        #from bias import Bias
-        from linear import Linear
-        #ffrom fixed import Fixed
-
-        for p1, is1 in zip(self._parameters_, self.input_slices):
-
+        from static import White, Bias
+        for p1 in self.parts:
             #compute the effective dL_dpsi1. Extra terms appear becaue of the cross terms in psi2!
             eff_dL_dpsi1 = dL_dpsi1.copy()
-            for p2, is2 in zip(self._parameters_, self.input_slices):
+            for p2 in self.parts:
                 if p2 is p1:
                     continue
                 if isinstance(p2, White):
                     continue
                 elif isinstance(p2, Bias):
                     eff_dL_dpsi1 += dL_dpsi2.sum(1) * p2.variance * 2.
-                else:
-                    eff_dL_dpsi1 += dL_dpsi2.sum(1) * p2.psi1(Z[:,is2], mu[:,is2], S[:,is2]) * 2.
-
-
-            p1.update_gradients_expectations(dL_dpsi0, eff_dL_dpsi1, dL_dpsi2, mu[:,is1], S[:,is1], Z[:,is1])
-
+                else:# np.setdiff1d(p1.active_dims, ar2, assume_unique): # TODO: Careful, not correct for overlapping active_dims
+                    eff_dL_dpsi1 += dL_dpsi2.sum(1) * p2.psi1(Z, variational_posterior) * 2.
+            p1.update_gradients_expectations(dL_dpsi0, eff_dL_dpsi1, dL_dpsi2, Z, variational_posterior)
 
     def gradients_Z_expectations(self, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
-        from white import White
-        from rbf import RBF
-        #from rbf_inv import rbfinv
-        from bias import Bias
-        from linear import Linear
-        #ffrom fixed import fixed
-
+        from static import White, Bias
         target = np.zeros(Z.shape)
-        for p1, is1 in zip(self._parameters_, self.input_slices):
-
+        for p1 in self.parts:
             #compute the effective dL_dpsi1. extra terms appear becaue of the cross terms in psi2!
             eff_dL_dpsi1 = dL_dpsi1.copy()
-            for p2, is2 in zip(self._parameters_, self.input_slices):
+            for p2 in self.parts:
                 if p2 is p1:
                     continue
                 if isinstance(p2, White):
@@ -148,63 +135,39 @@ class Add(Kern):
                 elif isinstance(p2, Bias):
                     eff_dL_dpsi1 += dL_dpsi2.sum(1) * p2.variance * 2.
                 else:
-                    eff_dL_dpsi1 += dL_dpsi2.sum(1) * p2.psi1(Z[:,is2], mu[:,is2], S[:,is2]) * 2.
-
-
-            target += p1.gradients_z_variational(dL_dpsi0, eff_dL_dpsi1, dL_dpsi2, mu[:,is1], S[:,is1], Z[:,is1])
+                    eff_dL_dpsi1 += dL_dpsi2.sum(1) * p2.psi1(Z, variational_posterior) * 2.
+            target += p1.gradients_Z_expectations(eff_dL_dpsi1, dL_dpsi2, Z, variational_posterior)
         return target
 
     def gradients_qX_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
-        from white import white
-        from rbf import rbf
-        #from rbf_inv import rbfinv
-        #from bias import bias
-        from linear import linear
-        #ffrom fixed import fixed
-
-        target_mu = np.zeros(mu.shape)
-        target_S = np.zeros(S.shape)
-        for p1, is1 in zip(self._parameters_, self.input_slices):
-
+        from static import White, Bias
+        target_mu = np.zeros(variational_posterior.shape)
+        target_S = np.zeros(variational_posterior.shape)
+        for p1 in self._parameters_:
             #compute the effective dL_dpsi1. extra terms appear becaue of the cross terms in psi2!
             eff_dL_dpsi1 = dL_dpsi1.copy()
-            for p2, is2 in zip(self._parameters_, self.input_slices):
+            for p2 in self._parameters_:
                 if p2 is p1:
                     continue
-                if isinstance(p2, white):
+                if isinstance(p2, White):
                     continue
-                elif isinstance(p2, bias):
+                elif isinstance(p2, Bias):
                     eff_dL_dpsi1 += dL_dpsi2.sum(1) * p2.variance * 2.
                 else:
-                    eff_dL_dpsi1 += dL_dpsi2.sum(1) * p2.psi1(z[:,is2], mu[:,is2], s[:,is2]) * 2.
-
-
-            a, b = p1.gradients_qX_expectations(dL_dpsi0, eff_dL_dpsi1, dL_dpsi2, mu[:,is1], s[:,is1], z[:,is1])
+                    eff_dL_dpsi1 += dL_dpsi2.sum(1) * p2.psi1(Z, variational_posterior) * 2.
+            a, b = p1.gradients_qX_expectations(dL_dpsi0, eff_dL_dpsi1, dL_dpsi2, Z, variational_posterior)
             target_mu += a
             target_S += b
         return target_mu, target_S
 
-    def input_sensitivity(self):
-        in_sen = np.zeros((self.num_params, self.input_dim))
-        for i, [p, i_s] in enumerate(zip(self._parameters_, self.input_slices)):
-            in_sen[i, i_s] = p.input_sensitivity()
-        return in_sen
-
     def _getstate(self):
         """
         Get the current state of the class,
         here just all the indices, rest can get recomputed
         """
-        return Parameterized._getstate(self) + [#self._parameters_,
-                self.input_dim,
-                self.input_slices,
-                self._param_slices_
-                ]
+        return super(Add, self)._getstate()
 
     def _setstate(self, state):
-        self._param_slices_ = state.pop()
-        self.input_slices = state.pop()
-        self.input_dim = state.pop()
-        Parameterized._setstate(self, state)
+        super(Add, self)._setstate(state)
 
 

GPy/kern/_src/brownian.py

@@ -17,9 +17,9 @@ class Brownian(Kern):
     :param variance:
     :type variance: float
     """
-    def __init__(self, input_dim=1, variance=1., name='Brownian'):
+    def __init__(self, input_dim=1, variance=1., active_dims=None, name='Brownian'):
         assert input_dim==1, "Brownian motion in 1D only"
-        super(Brownian, self).__init__(input_dim, name)
+        super(Brownian, self).__init__(input_dim, active_dims, name)
 
         self.variance = Param('variance', variance, Logexp())
         self.add_parameters(self.variance)

GPy/kern/_src/coregionalize.py

@@ -34,8 +34,8 @@ class Coregionalize(Kern):
 
     .. note: see coregionalization examples in GPy.examples.regression for some usage.
     """
-    def __init__(self, output_dim, rank=1, W=None, kappa=None, name='coregion'):
-        super(Coregionalize, self).__init__(input_dim=1, name=name)
+    def __init__(self, input_dim, output_dim, rank=1, W=None, kappa=None, active_dims=None, name='coregion'):
+        super(Coregionalize, self).__init__(input_dim, active_dims, name=name)
         self.output_dim = output_dim
         self.rank = rank
         if self.rank>output_dim:

GPy/kern/_src/independent_outputs.py

@@ -2,8 +2,9 @@
 # Licensed under the BSD 3-clause license (see LICENSE.txt)
 
 
-from kern import Kern
+from kern import Kern, CombinationKernel
 import numpy as np
+import itertools
 
 def index_to_slices(index):
     """
@@ -31,78 +32,109 @@ def index_to_slices(index):
     [ret[ind_i].append(slice(*indexes_i)) for ind_i,indexes_i in zip(ind[switchpoints[:-1]],zip(switchpoints,switchpoints[1:]))]
     return ret
 
-class IndependentOutputs(Kern):
+class IndependentOutputs(CombinationKernel):
     """
-    A kernel which can reopresent several independent functions.
+    A kernel which can represent several independent functions.
     this kernel 'switches off' parts of the matrix where the output indexes are different.
 
     The index of the functions is given by the last column in the input X
     the rest of the columns of X are passed to the underlying kernel for computation (in blocks).
-
+    
+    :param kernels: either a kernel, or list of kernels to work with. If it is a list of kernels 
+    the indices in the index_dim, index the kernels you gave!
     """
-    def __init__(self, kern, name='independ'):
-        super(IndependentOutputs, self).__init__(kern.input_dim+1, name)
-        self.kern = kern
-        self.add_parameters(self.kern)
+    def __init__(self, kernels, index_dim=-1, name='independ'):
+        assert isinstance(index_dim, int), "IndependentOutputs kernel is only defined with one input dimension being the indeces"
+        if not isinstance(kernels, list): 
+            self.single_kern = True
+            self.kern = kernels
+            kernels = [kernels]
+        else:
+            self.single_kern = False
+            self.kern = kernels
+        super(IndependentOutputs, self).__init__(kernels=kernels, extra_dims=[index_dim], name=name)
+        self.index_dim = index_dim
+        self.kerns = kernels if len(kernels) != 1 else itertools.repeat(kernels[0])
 
     def K(self,X ,X2=None):
-        X, slices = X[:,:-1], index_to_slices(X[:,-1])
+        slices = index_to_slices(X[:,self.index_dim])
         if X2 is None:
             target = np.zeros((X.shape[0], X.shape[0]))
-            [[np.copyto(target[s,s], self.kern.K(X[s], None)) for s in slices_i] for slices_i in slices]
+            [[target.__setitem__((s,ss), kern.K(X[s,:], X[ss,:])) for s,ss in itertools.product(slices_i, slices_i)] for kern, slices_i in zip(self.kerns, slices)]
         else:
-            X2, slices2 = X2[:,:-1],index_to_slices(X2[:,-1])
+            slices2 = index_to_slices(X2[:,self.index_dim])
             target = np.zeros((X.shape[0], X2.shape[0]))
-            [[[np.copyto(target[s, s2], self.kern.K(X[s],X2[s2])) for s in slices_i] for s2 in slices_j] for slices_i,slices_j in zip(slices,slices2)]
+            [[target.__setitem__((s,s2), kern.K(X[s,:],X2[s2,:])) for s,s2 in itertools.product(slices_i, slices_j)] for kern, slices_i,slices_j in zip(self.kerns, slices,slices2)]
         return target
 
     def Kdiag(self,X):
-        X, slices = X[:,:-1], index_to_slices(X[:,-1])
+        slices = index_to_slices(X[:,self.index_dim])
         target = np.zeros(X.shape[0])
-        [[np.copyto(target[s], self.kern.Kdiag(X[s])) for s in slices_i] for slices_i in slices]
+        [[np.copyto(target[s], kern.Kdiag(X[s])) for s in slices_i] for kern, slices_i in zip(self.kerns, slices)]
         return target
 
     def update_gradients_full(self,dL_dK,X,X2=None):
-        target = np.zeros(self.kern.size)
-        def collate_grads(dL, X, X2):
-            self.kern.update_gradients_full(dL,X,X2)
-            self.kern._collect_gradient(target)
-
-        X,slices = X[:,:-1],index_to_slices(X[:,-1])
+        slices = index_to_slices(X[:,self.index_dim])
+        if self.single_kern: target = np.zeros(self.kern.size)
+        else: target = [np.zeros(kern.size) for kern, _ in zip(self.kerns, slices)]
+        def collate_grads(kern, i, dL, X, X2):
+            kern.update_gradients_full(dL,X,X2)
+            if self.single_kern: target[:] += kern.gradient
+            else: target[i][:] += kern.gradient
         if X2 is None:
-            [[collate_grads(dL_dK[s,s], X[s], None) for s in slices_i] for slices_i in slices]
+            [[collate_grads(kern, i, dL_dK[s,ss], X[s], X[ss]) for s,ss in itertools.product(slices_i, slices_i)] for i,(kern,slices_i) in enumerate(zip(self.kerns,slices))]
         else:
-            X2, slices2 = X2[:,:-1], index_to_slices(X2[:,-1])
-            [[[collate_grads(dL_dK[s,s2],X[s],X2[s2]) for s in slices_i] for s2 in slices_j] for slices_i,slices_j in zip(slices,slices2)]
-
-        self.kern._set_gradient(target)
+            slices2 = index_to_slices(X2[:,self.index_dim])
+            [[[collate_grads(kern, i, dL_dK[s,s2],X[s],X2[s2]) for s in slices_i] for s2 in slices_j] for i,(kern,slices_i,slices_j) in enumerate(zip(self.kerns,slices,slices2))]
+        if self.single_kern: kern.gradient = target
+        else:[kern.gradient.__setitem__(Ellipsis, target[i]) for i, [kern, _] in enumerate(zip(self.kerns, slices))]
 
     def gradients_X(self,dL_dK, X, X2=None):
-        target = np.zeros_like(X)
-        X, slices = X[:,:-1],index_to_slices(X[:,-1])
+        target = np.zeros(X.shape)
         if X2 is None:
-            [[np.copyto(target[s,:-1], self.kern.gradients_X(dL_dK[s,s],X[s],None)) for s in slices_i] for slices_i in slices]
+            # TODO: make use of index_to_slices
+            values = np.unique(X[:,self.index_dim])
+            slices = [X[:,self.index_dim]==i for i in values]
+            [target.__setitem__(s, kern.gradients_X(dL_dK[s,s],X[s],None))
+              for kern, s in zip(self.kerns, slices)]
+            #slices = index_to_slices(X[:,self.index_dim])
+            #[[np.add(target[s], kern.gradients_X(dL_dK[s,s], X[s]), out=target[s]) 
+            #  for s in slices_i] for kern, slices_i in zip(self.kerns, slices)]
+            #import ipdb;ipdb.set_trace()
+            #[[(np.add(target[s ], kern.gradients_X(dL_dK[s ,ss],X[s ], X[ss]), out=target[s ]),
+            #   np.add(target[ss], kern.gradients_X(dL_dK[ss,s ],X[ss], X[s ]), out=target[ss]))
+            #  for s, ss in itertools.combinations(slices_i, 2)] for kern, slices_i in zip(self.kerns, slices)]
         else:
-            X2,slices2 = X2[:,:-1],index_to_slices(X2[:,-1])
-            [[[np.copyto(target[s,:-1], self.kern.gradients_X(dL_dK[s,s2], X[s], X2[s2])) for s in slices_i] for s2 in slices_j] for slices_i,slices_j in zip(slices,slices2)]
+            values = np.unique(X[:,self.index_dim])
+            slices = [X[:,self.index_dim]==i for i in values]
+            slices2 = [X2[:,self.index_dim]==i for i in values]
+            [target.__setitem__(s, kern.gradients_X(dL_dK[s, :][:, s2],X[s],X2[s2]))
+              for kern, s, s2 in zip(self.kerns, slices, slices2)]
+            # TODO: make work with index_to_slices
+            #slices = index_to_slices(X[:,self.index_dim])
+            #slices2 = index_to_slices(X2[:,self.index_dim])
+            #[[target.__setitem__(s, target[s] + kern.gradients_X(dL_dK[s,s2], X[s], X2[s2])) for s, s2 in itertools.product(slices_i, slices_j)] for kern, slices_i,slices_j in zip(self.kerns, slices,slices2)]
         return target
 
     def gradients_X_diag(self, dL_dKdiag, X):
-        X, slices = X[:,:-1], index_to_slices(X[:,-1])
+        slices = index_to_slices(X[:,self.index_dim])
         target = np.zeros(X.shape)
-        [[np.copyto(target[s,:-1], self.kern.gradients_X_diag(dL_dKdiag[s],X[s])) for s in slices_i] for slices_i in slices]
+        [[target.__setitem__(s, kern.gradients_X_diag(dL_dKdiag[s],X[s])) for s in slices_i] for kern, slices_i in zip(self.kerns, slices)]
         return target
 
-    def update_gradients_diag(self,dL_dKdiag,X,target):
-        target = np.zeros(self.kern.size)
-        def collate_grads(dL, X):
-            self.kern.update_gradients_diag(dL,X)
-            self.kern._collect_gradient(target)
-        X,slices = X[:,:-1],index_to_slices(X[:,-1])
-        [[collate_grads(dL_dKdiag[s], X[s,:]) for s in slices_i] for slices_i in slices]
-        self.kern._set_gradient(target)
+    def update_gradients_diag(self, dL_dKdiag, X):
+        slices = index_to_slices(X[:,self.index_dim])
+        if self.single_kern: target = np.zeros(self.kern.size)
+        else: target = [np.zeros(kern.size) for kern, _ in zip(self.kerns, slices)]
+        def collate_grads(kern, i, dL, X):
+            kern.update_gradients_diag(dL,X)
+            if self.single_kern: target[:] += kern.gradient
+            else: target[i][:] += kern.gradient
+        [[collate_grads(kern, i, dL_dKdiag[s], X[s,:]) for s in slices_i] for i, (kern, slices_i) in enumerate(zip(self.kerns, slices))]
+        if self.single_kern: kern.gradient = target
+        else:[kern.gradient.__setitem__(Ellipsis, target[i]) for i, [kern, _] in enumerate(zip(self.kerns, slices))]
 
-class Hierarchical(Kern):
+class Hierarchical(CombinationKernel):
     """
     A kernel which can reopresent a simple hierarchical model.
 
@@ -113,7 +145,7 @@ class Hierarchical(Kern):
     The index of the functions is given by additional columns in the input X.
 
     """
-    def __init__(self, kerns, name='hierarchy'):
+    def __init__(self, kern, name='hierarchy'):
         assert all([k.input_dim==kerns[0].input_dim for k in kerns])
         super(Hierarchical, self).__init__(kerns[0].input_dim + len(kerns) - 1, name)
         self.kerns = kerns

GPy/kern/_src/kern.py

@@ -3,28 +3,59 @@
 
 import sys
 import numpy as np
-import itertools
-from ...core.parameterization import Parameterized
-from ...core.parameterization.param import Param
+from ...core.parameterization.parameterized import Parameterized
+from kernel_slice_operations import KernCallsViaSlicerMeta
+from ...util.caching import Cache_this
+
 
 
 class Kern(Parameterized):
-    def __init__(self, input_dim, name, *a, **kw):
+    #===========================================================================
+    # This adds input slice support. The rather ugly code for slicing can be 
+    # found in kernel_slice_operations
+    __metaclass__ = KernCallsViaSlicerMeta
+    #===========================================================================
+    _debug=False
+    def __init__(self, input_dim, active_dims, name, *a, **kw):
         """
         The base class for a kernel: a positive definite function
         which forms of a covariance function (kernel).
 
-        :param input_dim: the number of input dimensions to the function
-        :type input_dim: int
+        :param int input_dim: the number of input dimensions to the function
+        :param array-like|slice active_dims: list of indices on which dimensions this kernel works on
 
         Do not instantiate.
         """
         super(Kern, self).__init__(name=name, *a, **kw)
+        self.active_dims = active_dims if active_dims is not None else slice(0, input_dim)
         self.input_dim = input_dim
+        assert isinstance(self.active_dims, (slice, list, tuple, np.ndarray)), 'active_dims needs to be an array-like or slice object over dimensions, {} given'.format(self.active_dims.__class__)
+        if isinstance(self.active_dims, slice):
+            self.active_dims = slice(self.active_dims.start or 0, self.active_dims.stop or self.input_dim, self.active_dims.step or 1)
+            active_dim_size = int(np.round((self.active_dims.stop-self.active_dims.start)/self.active_dims.step))
+        elif isinstance(self.active_dims, np.ndarray):
+            assert self.active_dims.ndim == 1, 'only flat indices allowed, given active_dims.shape={}, provide only indexes to the dimensions of the input'.format(self.active_dims.shape)
+            active_dim_size = self.active_dims.size
+        else:
+            active_dim_size = len(self.active_dims)
+        assert active_dim_size == self.input_dim, "input_dim={} does not match len(active_dim)={}, active_dims={}".format(self.input_dim, active_dim_size, self.active_dims)
+        self._sliced_X = 0
+
+    @Cache_this(limit=10)
+    def _slice_X(self, X):
+        return X[:, self.active_dims]
 
     def K(self, X, X2):
+        """
+        Compute the kernel function.
+
+        :param X: the first set of inputs to the kernel
+        :param X2: (optional) the second set of arguments to the kernel. If X2
+                   is None, this is passed throgh to the 'part' object, which
+                   handLes this as X2 == X.
+        """
         raise NotImplementedError
-    def Kdiag(self, Xa):
+    def Kdiag(self, X):
         raise NotImplementedError
     def psi0(self, Z, variational_posterior):
         raise NotImplementedError
@@ -34,7 +65,11 @@ class Kern(Parameterized):
         raise NotImplementedError
     def gradients_X(self, dL_dK, X, X2):
         raise NotImplementedError
-    def gradients_X_diag(self, dL_dK, X):
+    def gradients_X_diag(self, dL_dKdiag, X):
+        raise NotImplementedError
+
+    def update_gradients_diag(self, dL_dKdiag, X):
+        """ update the gradients of all parameters when using only the diagonal elements of the covariance matrix"""
         raise NotImplementedError
 
     def update_gradients_full(self, dL_dK, X, X2):
@@ -89,23 +124,16 @@ class Kern(Parameterized):
         """
         Returns the sensitivity for each dimension of this kernel.
         """
-        return self.kern.input_sensitivity()
+        return np.zeros(self.input_dim)
 
     def __add__(self, other):
         """ Overloading of the '+' operator. for more control, see self.add """
         return self.add(other)
 
-    def add(self, other, tensor=False):
+    def add(self, other, name='add'):
         """
         Add another kernel to this one.
 
-        If Tensor is False, both kernels are defined on the same _space_. then
-        the created kernel will have the same number of inputs as self and
-        other (which must be the same).
-
-        If Tensor is True, then the dimensions are stacked 'horizontally', so
-        that the resulting kernel has self.input_dim + other.input_dim
-
         :param other: the other kernel to be added
         :type other: GPy.kern
 
@@ -113,11 +141,11 @@ class Kern(Parameterized):
         assert isinstance(other, Kern), "only kernels can be added to kernels..."
         from add import Add
         kernels = []
-        if not tensor and isinstance(self, Add): kernels.extend(self._parameters_)
+        if isinstance(self, Add): kernels.extend(self._parameters_)
         else: kernels.append(self)
-        if not tensor and isinstance(other, Add): kernels.extend(other._parameters_)
+        if isinstance(other, Add): kernels.extend(other._parameters_)
         else: kernels.append(other)
-        return Add(kernels, tensor)
+        return Add(kernels, name=name)
 
     def __mul__(self, other):
         """ Here we overload the '*' operator. See self.prod for more information"""
@@ -127,9 +155,12 @@ class Kern(Parameterized):
         """
         Shortcut for tensor `prod`.
         """
-        return self.prod(other, tensor=True)
+        assert self.active_dims == range(self.input_dim), "Can only use kernels, which have their input_dims defined from 0"
+        assert other.active_dims == range(other.input_dim), "Can only use kernels, which have their input_dims defined from 0"
+        other.active_dims += self.input_dim
+        return self.prod(other)
 
-    def prod(self, other, tensor=False, name=None):
+    def prod(self, other, name='mul'):
         """
         Multiply two kernels (either on the same space, or on the tensor
         product of the input space).
@@ -142,4 +173,60 @@ class Kern(Parameterized):
         """
         assert isinstance(other, Kern), "only kernels can be added to kernels..."
         from prod import Prod
-        return Prod(self, other, tensor, name)
+        #kernels = []
+        #if isinstance(self, Prod): kernels.extend(self._parameters_)
+        #else: kernels.append(self)
+        #if isinstance(other, Prod): kernels.extend(other._parameters_)
+        #else: kernels.append(other)
+        return Prod([self, other], name)
+
+    def _getstate(self):
+        """
+        Get the current state of the class,
+        here just all the indices, rest can get recomputed
+        """
+        return super(Kern, self)._getstate() + [
+                self.active_dims,
+                self.input_dim,
+                self._sliced_X]
+
+    def _setstate(self, state):
+        self._sliced_X = state.pop()
+        self.input_dim = state.pop()
+        self.active_dims = state.pop()
+        super(Kern, self)._setstate(state)
+
+class CombinationKernel(Kern):
+    """
+    Abstract super class for combination kernels.
+    A combination kernel combines (a list of) kernels and works on those.
+    Examples are the HierarchicalKernel or Add and Prod kernels.
+    """
+    def __init__(self, kernels, name, extra_dims=[]):
+        """
+        Abstract super class for combination kernels.
+        A combination kernel combines (a list of) kernels and works on those.
+        Examples are the HierarchicalKernel or Add and Prod kernels.
+
+        :param list kernels: List of kernels to combine (can be only one element)
+        :param str name: name of the combination kernel
+        :param array-like|slice extra_dims: if needed extra dimensions for the combination kernel to work on
+        """
+        assert all([isinstance(k, Kern) for k in kernels])
+        active_dims = reduce(np.union1d, (np.r_[x.active_dims] for x in kernels), np.array([], dtype=int))
+        input_dim = active_dims.max()+1 + len(extra_dims)
+        active_dims = slice(active_dims.max()+1+len(extra_dims))
+        # initialize the kernel with the full input_dim
+        super(CombinationKernel, self).__init__(input_dim, active_dims, name)
+        self.extra_dims = extra_dims
+        self.add_parameters(*kernels)
+
+    @property
+    def parts(self):
+        return self._parameters_
+
+    def input_sensitivity(self):
+        in_sen = np.zeros((self.num_params, self.input_dim))
+        for i, p in enumerate(self.parts):
+            in_sen[i, p.active_dims] = p.input_sensitivity()
+        return in_sen

GPy/kern/_src/kernel_slice_operations.py

@@ -0,0 +1,134 @@
+'''
+Created on 11 Mar 2014
+
+@author: maxz
+'''
+from ...core.parameterization.parameterized import ParametersChangedMeta
+import numpy as np
+
+class KernCallsViaSlicerMeta(ParametersChangedMeta):
+    def __call__(self, *args, **kw):
+        instance = super(ParametersChangedMeta, self).__call__(*args, **kw)
+        instance.K = _slice_wrapper(instance, instance.K)
+        instance.Kdiag = _slice_wrapper(instance, instance.Kdiag, diag=True)
+        instance.update_gradients_full = _slice_wrapper(instance, instance.update_gradients_full, diag=False, derivative=True)
+        instance.update_gradients_diag = _slice_wrapper(instance, instance.update_gradients_diag, diag=True, derivative=True)
+        instance.gradients_X = _slice_wrapper(instance, instance.gradients_X, diag=False, derivative=True, ret_X=True)
+        instance.gradients_X_diag = _slice_wrapper(instance, instance.gradients_X_diag, diag=True, derivative=True, ret_X=True)
+        instance.psi0 = _slice_wrapper(instance, instance.psi0, diag=False, derivative=False)
+        instance.psi1 = _slice_wrapper(instance, instance.psi1, diag=False, derivative=False)
+        instance.psi2 = _slice_wrapper(instance, instance.psi2, diag=False, derivative=False)
+        instance.update_gradients_expectations = _slice_wrapper(instance, instance.update_gradients_expectations, derivative=True, psi_stat=True)
+        instance.gradients_Z_expectations = _slice_wrapper(instance, instance.gradients_Z_expectations, derivative=True, psi_stat_Z=True, ret_X=True)
+        instance.gradients_qX_expectations = _slice_wrapper(instance, instance.gradients_qX_expectations, derivative=True, psi_stat=True, ret_X=True)
+        instance.parameters_changed()
+        return instance
+
+def _slice_wrapper(kern, operation, diag=False, derivative=False, psi_stat=False, psi_stat_Z=False, ret_X=False):
+    """
+    This method wraps the functions in kernel to make sure all kernels allways see their respective input dimension.
+    The different switches are:
+        diag: if X2 exists
+        derivative: if first arg is dL_dK
+        psi_stat: if first 3 args are dL_dpsi0..2
+        psi_stat_Z: if first 2 args are dL_dpsi1..2
+    """
+    if derivative:
+        if diag:
+            def x_slice_wrapper(dL_dKdiag, X):
+                ret_X_not_sliced = ret_X and kern._sliced_X == 0
+                if ret_X_not_sliced:
+                    ret = np.zeros(X.shape)
+                X = kern._slice_X(X) if not kern._sliced_X else X
+                # if the return value is of shape X.shape, we need to make sure to return the right shape
+                kern._sliced_X += 1
+                try:
+                    if ret_X_not_sliced: ret[:, kern.active_dims] = operation(dL_dKdiag, X)
+                    else: ret = operation(dL_dKdiag, X)
+                except:
+                    raise
+                finally:
+                    kern._sliced_X -= 1
+                return ret
+        elif psi_stat:
+            def x_slice_wrapper(dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
+                ret_X_not_sliced = ret_X and kern._sliced_X == 0
+                if ret_X_not_sliced:
+                    ret1, ret2 = np.zeros(variational_posterior.shape), np.zeros(variational_posterior.shape)
+                Z, variational_posterior = kern._slice_X(Z) if not kern._sliced_X else Z, kern._slice_X(variational_posterior) if not kern._sliced_X else variational_posterior
+                kern._sliced_X += 1
+                # if the return value is of shape X.shape, we need to make sure to return the right shape
+                try:
+                    if ret_X_not_sliced:
+                        ret = list(operation(dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior))
+                        r2 = ret[:2]
+                        ret[0] = ret1
+                        ret[1] = ret2
+                        ret[0][:, kern.active_dims] = r2[0]
+                        ret[1][:, kern.active_dims] = r2[1]
+                        del r2
+                    else: ret = operation(dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior)
+                except:
+                    raise
+                finally:
+                    kern._sliced_X -= 1
+                return ret
+        elif psi_stat_Z:
+            def x_slice_wrapper(dL_dpsi1, dL_dpsi2, Z, variational_posterior):
+                ret_X_not_sliced = ret_X and kern._sliced_X == 0
+                if ret_X_not_sliced: ret = np.zeros(Z.shape)
+                Z, variational_posterior = kern._slice_X(Z) if not kern._sliced_X else Z, kern._slice_X(variational_posterior) if not kern._sliced_X else variational_posterior
+                kern._sliced_X += 1
+                try:
+                    if ret_X_not_sliced:
+                        ret[:, kern.active_dims] = operation(dL_dpsi1, dL_dpsi2, Z, variational_posterior)
+                    else: ret = operation(dL_dpsi1, dL_dpsi2, Z, variational_posterior)
+                except:
+                    raise
+                finally:
+                    kern._sliced_X -= 1
+                return ret
+        else:
+            def x_slice_wrapper(dL_dK, X, X2=None):
+                ret_X_not_sliced = ret_X and kern._sliced_X == 0
+                if ret_X_not_sliced:
+                    ret = np.zeros(X.shape)
+                X, X2 = kern._slice_X(X) if not kern._sliced_X else X, kern._slice_X(X2) if X2 is not None and not kern._sliced_X else X2
+                kern._sliced_X += 1
+                try:
+                    if ret_X_not_sliced: ret[:, kern.active_dims] = operation(dL_dK, X, X2)
+                    else: ret = operation(dL_dK, X, X2)
+                except:
+                    raise
+                finally:
+                    kern._sliced_X -= 1
+                return ret
+    else:
+        if diag:
+            def x_slice_wrapper(X, *args, **kw):
+                X = kern._slice_X(X) if not kern._sliced_X else X
+                kern._sliced_X += 1
+                try:
+                    ret = operation(X, *args, **kw)
+                except:
+                    raise
+                finally:
+                    kern._sliced_X -= 1
+                return ret
+        else: 
+            def x_slice_wrapper(X, X2=None, *args, **kw):
+                X, X2 = kern._slice_X(X) if not kern._sliced_X else X, kern._slice_X(X2) if X2 is not None and not kern._sliced_X else X2
+                kern._sliced_X += 1
+                try:
+                    ret = operation(X, X2, *args, **kw)
+                except: raise
+                finally:
+                    kern._sliced_X -= 1
+                return ret
+    x_slice_wrapper._operation = operation
+    x_slice_wrapper.__name__ = ("slicer("+operation.__name__
+                                +(","+str(bool(diag)) if diag else'')
+                                +(','+str(bool(derivative)) if derivative else '')
+                                +')')
+    x_slice_wrapper.__doc__ = "**sliced**\n" + (operation.__doc__ or "")
+    return x_slice_wrapper

GPy/kern/_src/linear.py

@@ -34,8 +34,8 @@ class Linear(Kern):
 
     """
 
-    def __init__(self, input_dim, variances=None, ARD=False, name='linear'):
-        super(Linear, self).__init__(input_dim, name)
+    def __init__(self, input_dim, variances=None, ARD=False, active_dims=None, name='linear'):
+        super(Linear, self).__init__(input_dim, active_dims, name)
         self.ARD = ARD
         if not ARD:
             if variances is not None:
@@ -147,7 +147,6 @@ class Linear(Kern):
             mu = variational_posterior.mean
             S = variational_posterior.variance
             mu2S = np.square(mu)+S
-            
             _dpsi2_dvariance, _, _, _, _ = linear_psi_comp._psi2computations(self.variances, Z, mu, S, gamma)
             grad = np.einsum('n,nq,nq->q',dL_dpsi0,gamma,mu2S) + np.einsum('nm,nq,mq,nq->q',dL_dpsi1,gamma,Z,mu) +\
                  np.einsum('nmo,nmoq->q',dL_dpsi2,_dpsi2_dvariance)
@@ -175,7 +174,7 @@ class Linear(Kern):
             mu = variational_posterior.mean
             S = variational_posterior.variance
             _, _, _, _, _dpsi2_dZ = linear_psi_comp._psi2computations(self.variances, Z, mu, S, gamma)
-            
+
             grad =  np.einsum('nm,nq,q,nq->mq',dL_dpsi1,gamma, self.variances,mu) +\
                  np.einsum('nmo,noq->mq',dL_dpsi2,_dpsi2_dZ)
             

GPy/kern/_src/mlp.py

@@ -31,8 +31,8 @@ class MLP(Kern):
 
     """
 
-    def __init__(self, input_dim, variance=1., weight_variance=1., bias_variance=100., name='mlp'):
-        super(MLP, self).__init__(input_dim, name)
+    def __init__(self, input_dim, variance=1., weight_variance=1., bias_variance=100., active_dims=None, name='mlp'):
+        super(MLP, self).__init__(input_dim, active_dims, name)
         self.variance = Param('variance', variance, Logexp())
         self.weight_variance = Param('weight_variance', weight_variance, Logexp())
         self.bias_variance = Param('bias_variance', bias_variance, Logexp())
@@ -96,12 +96,12 @@ class MLP(Kern):
             vec = (X*X).sum(1)*self.weight_variance+self.bias_variance + 1.
             return 2*four_over_tau*self.weight_variance*self.variance*((X[None, :, :]/denom[:, :, None] - vec[None, :, None]*X[:, None, :]*(numer/denom3)[:, :, None])*(dL_dK/np.sqrt(1-arg*arg))[:, :, None]).sum(1)
 
-    def dKdiag_dX(self, dL_dKdiag, X, target):
+    def gradients_X_diag(self, dL_dKdiag, X):
         """Gradient of diagonal of covariance with respect to X"""
         self._K_diag_computations(X)
         arg = self._K_diag_asin_arg
         denom = self._K_diag_denom
-        numer = self._K_diag_numer
+        #numer = self._K_diag_numer
         return four_over_tau*2.*self.weight_variance*self.variance*X*(1./denom*(1. - arg)*dL_dKdiag/(np.sqrt(1-arg*arg)))[:, None]
 
 

GPy/kern/_src/periodic.py

@@ -10,7 +10,7 @@ from ...core.parameterization.param import Param
 from ...core.parameterization.transformations import Logexp
 
 class Periodic(Kern):
-    def __init__(self, input_dim, variance, lengthscale, period, n_freq, lower, upper, name):
+    def __init__(self, input_dim, variance, lengthscale, period, n_freq, lower, upper, active_dims, name):
         """
         :type input_dim: int
         :param variance: the variance of the Matern kernel
@@ -25,7 +25,7 @@ class Periodic(Kern):
         """
 
         assert input_dim==1, "Periodic kernels are only defined for input_dim=1"
-        super(Periodic, self).__init__(input_dim, name)
+        super(Periodic, self).__init__(input_dim, active_dims, name)
         self.input_dim = input_dim
         self.lower,self.upper = lower, upper
         self.n_freq = n_freq
@@ -77,16 +77,17 @@ class PeriodicExponential(Periodic):
     Only defined for input_dim=1.
     """
 
-    def __init__(self, input_dim=1, variance=1., lengthscale=1., period=2.*np.pi, n_freq=10, lower=0., upper=4*np.pi, name='periodic_exponential'):
-        super(PeriodicExponential, self).__init__(input_dim, variance, lengthscale, period, n_freq, lower, upper, name)
+    def __init__(self, input_dim=1, variance=1., lengthscale=1., period=2.*np.pi, n_freq=10, lower=0., upper=4*np.pi, active_dims=None, name='periodic_exponential'):
+        super(PeriodicExponential, self).__init__(input_dim, variance, lengthscale, period, n_freq, lower, upper, active_dims, name)
 
     def parameters_changed(self):
         self.a = [1./self.lengthscale, 1.]
         self.b = [1]
 
         self.basis_alpha = np.ones((self.n_basis,))
-        self.basis_omega = np.array(sum([[i*2*np.pi/self.period]*2 for i in  range(1,self.n_freq+1)],[]))[:,0]
-        self.basis_phi =   np.array(sum([[-np.pi/2, 0.]  for i in range(1,self.n_freq+1)],[]))
+        self.basis_omega = (2*np.pi*np.arange(1,self.n_freq+1)/self.period).repeat(2)
+        self.basis_phi =   np.zeros(self.n_freq * 2)
+        self.basis_phi[::2] = -np.pi/2
 
         self.G = self.Gram_matrix()
         self.Gi = np.linalg.inv(self.G)
@@ -100,7 +101,6 @@ class PeriodicExponential(Periodic):
         Flower = np.array(self._cos(self.basis_alpha,self.basis_omega,self.basis_phi)(self.lower))[:,None]
         return(self.lengthscale/(2*self.variance) * Gint + 1./self.variance*np.dot(Flower,Flower.T))
 
-    #@silence_errors
     def update_gradients_full(self, dL_dK, X, X2=None):
         """derivative of the covariance matrix with respect to the parameters (shape is N x num_inducing x num_params)"""
         if X2 is None: X2 = X
@@ -187,15 +187,16 @@ class PeriodicMatern32(Periodic):
 
     """
 
-    def __init__(self, input_dim=1, variance=1., lengthscale=1., period=2.*np.pi, n_freq=10, lower=0., upper=4*np.pi, name='periodic_Matern32'):
-        super(PeriodicMatern32, self).__init__(input_dim, variance, lengthscale, period, n_freq, lower, upper, name)
+    def __init__(self, input_dim=1, variance=1., lengthscale=1., period=2.*np.pi, n_freq=10, lower=0., upper=4*np.pi, active_dims=None, name='periodic_Matern32'):
+        super(PeriodicMatern32, self).__init__(input_dim, variance, lengthscale, period, n_freq, lower, upper, active_dims, name)
     def parameters_changed(self):
         self.a = [3./self.lengthscale**2, 2*np.sqrt(3)/self.lengthscale, 1.]
         self.b = [1,self.lengthscale**2/3]
 
         self.basis_alpha = np.ones((self.n_basis,))
-        self.basis_omega = np.array(sum([[i*2*np.pi/self.period]*2 for i in  range(1,self.n_freq+1)],[]))
-        self.basis_phi =   np.array(sum([[-np.pi/2, 0.]  for i in range(1,self.n_freq+1)],[]))
+        self.basis_omega = (2*np.pi*np.arange(1,self.n_freq+1)/self.period).repeat(2)
+        self.basis_phi =   np.zeros(self.n_freq * 2)
+        self.basis_phi[::2] = -np.pi/2
 
         self.G = self.Gram_matrix()
         self.Gi = np.linalg.inv(self.G)
@@ -212,8 +213,8 @@ class PeriodicMatern32(Periodic):
         return(self.lengthscale**3/(12*np.sqrt(3)*self.variance) * Gint + 1./self.variance*np.dot(Flower,Flower.T) + self.lengthscale**2/(3.*self.variance)*np.dot(F1lower,F1lower.T))
 
 
-    @silence_errors
-    def update_gradients_full(self,dL_dK,X,X2,target):
+    #@silence_errors
+    def update_gradients_full(self,dL_dK,X,X2):
         """derivative of the covariance matrix with respect to the parameters (shape is num_data x num_inducing x num_params)"""
         if X2 is None: X2 = X
         FX  = self._cos(self.basis_alpha[None,:],self.basis_omega[None,:],self.basis_phi[None,:])(X)
@@ -299,16 +300,17 @@ class PeriodicMatern52(Periodic):
 
     """
 
-    def __init__(self, input_dim=1, variance=1., lengthscale=1., period=2.*np.pi, n_freq=10, lower=0., upper=4*np.pi, name='periodic_Matern52'):
-        super(PeriodicMatern52, self).__init__(input_dim, variance, lengthscale, period, n_freq, lower, upper, name)
+    def __init__(self, input_dim=1, variance=1., lengthscale=1., period=2.*np.pi, n_freq=10, lower=0., upper=4*np.pi, active_dims=None, name='periodic_Matern52'):
+        super(PeriodicMatern52, self).__init__(input_dim, variance, lengthscale, period, n_freq, lower, upper, active_dims, name)
 
     def parameters_changed(self):
         self.a = [5*np.sqrt(5)/self.lengthscale**3, 15./self.lengthscale**2,3*np.sqrt(5)/self.lengthscale, 1.]
         self.b  = [9./8, 9*self.lengthscale**4/200., 3*self.lengthscale**2/5., 3*self.lengthscale**2/(5*8.), 3*self.lengthscale**2/(5*8.)]
 
         self.basis_alpha = np.ones((2*self.n_freq,))
-        self.basis_omega = np.array(sum([[i*2*np.pi/self.period]*2 for i in  range(1,self.n_freq+1)],[]))
-        self.basis_phi =   np.array(sum([[-np.pi/2, 0.]  for i in range(1,self.n_freq+1)],[]))
+        self.basis_omega = (2*np.pi*np.arange(1,self.n_freq+1)/self.period).repeat(2)
+        self.basis_phi =   np.zeros(self.n_freq * 2)
+        self.basis_phi[::2] = -np.pi/2
 
         self.G = self.Gram_matrix()
         self.Gi = np.linalg.inv(self.G)

GPy/kern/_src/prod.py

@@ -1,10 +1,12 @@
 # Copyright (c) 2012, GPy authors (see AUTHORS.txt).
 # Licensed under the BSD 3-clause license (see LICENSE.txt)
 
-from kern import Kern
 import numpy as np
+from kern import CombinationKernel
+from ...util.caching import Cache_this
+import itertools
 
-class Prod(Kern):
+class Prod(CombinationKernel):
     """
     Computes the product of 2 kernels
 
@@ -15,49 +17,49 @@ class Prod(Kern):
     :rtype: kernel object
 
     """
-    def __init__(self, k1, k2, tensor=False,name=None):
-        if tensor:
-            name = k1.name + '_xx_' + k2.name if name is None else name
-            super(Prod, self).__init__(k1.input_dim + k2.input_dim, name)
-            self.slice1 = slice(0,k1.input_dim)
-            self.slice2 = slice(k1.input_dim,k1.input_dim+k2.input_dim)
-        else:
-            assert k1.input_dim == k2.input_dim, "Error: The input spaces of the kernels to multiply don't have the same dimension."
-            name = k1.name + '_x_' + k2.name if name is None else name
-            super(Prod, self).__init__(k1.input_dim, name)
-            self.slice1 = slice(0, self.input_dim)
-            self.slice2 = slice(0, self.input_dim)
-        self.k1 = k1
-        self.k2 = k2
-        self.add_parameters(self.k1, self.k2)
+    def __init__(self, kernels, name='mul'):
+        assert len(kernels) == 2, 'only implemented for two kernels as of yet'
+        super(Prod, self).__init__(kernels, name)
 
-    def K(self, X, X2=None):
-        if X2 is None:
-            return self.k1.K(X[:,self.slice1], None) * self.k2.K(X[:,self.slice2], None)
-        else:
-            return self.k1.K(X[:,self.slice1], X2[:,self.slice1]) * self.k2.K(X[:,self.slice2], X2[:,self.slice2])
+    @Cache_this(limit=2, force_kwargs=['which_parts'])
+    def K(self, X, X2=None, which_parts=None):
+        assert X.shape[1] == self.input_dim
+        if which_parts is None:
+            which_parts = self.parts
+        elif not isinstance(which_parts, (list, tuple)):
+            # if only one part is given
+            which_parts = [which_parts]
+        return reduce(np.multiply, (p.K(X, X2) for p in which_parts))
 
-    def Kdiag(self, X):
-        return self.k1.Kdiag(X[:,self.slice1]) * self.k2.Kdiag(X[:,self.slice2])
+    @Cache_this(limit=2, force_kwargs=['which_parts'])
+    def Kdiag(self, X, which_parts=None):
+        assert X.shape[1] == self.input_dim
+        if which_parts is None:
+            which_parts = self.parts
+        return reduce(np.multiply, (p.Kdiag(X) for p in which_parts))
 
-    def update_gradients_full(self, dL_dK, X):
-        self.k1.update_gradients_full(dL_dK*self.k2.K(X[:,self.slice2]), X[:,self.slice1])
-        self.k2.update_gradients_full(dL_dK*self.k1.K(X[:,self.slice1]), X[:,self.slice2])
+    def update_gradients_full(self, dL_dK, X, X2=None):
+        for k1,k2 in itertools.combinations(self.parts, 2):
+            k1.update_gradients_full(dL_dK*k2.K(X, X2), X, X2)
+            k2.update_gradients_full(dL_dK*k1.K(X, X2), X, X2)
+
+    def update_gradients_diag(self, dL_dKdiag, X):
+        for k1,k2 in itertools.combinations(self.parts, 2):
+            k1.update_gradients_diag(dL_dKdiag*k2.Kdiag(X), X)
+            k2.update_gradients_diag(dL_dKdiag*k1.Kdiag(X), X)
 
     def gradients_X(self, dL_dK, X, X2=None):
         target = np.zeros(X.shape)
-        if X2 is None:
-            target[:,self.slice1] += self.k1.gradients_X(dL_dK*self.k2.K(X[:,self.slice2]), X[:,self.slice1], None)
-            target[:,self.slice2] += self.k2.gradients_X(dL_dK*self.k1.K(X[:,self.slice1]), X[:,self.slice2], None)
-        else:
-            target[:,self.slice1] += self.k1.gradients_X(dL_dK*self.k2.K(X[:,self.slice2], X2[:,self.slice2]), X[:,self.slice1], X2[:,self.slice1])
-            target[:,self.slice2] += self.k2.gradients_X(dL_dK*self.k1.K(X[:,self.slice1], X2[:,self.slice1]), X[:,self.slice2], X2[:,self.slice2])
+        for k1,k2 in itertools.combinations(self.parts, 2):
+            target += k1.gradients_X(dL_dK*k2.K(X, X2), X, X2)
+            target += k2.gradients_X(dL_dK*k1.K(X, X2), X, X2)
         return target
 
     def gradients_X_diag(self, dL_dKdiag, X):
         target = np.zeros(X.shape)
-        target[:,self.slice1] = self.k1.gradients_X(dL_dKdiag*self.k2.Kdiag(X[:,self.slice2]), X[:,self.slice1])
-        target[:,self.slice2] += self.k2.gradients_X(dL_dKdiag*self.k1.Kdiag(X[:,self.slice1]), X[:,self.slice2])
+        for k1,k2 in itertools.combinations(self.parts, 2):
+            target += k1.gradients_X(dL_dKdiag*k2.Kdiag(X), X)
+            target += k2.gradients_X(dL_dKdiag*k1.Kdiag(X), X)
         return target
 
 

GPy/kern/_src/rbf.py

@@ -19,9 +19,8 @@ class RBF(Stationary):
        k(r) = \sigma^2 \exp \\bigg(- \\frac{1}{2} r^2 \\bigg)
 
     """
-
-    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=False, name='rbf'):
-        super(RBF, self).__init__(input_dim, variance, lengthscale, ARD, name)
+    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=False, active_dims=None, name='rbf'):
+        super(RBF, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name)
         self.weave_options = {}
 
     def K_of_r(self, r):
@@ -56,31 +55,33 @@ class RBF(Stationary):
         if isinstance(variational_posterior, variational.SpikeAndSlabPosterior):
             _, _dpsi1_dvariance, _, _, _, _, _dpsi1_dlengthscale = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
             _, _dpsi2_dvariance, _, _, _, _, _dpsi2_dlengthscale = ssrbf_psi_comp._psi2computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
-    
+
             #contributions from psi0:
             self.variance.gradient = np.sum(dL_dpsi0)
-    
+
             #from psi1
             self.variance.gradient += np.sum(dL_dpsi1 * _dpsi1_dvariance)
             if self.ARD:
                 self.lengthscale.gradient = (dL_dpsi1[:,:,None]*_dpsi1_dlengthscale).reshape(-1,self.input_dim).sum(axis=0)
             else:
                 self.lengthscale.gradient = (dL_dpsi1[:,:,None]*_dpsi1_dlengthscale).sum()  
-        
-    
+
             #from psi2
             self.variance.gradient += (dL_dpsi2 * _dpsi2_dvariance).sum()
             if self.ARD:
                 self.lengthscale.gradient += (dL_dpsi2[:,:,:,None] * _dpsi2_dlengthscale).reshape(-1,self.input_dim).sum(axis=0)
             else:
                 self.lengthscale.gradient += (dL_dpsi2[:,:,:,None] * _dpsi2_dlengthscale).sum()
-        
+
         elif isinstance(variational_posterior, variational.NormalPosterior):
-        
-            l2 = self.lengthscale **2
+            l2 = self.lengthscale**2
+            if l2.size != self.input_dim:
+                l2 = l2*np.ones(self.input_dim)
 
             #contributions from psi0:
             self.variance.gradient = np.sum(dL_dpsi0)
+            if self._debug:
+                num_grad = self.lengthscale.gradient.copy()
             self.lengthscale.gradient = 0.
 
             #from psi1
@@ -92,16 +93,16 @@ class RBF(Stationary):
             else:
                 self.lengthscale.gradient += dpsi1_dlength.sum()
             self.variance.gradient += np.sum(dL_dpsi1 * psi1) / self.variance
-
             #from psi2
             S = variational_posterior.variance
             _, Zdist_sq, _, mudist_sq, psi2 = self._psi2computations(Z, variational_posterior)
-
             if not self.ARD:
                 self.lengthscale.gradient += self._weave_psi2_lengthscale_grads(dL_dpsi2, psi2, Zdist_sq, S, mudist_sq, l2).sum()
             else:
                 self.lengthscale.gradient += self._weave_psi2_lengthscale_grads(dL_dpsi2, psi2, Zdist_sq, S, mudist_sq, l2)
 
+            if self._debug:
+                import ipdb;ipdb.set_trace()
             self.variance.gradient += 2.*np.sum(dL_dpsi2 * psi2)/self.variance
 
         else:
@@ -112,17 +113,16 @@ class RBF(Stationary):
         if isinstance(variational_posterior, variational.SpikeAndSlabPosterior):
             _, _, _, _, _, _dpsi1_dZ, _ = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
             _, _, _, _, _, _dpsi2_dZ, _ = ssrbf_psi_comp._psi2computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
-    
+
             #psi1
             grad = (dL_dpsi1[:, :, None] * _dpsi1_dZ).sum(axis=0)
-    
+
             #psi2
             grad += (dL_dpsi2[:, :, :, None] * _dpsi2_dZ).sum(axis=0).sum(axis=1)
-    
+
             return grad
 
         elif isinstance(variational_posterior, variational.NormalPosterior):
-            
             l2 = self.lengthscale **2
 
             #psi1
@@ -145,23 +145,24 @@ class RBF(Stationary):
         # Spike-and-Slab GPLVM
         if isinstance(variational_posterior, variational.SpikeAndSlabPosterior):
             ndata = variational_posterior.mean.shape[0]
-            
+
             _, _, _dpsi1_dgamma, _dpsi1_dmu, _dpsi1_dS, _, _ = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
             _, _, _dpsi2_dgamma, _dpsi2_dmu, _dpsi2_dS, _, _ = ssrbf_psi_comp._psi2computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
-    
+
             #psi1
             grad_mu = (dL_dpsi1[:, :, None] * _dpsi1_dmu).sum(axis=1)
             grad_S = (dL_dpsi1[:, :, None] * _dpsi1_dS).sum(axis=1)
             grad_gamma = (dL_dpsi1[:,:,None] * _dpsi1_dgamma).sum(axis=1)
+
             #psi2
             grad_mu += (dL_dpsi2[:, :, :, None] * _dpsi2_dmu).reshape(ndata,-1,self.input_dim).sum(axis=1)
             grad_S += (dL_dpsi2[:, :, :, None] * _dpsi2_dS).reshape(ndata,-1,self.input_dim).sum(axis=1)
             grad_gamma += (dL_dpsi2[:,:,:, None] * _dpsi2_dgamma).reshape(ndata,-1,self.input_dim).sum(axis=1)
-            
+
             return grad_mu, grad_S, grad_gamma
 
         elif isinstance(variational_posterior, variational.NormalPosterior):
-        
+
             l2 = self.lengthscale **2
             #psi1
             denom, dist, dist_sq, psi1 = self._psi1computations(Z, variational_posterior)

GPy/kern/_src/ssrbf.py

@@ -33,9 +33,9 @@ class SSRBF(Stationary):
     .. Note: this object implements both the ARD and 'spherical' version of the function
     """
 
-    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=True, name='SSRBF'):
+    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=True, active_dims=None, name='SSRBF'):
         assert ARD==True, "Not Implemented!"
-        super(SSRBF, self).__init__(input_dim, variance, lengthscale, ARD, name)
+        super(SSRBF, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name)
         
     def K_of_r(self, r):
         return self.variance * np.exp(-0.5 * r**2)

GPy/kern/_src/static.py

@@ -9,8 +9,8 @@ from ...core.parameterization.transformations import Logexp
 import numpy as np
 
 class Static(Kern):
-    def __init__(self, input_dim, variance, name):
-        super(Static, self).__init__(input_dim, name)
+    def __init__(self, input_dim, variance, active_dims, name):
+        super(Static, self).__init__(input_dim, active_dims, name)
         self.variance = Param('variance', variance, Logexp())
         self.add_parameters(self.variance)
 
@@ -43,8 +43,8 @@ class Static(Kern):
 
 
 class White(Static):
-    def __init__(self, input_dim, variance=1., name='white'):
-        super(White, self).__init__(input_dim, variance, name)
+    def __init__(self, input_dim, variance=1., active_dims=None, name='white'):
+        super(White, self).__init__(input_dim, variance, active_dims, name)
 
     def K(self, X, X2=None):
         if X2 is None:
@@ -55,7 +55,7 @@ class White(Static):
     def psi2(self, Z, variational_posterior):
         return np.zeros((variational_posterior.shape[0], Z.shape[0], Z.shape[0]), dtype=np.float64)
 
-    def update_gradients_full(self, dL_dK, X):
+    def update_gradients_full(self, dL_dK, X, X2=None):
         self.variance.gradient = np.trace(dL_dK)
 
     def update_gradients_diag(self, dL_dKdiag, X):
@@ -66,8 +66,8 @@ class White(Static):
 
 
 class Bias(Static):
-    def __init__(self, input_dim, variance=1., name='bias'):
-        super(Bias, self).__init__(input_dim, variance, name)
+    def __init__(self, input_dim, variance=1., active_dims=None, name='bias'):
+        super(Bias, self).__init__(input_dim, variance, active_dims, name)
 
     def K(self, X, X2=None):
         shape = (X.shape[0], X.shape[0] if X2 is None else X2.shape[0])
@@ -79,13 +79,41 @@ class Bias(Static):
         self.variance.gradient = dL_dK.sum()
 
     def update_gradients_diag(self, dL_dKdiag, X):
-        self.variance.gradient = dL_dK.sum()
+        self.variance.gradient = dL_dKdiag.sum()
 
     def psi2(self, Z, variational_posterior):
-        ret = np.empty((mu.shape[0], Z.shape[0], Z.shape[0]), dtype=np.float64)
+        ret = np.empty((variational_posterior.shape[0], Z.shape[0], Z.shape[0]), dtype=np.float64)
         ret[:] = self.variance**2
         return ret
 
     def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
         self.variance.gradient = dL_dpsi0.sum() + dL_dpsi1.sum() + 2.*self.variance*dL_dpsi2.sum()
 
+class Fixed(Static):
+    def __init__(self, input_dim, covariance_matrix, variance=1., active_dims=None, name='fixed'):
+        """
+        :param input_dim: the number of input dimensions
+        :type input_dim: int
+        :param variance: the variance of the kernel
+        :type variance: float
+        """
+        super(Bias, self).__init__(input_dim, variance, active_dims, name)
+        self.fixed_K = covariance_matrix
+    def K(self, X, X2):
+        return self.variance * self.fixed_K
+
+    def Kdiag(self, X):
+        return self.variance * self.fixed_K.diag()
+
+    def update_gradients_full(self, dL_dK, X, X2=None):
+        self.variance.gradient = np.einsum('ij,ij', dL_dK, self.fixed_K)
+
+    def update_gradients_diag(self, dL_dKdiag, X):
+        self.variance.gradient = np.einsum('i,i', dL_dKdiag, self.fixed_K)
+
+    def psi2(self, Z, variational_posterior):
+        return np.zeros((variational_posterior.shape[0], Z.shape[0], Z.shape[0]), dtype=np.float64)
+
+    def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
+        self.variance.gradient = dL_dpsi0.sum()
+

GPy/kern/_src/stationary.py

@@ -15,21 +15,21 @@ class Stationary(Kern):
     """
     Stationary kernels (covariance functions).
 
-    Stationary covariance fucntion depend only on r, where r is defined as 
+    Stationary covariance fucntion depend only on r, where r is defined as
 
       r = \sqrt{ \sum_{q=1}^Q (x_q - x'_q)^2 }
 
-    The covariance function k(x, x' can then be written k(r). 
+    The covariance function k(x, x' can then be written k(r).
 
     In this implementation, r is scaled by the lengthscales parameter(s):
 
-      r = \sqrt{ \sum_{q=1}^Q \frac{(x_q - x'_q)^2}{\ell_q^2} }. 
-    
+      r = \sqrt{ \sum_{q=1}^Q \frac{(x_q - x'_q)^2}{\ell_q^2} }.
+
     By default, there's only one lengthscale: seaprate lengthscales for each
-    dimension can be enables by setting ARD=True. 
+    dimension can be enables by setting ARD=True.
 
     To implement a stationary covariance function using this class, one need
-    only define the covariance function k(r), and it derivative. 
+    only define the covariance function k(r), and it derivative.
 
       ...
       def K_of_r(self, r):
@@ -37,12 +37,12 @@ class Stationary(Kern):
       def dK_dr(self, r):
           return bar
 
-    The lengthscale(s) and variance parameters are added to the structure automatically. 
-          
+    The lengthscale(s) and variance parameters are added to the structure automatically.
+
     """
-    
-    def __init__(self, input_dim, variance, lengthscale, ARD, name):
-        super(Stationary, self).__init__(input_dim, name)
+
+    def __init__(self, input_dim, variance, lengthscale, ARD, active_dims, name):
+        super(Stationary, self).__init__(input_dim, active_dims, name)
         self.ARD = ARD
         if not ARD:
             if lengthscale is None:
@@ -85,15 +85,19 @@ class Stationary(Kern):
         Compute the Euclidean distance between each row of X and X2, or between
         each pair of rows of X if X2 is None.
         """
+        #X, = self._slice_X(X)
         if X2 is None:
             Xsq = np.sum(np.square(X),1)
             r2 = -2.*tdot(X) + (Xsq[:,None] + Xsq[None,:])
             util.diag.view(r2)[:,]= 0. # force diagnoal to be zero: sometime numerically a little negative
             return np.sqrt(r2)
         else:
+            #X2, = self._slice_X(X2)
             X1sq = np.sum(np.square(X),1)
             X2sq = np.sum(np.square(X2),1)
-            return np.sqrt(-2.*np.dot(X, X2.T) + (X1sq[:,None] + X2sq[None,:]))
+            r2 = -2.*np.dot(X, X2.T) + X1sq[:,None] + X2sq[None,:]
+            r2[r2<0] = 0. # A bit hacky
+            return np.sqrt(r2)
 
     @Cache_this(limit=5, ignore_args=())
     def _scaled_dist(self, X, X2=None):
@@ -124,7 +128,6 @@ class Stationary(Kern):
         self.lengthscale.gradient = 0.
 
     def update_gradients_full(self, dL_dK, X, X2=None):
-
         self.variance.gradient = np.einsum('ij,ij,i', self.K(X, X2), dL_dK, 1./self.variance)
 
         #now the lengthscale gradient(s)
@@ -132,11 +135,11 @@ class Stationary(Kern):
         if self.ARD:
             #rinv = self._inv_dis# this is rather high memory? Should we loop instead?t(X, X2)
             #d =  X[:, None, :] - X2[None, :, :]
-            #x_xl3 = np.square(d) 
+            #x_xl3 = np.square(d)
             #self.lengthscale.gradient = -((dL_dr*rinv)[:,:,None]*x_xl3).sum(0).sum(0)/self.lengthscale**3
             tmp = dL_dr*self._inv_dist(X, X2)
             if X2 is None: X2 = X
-            self.lengthscale.gradient = np.array([np.einsum('ij,ij,...', tmp, np.square(X[:,q:q+1] - X2[:,q:q+1].T), -1./self.lengthscale[q]**3) for q in xrange(self.input_dim)])
+            self.lengthscale.gradient = np.array([np.einsum('ij,ij,...', tmp, np.square(self._slice_X(X)[:,q:q+1] - self._slice_X(X2)[:,q:q+1].T), -1./self.lengthscale[q]**3) for q in xrange(self.input_dim)])
         else:
             r = self._scaled_dist(X, X2)
             self.lengthscale.gradient = -np.sum(dL_dr*r)/self.lengthscale
@@ -176,7 +179,6 @@ class Stationary(Kern):
         ret = np.empty(X.shape, dtype=np.float64)
         [np.einsum('ij,ij->i', tmp, X[:,q][:,None]-X2[:,q][None,:], out=ret[:,q]) for q in xrange(self.input_dim)]
         ret /= self.lengthscale**2
-
         return ret
 
     def gradients_X_diag(self, dL_dKdiag, X):
@@ -186,8 +188,8 @@ class Stationary(Kern):
         return np.ones(self.input_dim)/self.lengthscale
 
 class Exponential(Stationary):
-    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=False, name='Exponential'):
-        super(Exponential, self).__init__(input_dim, variance, lengthscale, ARD, name)
+    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=False, active_dims=None, name='Exponential'):
+        super(Exponential, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name)
 
     def K_of_r(self, r):
         return self.variance * np.exp(-0.5 * r)
@@ -205,8 +207,8 @@ class Matern32(Stationary):
 
     """
 
-    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=False, name='Mat32'):
-        super(Matern32, self).__init__(input_dim, variance, lengthscale, ARD, name)
+    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=False, active_dims=None, name='Mat32'):
+        super(Matern32, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name)
 
     def K_of_r(self, r):
         return self.variance * (1. + np.sqrt(3.) * r) * np.exp(-np.sqrt(3.) * r)
@@ -247,10 +249,10 @@ class Matern52(Stationary):
 
     .. math::
 
-       k(r) = \sigma^2 (1 + \sqrt{5} r + \\frac53 r^2) \exp(- \sqrt{5} r) 
+       k(r) = \sigma^2 (1 + \sqrt{5} r + \\frac53 r^2) \exp(- \sqrt{5} r)
        """
-    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=False, name='Mat52'):
-        super(Matern52, self).__init__(input_dim, variance, lengthscale, ARD, name)
+    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=False, active_dims=None, name='Mat52'):
+        super(Matern52, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name)
 
     def K_of_r(self, r):
         return self.variance*(1+np.sqrt(5.)*r+5./3*r**2)*np.exp(-np.sqrt(5.)*r)
@@ -291,8 +293,8 @@ class Matern52(Stationary):
 
 
 class ExpQuad(Stationary):
-    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=False, name='ExpQuad'):
-        super(ExpQuad, self).__init__(input_dim, variance, lengthscale, ARD, name)
+    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=False, active_dims=None, name='ExpQuad'):
+        super(ExpQuad, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name)
 
     def K_of_r(self, r):
         return self.variance * np.exp(-0.5 * r**2)
@@ -301,8 +303,8 @@ class ExpQuad(Stationary):
         return -r*self.K_of_r(r)
 
 class Cosine(Stationary):
-    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=False, name='Cosine'):
-        super(Cosine, self).__init__(input_dim, variance, lengthscale, ARD, name)
+    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=False, active_dims=None, name='Cosine'):
+        super(Cosine, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name)
 
     def K_of_r(self, r):
         return self.variance * np.cos(r)
@@ -322,8 +324,8 @@ class RatQuad(Stationary):
     """
 
 
-    def __init__(self, input_dim, variance=1., lengthscale=None, power=2., ARD=False, name='ExpQuad'):
-        super(RatQuad, self).__init__(input_dim, variance, lengthscale, ARD, name)
+    def __init__(self, input_dim, variance=1., lengthscale=None, power=2., ARD=False, active_dims=None, name='ExpQuad'):
+        super(RatQuad, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name)
         self.power = Param('power', power, Logexp())
         self.add_parameters(self.power)
 

GPy/kern/_src/sympykern.py

@@ -26,13 +26,13 @@ class Sympykern(Kern):
      - to handle multiple inputs, call them x_1, z_1, etc
      - to handle multpile correlated outputs, you'll need to add parameters with an index, such as lengthscale_i and lengthscale_j.
     """
-    def __init__(self, input_dim, k=None, output_dim=1, name=None, param=None):
+    def __init__(self, input_dim, k=None, output_dim=1, name=None, param=None, active_dims=None):
 
         if name is None:
             name='sympykern'
         if k is None:
             raise ValueError, "You must provide an argument for the covariance function."
-        super(Sympykern, self).__init__(input_dim, name)
+        super(Sympykern, self).__init__(input_dim, active_dims, name)
 
         self._sp_k = k
 
@@ -116,6 +116,7 @@ class Sympykern(Kern):
         if self.output_dim > 1:
             self.arg_list += self._sp_theta_i + self._sp_theta_j
             self.diag_arg_list += self._sp_theta_i
+
         # psi_stats aren't yet implemented.
         if False:
             self.compute_psi_stats()

GPy/likelihoods/__init__.py

@@ -5,3 +5,4 @@ from gamma import Gamma
 from poisson import Poisson
 from student_t import StudentT
 from likelihood import Likelihood
+from mixed_noise import MixedNoise

GPy/likelihoods/bernoulli.py

@@ -5,6 +5,7 @@ import numpy as np
 from ..util.univariate_Gaussian import std_norm_pdf, std_norm_cdf
 import link_functions
 from likelihood import Likelihood
+from scipy import stats
 
 class Bernoulli(Likelihood):
     """
@@ -43,7 +44,7 @@ class Bernoulli(Likelihood):
         Y_prep[Y.flatten() == 0] = -1
         return Y_prep
 
-    def moments_match_ep(self, data_i, tau_i, v_i):
+    def moments_match_ep(self, Y_i, tau_i, v_i):
         """
         Moments match of the marginal approximation in EP algorithm
 
@@ -51,9 +52,9 @@ class Bernoulli(Likelihood):
         :param tau_i: precision of the cavity distribution (float)
         :param v_i: mean/variance of the cavity distribution (float)
         """
-        if data_i == 1:
+        if Y_i == 1:
             sign = 1.
-        elif data_i == 0:
+        elif Y_i == 0:
             sign = -1
         else:
             raise ValueError("bad value for Bernouilli observation (0, 1)")
@@ -76,7 +77,7 @@ class Bernoulli(Likelihood):
 
         return Z_hat, mu_hat, sigma2_hat
 
-    def predictive_mean(self, mu, variance):
+    def predictive_mean(self, mu, variance, Y_metadata=None):
 
         if isinstance(self.gp_link, link_functions.Probit):
             return stats.norm.cdf(mu/np.sqrt(1+variance))
@@ -87,13 +88,12 @@ class Bernoulli(Likelihood):
         else:
             raise NotImplementedError
 
-    def predictive_variance(self, mu, variance, pred_mean):
+    def predictive_variance(self, mu, variance, pred_mean, Y_metadata=None):
 
         if isinstance(self.gp_link, link_functions.Heaviside):
             return 0.
         else:
             return np.nan
-            #raise NotImplementedError
 
     def pdf_link(self, link_f, y, Y_metadata=None):
         """
@@ -212,7 +212,7 @@ class Bernoulli(Likelihood):
         np.seterr(**state)
         return d3logpdf_dlink3
 
-    def samples(self, gp):
+    def samples(self, gp, Y_metadata=None):
         """
         Returns a set of samples of observations based on a given value of the latent variable.
 

GPy/likelihoods/gaussian.py

@@ -2,7 +2,7 @@
 # Licensed under the BSD 3-clause license (see LICENSE.txt)
 #TODO
 """
-A lot of this code assumes that the link function is the identity. 
+A lot of this code assumes that the link function is the identity.
 
 I think laplace code is okay, but I'm quite sure that the EP moments will only work if the link is identity.
 
@@ -18,6 +18,7 @@ import link_functions
 from likelihood import Likelihood
 from ..core.parameterization import Param
 from ..core.parameterization.transformations import Logexp
+from scipy import stats
 
 class Gaussian(Likelihood):
     """
@@ -49,11 +50,18 @@ class Gaussian(Likelihood):
         if isinstance(gp_link, link_functions.Identity):
             self.log_concave = True
 
-    def covariance_matrix(self, Y, Y_metadata=None):
-        return np.eye(Y.shape[0]) * self.variance
+    def betaY(self,Y,Y_metadata=None):
+        #TODO: ~Ricardo this does not live here
+        return Y/self.gaussian_variance(Y_metadata)
 
-    def update_gradients(self, partial):
-        self.variance.gradient = np.sum(partial)
+    def gaussian_variance(self, Y_metadata=None):
+        return self.variance
+
+    def update_gradients(self, grad):
+        self.variance.gradient = grad
+
+    def exact_inference_gradients(self, dL_dKdiag,Y_metadata=None):
+        return dL_dKdiag.sum()
 
     def _preprocess_values(self, Y):
         """
@@ -76,16 +84,12 @@ class Gaussian(Likelihood):
         Z_hat = 1./np.sqrt(2.*np.pi*sum_var)*np.exp(-.5*(data_i - v_i/tau_i)**2./sum_var)
         return Z_hat, mu_hat, sigma2_hat
 
-    def predictive_values(self, mu, var, full_cov=False):
+    def predictive_values(self, mu, var, full_cov=False, Y_metadata=None):
         if full_cov:
             var += np.eye(var.shape[0])*self.variance
-            d = 2*np.sqrt(np.diag(var))
-            low, up = mu - d, mu + d
         else:
             var += self.variance
-            d = 2*np.sqrt(var)
-            low, up = mu - d, mu + d
-        return mu, var, low, up
+        return mu, var
 
     def predictive_mean(self, mu, sigma):
         return mu
@@ -93,7 +97,14 @@ class Gaussian(Likelihood):
     def predictive_variance(self, mu, sigma, predictive_mean=None):
         return self.variance + sigma**2
 
+<<<<<<< HEAD
     def pdf_link(self, link_f, y, Y_metadata=None):
+=======
+    def predictive_quantiles(self, mu, var, quantiles, Y_metadata):
+        return  [stats.norm.ppf(q/100.)*np.sqrt(var) + mu for q in quantiles]
+
+    def pdf_link(self, link_f, y, extra_data=None):
+>>>>>>> a3287c38ea775155df4e90f7fe1883d12ffb54b9
         """
         Likelihood function given link(f)
 
@@ -292,7 +303,7 @@ class Gaussian(Likelihood):
         """
         return self.variance
 
-    def samples(self, gp):
+    def samples(self, gp, Y_metadata=None):
         """
         Returns a set of samples of observations based on a given value of the latent variable.
 
@@ -300,6 +311,8 @@ class Gaussian(Likelihood):
         """
         orig_shape = gp.shape
         gp = gp.flatten()
+        #orig_shape = gp.shape
+        gp = gp.flatten()
         Ysim = np.array([np.random.normal(self.gp_link.transf(gpj), scale=np.sqrt(self.variance), size=1) for gpj in gp])
         return Ysim.reshape(orig_shape)
 

GPy/likelihoods/likelihood.py

@@ -58,6 +58,18 @@ class Likelihood(Parameterized):
         """
         return Y
 
+    def conditional_mean(self, gp):
+        """
+        The mean of the random variable conditioned on one value of the GP
+        """
+        raise NotImplementedError
+
+    def conditional_variance(self, gp):
+        """
+        The variance of the random variable conditioned on one value of the GP
+        """
+        raise NotImplementedError
+
     def log_predictive_density(self, y_test, mu_star, var_star):
         """
         Calculation of the log predictive density
@@ -120,7 +132,7 @@ class Likelihood(Parameterized):
 
         return z, mean, variance
 
-    def _predictive_mean(self, mu, variance):
+    def predictive_mean(self, mu, variance, Y_metadata=None):
         """
         Quadrature calculation of the predictive mean: E(Y_star|Y) = E( E(Y_star|f_star, Y) )
 
@@ -128,8 +140,14 @@ class Likelihood(Parameterized):
         :param sigma: standard deviation of posterior
 
         """
+        #conditional_mean: the edpected value of y given some f, under this likelihood
         def int_mean(f,m,v):
-            return self._mean(f)*np.exp(-(0.5/v)*np.square(f - m))
+            p = np.exp(-(0.5/v)*np.square(f - m))
+            #If p is zero then conditional_mean will overflow
+            if p < 1e-10:
+                return 0.
+            else:
+                return self.conditional_mean(f)*p
         scaled_mean = [quad(int_mean, -np.inf, np.inf,args=(mj,s2j))[0] for mj,s2j in zip(mu,variance)]
         mean = np.array(scaled_mean)[:,None] / np.sqrt(2*np.pi*(variance))
 
@@ -139,7 +157,7 @@ class Likelihood(Parameterized):
         """Quadrature calculation of the conditional mean: E(Y_star|f)"""
         raise NotImplementedError, "implement this function to make predictions"
 
-    def _predictive_variance(self,mu,variance,predictive_mean=None):
+    def predictive_variance(self, mu,variance, predictive_mean=None, Y_metadata=None):
         """
         Numerical approximation to the predictive variance: V(Y_star)
 
@@ -156,7 +174,12 @@ class Likelihood(Parameterized):
 
         # E( V(Y_star|f_star) )
         def int_var(f,m,v):
-            return self._variance(f)*np.exp(-(0.5/v)*np.square(f - m))
+            p = np.exp(-(0.5/v)*np.square(f - m))
+            #If p is zero then conditional_variance will overflow
+            if p < 1e-10:
+                return 0.
+            else:
+                return self.conditional_variance(f)*p
         scaled_exp_variance = [quad(int_var, -np.inf, np.inf,args=(mj,s2j))[0] for mj,s2j in zip(mu,variance)]
         exp_var = np.array(scaled_exp_variance)[:,None] / normalizer
 
@@ -169,13 +192,20 @@ class Likelihood(Parameterized):
 
         #E( E(Y_star|f_star)**2 )
         def int_pred_mean_sq(f,m,v,predictive_mean_sq):
-            return self._mean(f)**2*np.exp(-(0.5/v)*np.square(f - m))
+            p = np.exp(-(0.5/v)*np.square(f - m))
+            #If p is zero then conditional_mean**2 will overflow
+            if p < 1e-10:
+                return 0.
+            else:
+                return self.conditional_mean(f)**2*p
+
         scaled_exp_exp2 = [quad(int_pred_mean_sq, -np.inf, np.inf,args=(mj,s2j,pm2j))[0] for mj,s2j,pm2j in zip(mu,variance,predictive_mean_sq)]
         exp_exp2 = np.array(scaled_exp_exp2)[:,None] / normalizer
 
         var_exp = exp_exp2 - predictive_mean_sq
 
-        # V(Y_star) = E( V(Y_star|f_star) ) + V( E(Y_star|f_star) )
+        # V(Y_star) = E[ V(Y_star|f_star) ] + V[ E(Y_star|f_star) ]
+        # V(Y_star) = E[ V(Y_star|f_star) ] + E(Y_star**2|f_star) - E[Y_star|f_star]**2
         return exp_var + var_exp
 
     def pdf_link(self, link_f, y, Y_metadata=None):
@@ -362,18 +392,33 @@ class Likelihood(Parameterized):
 
         return dlogpdf_dtheta, dlogpdf_df_dtheta, d2logpdf_df2_dtheta
 
-    def predictive_values(self, mu, var):
+    def predictive_values(self, mu, var, full_cov=False, Y_metadata=None):
         """
         Compute  mean, variance of the  predictive distibution.
 
         :param mu: mean of the latent variable, f, of posterior
         :param var: variance of the latent variable, f, of posterior
+        :param full_cov: whether to use the full covariance or just the diagonal
+        :type full_cov: Boolean
         """
-        pred_mean = self.predictive_mean(mu, var)
-        pred_var = self.predictive_variance(mu, var, pred_mean)
+
+        pred_mean = self.predictive_mean(mu, var, Y_metadata)
+        pred_var = self.predictive_variance(mu, var, pred_mean, Y_metadata)
+
         return pred_mean, pred_var
 
-    def samples(self, gp):
+    def predictive_quantiles(self, mu, var, quantiles, Y_metadata=None):
+        #compute the quantiles by sampling!!!
+        N_samp = 1000
+        s = np.random.randn(mu.shape[0], N_samp)*np.sqrt(var) + mu
+        #ss_f = s.flatten()
+        #ss_y = self.samples(ss_f, Y_metadata)
+        ss_y = self.samples(s, Y_metadata)
+        #ss_y = ss_y.reshape(mu.shape[0], N_samp)
+
+        return [np.percentile(ss_y ,q, axis=1)[:,None] for q in quantiles]
+
+    def samples(self, gp, Y_metadata=None):
         """
         Returns a set of samples of observations based on a given value of the latent variable.
 

GPy/likelihoods/link_functions.py

@@ -6,6 +6,9 @@ from scipy import stats
 import scipy as sp
 from GPy.util.univariate_Gaussian import std_norm_pdf,std_norm_cdf,inv_std_norm_cdf
 
+_exp_lim_val = np.finfo(np.float64).max
+_lim_val = np.log(_exp_lim_val)
+
 class GPTransformation(object):
     """
     Link function class for doing non-Gaussian likelihoods approximation
@@ -92,16 +95,16 @@ class Log(GPTransformation):
 
     """
     def transf(self,f):
-        return np.exp(f)
+        return np.exp(np.clip(f, -_lim_val, _lim_val))
 
     def dtransf_df(self,f):
-        return np.exp(f)
+        return np.exp(np.clip(f, -_lim_val, _lim_val))
 
     def d2transf_df2(self,f):
-        return np.exp(f)
+        return np.exp(np.clip(f, -_lim_val, _lim_val))
 
     def d3transf_df3(self,f):
-        return np.exp(f)
+        return np.exp(np.clip(f, -_lim_val, _lim_val))
 
 class Log_ex_1(GPTransformation):
     """

GPy/likelihoods/mixed_noise.py

@@ -0,0 +1,87 @@
+import numpy as np
+from scipy import stats, special
+from GPy.util.univariate_Gaussian import std_norm_pdf, std_norm_cdf
+import link_functions
+from likelihood import Likelihood
+from gaussian import Gaussian
+from ..core.parameterization import Param
+from ..core.parameterization.transformations import Logexp
+from ..core.parameterization import Parameterized
+import itertools
+
+class MixedNoise(Likelihood):
+    def __init__(self, likelihoods_list, name='mixed_noise'):
+
+        super(Likelihood, self).__init__(name=name)
+
+        self.add_parameters(*likelihoods_list)
+        self.likelihoods_list = likelihoods_list
+        self.log_concave = False
+
+    def gaussian_variance(self, Y_metadata):
+        assert all([isinstance(l, Gaussian) for l in self.likelihoods_list])
+        ind = Y_metadata['output_index'].flatten()
+        variance = np.zeros(ind.size)
+        for lik, j in zip(self.likelihoods_list, range(len(self.likelihoods_list))):
+            variance[ind==j] = lik.variance
+        return variance[:,None]
+
+    def betaY(self,Y,Y_metadata):
+        return Y/self.gaussian_variance(Y_metadata=Y_metadata)
+
+    def update_gradients(self, gradients):
+        self.gradient = gradients
+
+    def exact_inference_gradients(self, dL_dKdiag, Y_metadata):
+        assert all([isinstance(l, Gaussian) for l in self.likelihoods_list])
+        ind = Y_metadata['output_index'].flatten()
+        return np.array([dL_dKdiag[ind==i].sum() for i in range(len(self.likelihoods_list))])
+
+    def predictive_values(self, mu, var, full_cov=False, Y_metadata=None):
+        if all([isinstance(l, Gaussian) for l in self.likelihoods_list]):
+            ind = Y_metadata['output_index'].flatten()
+            _variance = np.array([self.likelihoods_list[j].variance for j in ind ])
+            if full_cov:
+                var += np.eye(var.shape[0])*_variance
+            else:
+                var += _variance
+            return mu, var
+        else:
+            raise NotImplementedError
+
+    def predictive_variance(self, mu, sigma, **other_shit):
+        if isinstance(noise_index,int):
+            _variance = self.variance[noise_index]
+        else:
+            _variance = np.array([ self.variance[j] for j in noise_index ])[:,None]
+        return _variance + sigma**2
+
+
+    def covariance_matrix(self, Y, Y_metadata):
+        #assert all([isinstance(l, Gaussian) for l in self.likelihoods_list])
+        #ind = Y_metadata['output_index'].flatten()
+        #variance = np.zeros(Y.shape[0])
+        #for lik, j in zip(self.likelihoods_list, range(len(self.likelihoods_list))):
+        #    variance[ind==j] = lik.variance
+        #return np.diag(variance)
+        return np.diag(self.gaussian_variance(Y_metadata).flatten())
+
+
+    def samples(self, gp, Y_metadata):
+        """
+        Returns a set of samples of observations based on a given value of the latent variable.
+
+        :param gp: latent variable
+        """
+        N1, N2 = gp.shape
+        Ysim = np.zeros((N1,N2))
+        ind = Y_metadata['output_index'].flatten()
+        for j in np.unique(ind):
+            flt = ind==j
+            gp_filtered = gp[flt,:]
+            n1 = gp_filtered.shape[0]
+            lik = self.likelihoods_list[j]
+            _ysim = np.array([np.random.normal(lik.gp_link.transf(gpj), scale=np.sqrt(lik.variance), size=1) for gpj in gp_filtered.flatten()])
+            Ysim[flt,:] = _ysim.reshape(n1,N2)
+        return Ysim
+

GPy/likelihoods/poisson.py

@@ -21,7 +21,7 @@ class Poisson(Likelihood):
     """
     def __init__(self, gp_link=None):
         if gp_link is None:
-            gp_link = link_functions.Log_ex_1()
+            gp_link = link_functions.Log()
 
         super(Poisson, self).__init__(gp_link, name='Poisson')
 
@@ -134,7 +134,19 @@ class Poisson(Likelihood):
         d3lik_dlink3 = 2*y/(link_f)**3
         return d3lik_dlink3
 
-    def samples(self, gp):
+    def conditional_mean(self,gp):
+        """
+        The mean of the random variable conditioned on one value of the GP
+        """
+        return self.gp_link.transf(gp)
+
+    def conditional_variance(self,gp):
+        """
+        The variance of the random variable conditioned on one value of the GP
+        """
+        return self.gp_link.transf(gp)
+
+    def samples(self, gp, Y_metadata=None):
         """
         Returns a set of samples of observations based on a given value of the latent variable.
 

GPy/likelihoods/student_t.py

@@ -9,6 +9,7 @@ from scipy import stats, integrate
 from scipy.special import gammaln, gamma
 from likelihood import Likelihood
 from ..core.parameterization import Param
+from ..core.parameterization.transformations import Logexp
 
 class StudentT(Likelihood):
     """
@@ -26,7 +27,7 @@ class StudentT(Likelihood):
 
         super(StudentT, self).__init__(gp_link, name='Student_T')
 
-        self.sigma2 = Param('t_noise', float(sigma2))
+        self.sigma2 = Param('t_noise', float(sigma2), Logexp())
         self.v = Param('deg_free', float(deg_free))
         self.add_parameter(self.sigma2)
         self.add_parameter(self.v)
@@ -37,7 +38,7 @@ class StudentT(Likelihood):
     def parameters_changed(self):
         self.variance = (self.v / float(self.v - 2)) * self.sigma2
 
-    def update_gradients(self, derivatives):
+    def update_gradients(self, grads):
         """
         Pull out the gradients, be careful as the order must match the order
         in which the parameters are added
@@ -244,33 +245,33 @@ class StudentT(Likelihood):
         d2logpdf_dlink2_dv = np.zeros_like(d2logpdf_dlink2_dvar) #FIXME: Not done yet
         return np.hstack((d2logpdf_dlink2_dvar, d2logpdf_dlink2_dv))
 
-    def predictive_variance(self, mu, sigma, predictive_mean=None):
+    def predictive_mean(self, mu, sigma, Y_metadata=None):
         """
-        Compute predictive variance of student_t*normal p(y*|f*)p(f*)
-
-        Need to find what the variance is at the latent points for a student t*normal p(y*|f*)p(f*)
-        (((g((v+1)/2))/(g(v/2)*s*sqrt(v*pi)))*(1+(1/v)*((y-f)/s)^2)^(-(v+1)/2))
-        *((1/(s*sqrt(2*pi)))*exp(-(1/(2*(s^2)))*((y-f)^2)))
+        Compute mean of the prediction
         """
+        return self.gp_link.transf(mu) # only true in link is monotoci, which it is.
 
-        #FIXME: Not correct
-        #We want the variance around test points y which comes from int p(y*|f*)p(f*) df*
-        #Var(y*) = Var(E[y*|f*]) + E[Var(y*|f*)]
-        #Since we are given f* (mu) which is our mean (expected) value of y*|f* then the variance is the variance around this
-        #Which was also given to us as (var)
-        #We also need to know the expected variance of y* around samples f*, this is the variance of the student t distribution
-        #However the variance of the student t distribution is not dependent on f, only on sigma and the degrees of freedom
-        true_var = 1/(1/sigma**2 + 1/self.variance)
+    def predictive_variance(self, mu,variance, predictive_mean=None, Y_metadata=None):
+        if self.deg_free <2.:
+            return np.empty(mu.shape)*np.nan #not defined for small degress fo freedom
+        else:
+            return super(StudentT, self).predictive_variance(mu, variance, predictive_mean, Y_metadata)
 
-        return true_var
+    def conditional_mean(self, gp):
+        return self.gp_link.transf(gp)
 
+<<<<<<< HEAD
     def predictive_mean(self, mu, sigma):
         """
         Compute mean of the prediction
         """
         return mu
+=======
+    def conditional_variance(self, gp):
+        return self.deg_free/(self.deg_free - 2.)
+>>>>>>> a3287c38ea775155df4e90f7fe1883d12ffb54b9
 
-    def samples(self, gp):
+    def samples(self, gp, Y_metadata=None):
         """
         Returns a set of samples of observations based on a given value of the latent variable.
 

GPy/models/__init__.py

@@ -13,6 +13,7 @@ from warped_gp import WarpedGP
 from bayesian_gplvm import BayesianGPLVM
 from mrd import MRD
 from gradient_checker import GradientChecker
-from gp_multioutput_regression import GPMultioutputRegression
-from sparse_gp_multioutput_regression import SparseGPMultioutputRegression
-from ss_gplvm import SSGPLVM
+from ss_gplvm import SSGPLVM
+from gp_coregionalized_regression import GPCoregionalizedRegression
+from sparse_gp_coregionalized_regression import SparseGPCoregionalizedRegression
+#.py file not included!!! #from sparse_gp_coregionalized_regression import SparseGPCoregionalizedRegression

GPy/models/bayesian_gplvm.py

@@ -66,7 +66,7 @@ class BayesianGPLVM(SparseGP):
         super(BayesianGPLVM, self).parameters_changed()
         self._log_marginal_likelihood -= self.variational_prior.KL_divergence(self.X)
 
-        self.X.mean.gradient, self.X.variance.gradient = self.kern.gradients_qX_expectations(variational_posterior=self.X, Z=self.Z, **self.grad_dict)
+        self.X.mean.gradient, self.X.variance.gradient = self.kern.gradients_qX_expectations(variational_posterior=self.X, Z=self.Z, dL_dpsi0=self.grad_dict['dL_dpsi0'], dL_dpsi1=self.grad_dict['dL_dpsi1'], dL_dpsi2=self.grad_dict['dL_dpsi2'])
 
         # update for the KL divergence
         self.variational_prior.update_gradients_KL(self.X)

GPy/models/gp_classification.py

@@ -23,7 +23,7 @@ class GPClassification(GP):
 
     def __init__(self, X, Y, kernel=None):
         if kernel is None:
-            kernel = kern.rbf(X.shape[1])
+            kernel = kern.RBF(X.shape[1])
 
         likelihood = likelihoods.Bernoulli()
 

GPy/models/gp_coregionalized_regression.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2012 - 2014 the GPy Austhors (see AUTHORS.txt)
+# Licensed under the BSD 3-clause license (see LICENSE.txt)
+
+import numpy as np
+from ..core import GP
+from .. import likelihoods
+from .. import kern
+from .. import util
+
+class GPCoregionalizedRegression(GP):
+    """
+    Gaussian Process model for heteroscedastic multioutput regression
+
+    This is a thin wrapper around the models.GP class, with a set of sensible defaults
+
+    :param X_list: list of input observations corresponding to each output
+    :type X_list: list of numpy arrays
+    :param Y_list: list of observed values related to the different noise models
+    :type Y_list: list of numpy arrays
+    :param kernel: a GPy kernel, defaults to RBF ** Coregionalized
+    :type kernel: None | GPy.kernel defaults
+    :likelihoods_list: a list of likelihoods, defaults to list of Gaussian likelihoods
+    :type likelihoods_list: None | a list GPy.likelihoods
+    :param name: model name
+    :type name: string
+    :param W_rank: number tuples of the corregionalization parameters 'W' (see coregionalize kernel documentation)
+    :type W_rank: integer
+    :param kernel_name: name of the kernel
+    :type kernel_name: string
+    """
+    def __init__(self, X_list, Y_list, kernel=None, likelihoods_list=None, name='GPCR',W_rank=1,kernel_name='X'):
+
+        #Input and Output
+        X,Y,self.output_index = util.multioutput.build_XY(X_list,Y_list)
+        Ny = len(Y_list)
+
+        #Kernel
+        if kernel is None:
+            kernel = util.multioutput.ICM(input_dim=X.shape[1]-1, num_outputs=Ny, kernel=GPy.kern.rbf(X.shape[1]-1), W_rank=1,name=kernel_name)
+
+        #Likelihood
+        likelihood = util.multioutput.build_likelihood(Y_list,self.output_index,likelihoods_list)
+
+        super(GPCoregionalizedRegression, self).__init__(X,Y,kernel,likelihood, Y_metadata={'output_index':self.output_index})

GPy/models/gp_regression.py

@@ -20,14 +20,14 @@ class GPRegression(GP):
 
     """
 
-    def __init__(self, X, Y, kernel=None):
+    def __init__(self, X, Y, kernel=None, Y_metadata=None):
 
         if kernel is None:
             kernel = kern.RBF(X.shape[1])
 
         likelihood = likelihoods.Gaussian()
 
-        super(GPRegression, self).__init__(X, Y, kernel, likelihood, name='GP regression')
+        super(GPRegression, self).__init__(X, Y, kernel, likelihood, name='GP regression', Y_metadata=Y_metadata)
 
     def _getstate(self):
         return GP._getstate(self)

GPy/models/gplvm.py

@@ -41,7 +41,7 @@ class GPLVM(GP):
 
     def parameters_changed(self):
         super(GPLVM, self).parameters_changed()
-        self.X.gradient = self.kern.gradients_X(self.dL_dK, self.X, None)
+        self.X.gradient = self.kern.gradients_X(self.grad_dict['dL_dK'], self.X, None)
 
     def _getstate(self):
         return GP._getstate(self)

GPy/models/mrd.py

@@ -5,23 +5,23 @@ import numpy as np
 import itertools
 import pylab
 
-from ..core import Model, SparseGP
+from ..core import Model
 from ..util.linalg import PCA
 from ..kern import Kern
-from bayesian_gplvm import BayesianGPLVM
 from ..core.parameterization.variational import NormalPosterior, NormalPrior
-from ..inference.latent_function_inference.var_dtc import VarDTCMissingData
-from ..likelihoods.gaussian import Gaussian
+from ..core.parameterization import Param, Parameterized
+from ..inference.latent_function_inference.var_dtc import VarDTCMissingData, VarDTC
+from ..likelihoods import Gaussian
 
-class MRD2(Model):
+class MRD(Model):
     """
-    Apply MRD to all given datasets Y in Ylist. 
-    
+    Apply MRD to all given datasets Y in Ylist.
+
     Y_i in [n x p_i]
-    
-    The samples n in the datasets need 
+
+    The samples n in the datasets need
     to match up, whereas the dimensionality p_d can differ.
-    
+
     :param [array-like] Ylist: List of datasets to apply MRD on
     :param input_dim: latent dimensionality
     :type input_dim: int
@@ -43,61 +43,109 @@ class MRD2(Model):
     :param :class:`~GPy.inference.latent_function_inference inference_method: the inference method to use
     :param :class:`~GPy.likelihoods.likelihood.Likelihood` likelihood: the likelihood to use
     :param str name: the name of this model
+    :param [str] Ynames: the names for the datasets given, must be of equal length as Ylist or None
     """
-    
-    def __init__(self, Ylist, input_dim, X=None, X_variance=None, 
+    def __init__(self, Ylist, input_dim, X=None, X_variance=None,
                  initx = 'PCA', initz = 'permute',
-                 num_inducing=10, Z=None, kernel=None, 
-                 inference_method=None, likelihood=None, name='mrd'):
-        super(MRD2, self).__init__(name)
-        
+                 num_inducing=10, Z=None, kernel=None,
+                 inference_method=None, likelihood=None, name='mrd', Ynames=None):
+        super(MRD, self).__init__(name)
+
         # sort out the kernels
         if kernel is None:
             from ..kern import RBF
-            self.kern = [RBF(input_dim, ARD=1, name='Y_{}'.format(i)) for i in range(len(Ylist))]
+            self.kern = [RBF(input_dim, ARD=1, name='rbf'.format(i)) for i in range(len(Ylist))]
         elif isinstance(kernel, Kern):
-            self.kern = [kernel.copy(name='Y_{}'.format(i)) for i in range(len(Ylist))]
+            self.kern = [kernel.copy(name='{}'.format(kernel.name, i)) for i in range(len(Ylist))]
         else:
             assert len(kernel) == len(Ylist), "need one kernel per output"
             assert all([isinstance(k, Kern) for k in kernel]), "invalid kernel object detected!"
-
+            self.kern = kernel
         self.input_dim = input_dim
         self.num_inducing = num_inducing
 
+        self.Ylist = Ylist
         self._in_init_ = True
         X = self._init_X(initx, Ylist)
-        self.Z = self._init_Z(initz, X)
+        self.Z = Param('inducing inputs', self._init_Z(initz, X))
         self.num_inducing = self.Z.shape[0] # ensure M==N if M>N
-        
+
         if X_variance is None:
-            X_variance = np.random.uniform(0,.2,X.shape)
-        
+            X_variance = np.random.uniform(0, .2, X.shape)
+
         self.variational_prior = NormalPrior()
         self.X = NormalPosterior(X, X_variance)
-        
+
         if likelihood is None:
-            likelihood = Gaussian()
-        
+            self.likelihood = [Gaussian(name='Gaussian_noise'.format(i)) for i in range(len(Ylist))]
+        else: self.likelihood = likelihood
+
         if inference_method is None:
-            if any(np.any(np.isnan(y)) for y in Ylist):
-                self.inference_method = VarDTCMissingData(limit=len(Ylist))
-                
-        self.Ylist = Ylist
+            self.inference_method= []
+            for y in Ylist:
+                if np.any(np.isnan(y)):
+                    self.inference_method.append(VarDTCMissingData(limit=1))
+                else:
+                    self.inference_method.append(VarDTC(limit=1))
+        else:
+            self.inference_method = inference_method
+            self.inference_method.set_limit(len(Ylist))
+
+        self.add_parameters(self.X, self.Z)
+
+        if Ynames is None:
+            Ynames = ['Y{}'.format(i) for i in range(len(Ylist))]
+
+        for i, n, k, l in itertools.izip(itertools.count(), Ynames, self.kern, self.likelihood):
+            p = Parameterized(name=n)
+            p.add_parameter(k)
+            p.add_parameter(l)
+            setattr(self, 'Y{}'.format(i), p)
+            self.add_parameter(p)
+        self._in_init_ = False
 
     def parameters_changed(self):
-        for y in self.Ylist:
-            pass
+        self._log_marginal_likelihood = 0
+        self.posteriors = []
+        self.Z.gradient = 0.
+        self.X.mean.gradient = 0.
+        self.X.variance.gradient = 0.
 
-    def _init_X(self, init='PCA', likelihood_list=None):
-        if likelihood_list is None:
-            likelihood_list = self.likelihood_list
-        Ylist = []
-        for likelihood_or_Y in likelihood_list:
-            if type(likelihood_or_Y) is np.ndarray:
-                Ylist.append(likelihood_or_Y)
-            else:
-                Ylist.append(likelihood_or_Y.Y)
-        del likelihood_list
+        for y, k, l, i in itertools.izip(self.Ylist, self.kern, self.likelihood, self.inference_method):
+            posterior, lml, grad_dict = i.inference(k, self.X, self.Z, l, y)
+
+            self.posteriors.append(posterior)
+            self._log_marginal_likelihood += lml
+
+            # likelihood gradients
+            l.update_gradients(grad_dict.pop('dL_dthetaL'))
+
+            #gradients wrt kernel
+            dL_dKmm = grad_dict.pop('dL_dKmm')
+            k.update_gradients_full(dL_dKmm, self.Z, None)
+            target = k.gradient.copy()
+            k.update_gradients_expectations(variational_posterior=self.X, Z=self.Z, **grad_dict)
+            k.gradient += target
+
+            #gradients wrt Z
+            self.Z.gradient += k.gradients_X(dL_dKmm, self.Z)
+            self.Z.gradient += k.gradients_Z_expectations(
+                               grad_dict['dL_dpsi1'], grad_dict['dL_dpsi2'], Z=self.Z, variational_posterior=self.X)
+
+            dL_dmean, dL_dS = k.gradients_qX_expectations(variational_posterior=self.X, Z=self.Z, **grad_dict)
+            self.X.mean.gradient += dL_dmean
+            self.X.variance.gradient += dL_dS
+
+        # update for the KL divergence
+        self.variational_prior.update_gradients_KL(self.X)
+        self._log_marginal_likelihood -= self.variational_prior.KL_divergence(self.X)
+
+    def log_likelihood(self):
+        return self._log_marginal_likelihood
+
+    def _init_X(self, init='PCA', Ylist=None):
+        if Ylist is None:
+            Ylist = self.Ylist
         if init in "PCA_concat":
             X = PCA(np.hstack(Ylist), self.input_dim)[0]
         elif init in "PCA_single":
@@ -106,7 +154,6 @@ class MRD2(Model):
                 X[:, qs] = PCA(Y, len(qs))[0]
         else: # init == 'random':
             X = np.random.randn(Ylist[0].shape[0], self.input_dim)
-        self.X = X
         return X
 
     def _init_Z(self, init="permute", X=None):
@@ -116,259 +163,8 @@ class MRD2(Model):
             Z = np.random.permutation(X.copy())[:self.num_inducing]
         elif init in "random":
             Z = np.random.randn(self.num_inducing, self.input_dim) * X.var()
-        self.Z = Z
         return Z
 
-class MRD(Model):
-    """
-    Do MRD on given Datasets in Ylist.
-    All Ys in likelihood_list are in [N x Dn], where Dn can be different per Yn,
-    N must be shared across datasets though.
-
-    :param likelihood_list: list of observed datasets (:py:class:`~GPy.likelihoods.gaussian.Gaussian` if not supplied directly)
-    :type likelihood_list: [:py:class:`~GPy.likelihoods.likelihood.likelihood` | :py:class:`ndarray`]
-    :param names: names for different gplvm models
-    :type names: [str]
-    :param input_dim: latent dimensionality
-    :type input_dim: int
-    :param initx: initialisation method for the latent space :
-
-        * 'concat' - PCA on concatenation of all datasets
-        * 'single' - Concatenation of PCA on datasets, respectively
-        * 'random' - Random draw from a normal
-
-    :type initx: ['concat'|'single'|'random']
-    :param initz: initialisation method for inducing inputs
-    :type initz: 'permute'|'random'
-    :param X: Initial latent space
-    :param X_variance: Initial latent space variance
-    :param Z: initial inducing inputs
-    :param num_inducing: number of inducing inputs to use
-    :param kernels: list of kernels or kernel shared for all BGPLVMS
-    :type kernels: [GPy.kern.kern] | GPy.kern.kern | None (default)
-
-    """
-    def __init__(self, likelihood_or_Y_list, input_dim, num_inducing=10, names=None,
-                 kernels=None, initx='PCA',
-                 initz='permute', _debug=False, **kw):
-        if names is None:
-            self.names = ["{}".format(i) for i in range(len(likelihood_or_Y_list))]
-
-        # sort out the kernels
-        if kernels is None:
-            kernels = [None] * len(likelihood_or_Y_list)
-        elif isinstance(kernels, Kern):
-            kernels = [kernels.copy() for i in range(len(likelihood_or_Y_list))]
-        else:
-            assert len(kernels) == len(likelihood_or_Y_list), "need one kernel per output"
-            assert all([isinstance(k, Kern) for k in kernels]), "invalid kernel object detected!"
-        assert not ('kernel' in kw), "pass kernels through `kernels` argument"
-
-        self.input_dim = input_dim
-        self._debug = _debug
-        self.num_inducing = num_inducing
-
-        self._in_init_ = True
-        X = self._init_X(initx, likelihood_or_Y_list)
-        Z = self._init_Z(initz, X)
-        self.num_inducing = Z.shape[0] # ensure M==N if M>N
-
-        self.bgplvms = [BayesianGPLVM(l, input_dim=input_dim, kernel=k, X=X, Z=Z, num_inducing=self.num_inducing, **kw) for l, k in zip(likelihood_or_Y_list, kernels)]
-        del self._in_init_
-
-        self.gref = self.bgplvms[0]
-        nparams = np.array([0] + [SparseGP._get_params(g).size - g.Z.size for g in self.bgplvms])
-        self.nparams = nparams.cumsum()
-
-        self.num_data = self.gref.num_data
-
-        self.NQ = self.num_data * self.input_dim
-        self.MQ = self.num_inducing * self.input_dim
-
-        Model.__init__(self)
-        self.ensure_default_constraints()
-
-    def _getstate(self):
-        return Model._getstate(self) + [self.names,
-                self.bgplvms,
-                self.gref,
-                self.nparams,
-                self.input_dim,
-                self.num_inducing,
-                self.num_data,
-                self.NQ,
-                self.MQ]
-
-    def _setstate(self, state):
-        self.MQ = state.pop()
-        self.NQ = state.pop()
-        self.num_data = state.pop()
-        self.num_inducing = state.pop()
-        self.input_dim = state.pop()
-        self.nparams = state.pop()
-        self.gref = state.pop()
-        self.bgplvms = state.pop()
-        self.names = state.pop()
-        Model._setstate(self, state)
-
-    @property
-    def X(self):
-        return self.gref.X
-    @X.setter
-    def X(self, X):
-        try:
-            self.propagate_param(X=X)
-        except AttributeError:
-            if not self._in_init_:
-                raise AttributeError("bgplvm list not initialized")
-    @property
-    def Z(self):
-        return self.gref.Z
-    @Z.setter
-    def Z(self, Z):
-        try:
-            self.propagate_param(Z=Z)
-        except AttributeError:
-            if not self._in_init_:
-                raise AttributeError("bgplvm list not initialized")
-    @property
-    def X_variance(self):
-        return self.gref.X_variance
-    @X_variance.setter
-    def X_variance(self, X_var):
-        try:
-            self.propagate_param(X_variance=X_var)
-        except AttributeError:
-            if not self._in_init_:
-                raise AttributeError("bgplvm list not initialized")
-    @property
-    def likelihood_list(self):
-        return [g.likelihood.Y for g in self.bgplvms]
-    @likelihood_list.setter
-    def likelihood_list(self, likelihood_list):
-        for g, Y in itertools.izip(self.bgplvms, likelihood_list):
-            g.likelihood.Y = Y
-
-    @property
-    def auto_scale_factor(self):
-        """
-        set auto_scale_factor for all gplvms
-        :param b: auto_scale_factor
-        :type b:
-        """
-        return self.gref.auto_scale_factor
-    @auto_scale_factor.setter
-    def auto_scale_factor(self, b):
-        self.propagate_param(auto_scale_factor=b)
-
-    def propagate_param(self, **kwargs):
-        for key, val in kwargs.iteritems():
-            for g in self.bgplvms:
-                g.__setattr__(key, val)
-
-    def randomize(self, initx='concat', initz='permute', *args, **kw):
-        super(MRD, self).randomize(*args, **kw)
-        self._init_X(initx, self.likelihood_list)
-        self._init_Z(initz, self.X)
-
-    #def _get_latent_param_names(self):
-    def _get_param_names(self):
-        n1 = self.gref._get_param_names()
-        n1var = n1[:self.NQ * 2 + self.MQ]
-    #    return n1var
-    #
-    #def _get_kernel_names(self):
-        map_names = lambda ns, name: map(lambda x: "{1}_{0}".format(*x),
-                                         itertools.izip(ns,
-                                                        itertools.repeat(name)))
-        return list(itertools.chain(n1var, *(map_names(\
-                SparseGP._get_param_names(g)[self.MQ:], n) \
-                for g, n in zip(self.bgplvms, self.names))))
-    #    kernel_names = (map_names(SparseGP._get_param_names(g)[self.MQ:], n) for g, n in zip(self.bgplvms, self.names))
-    #    return kernel_names
-
-    #def _get_param_names(self):
-        # X_names = sum([['X_%i_%i' % (n, q) for q in range(self.input_dim)] for n in range(self.num_data)], [])
-        # S_names = sum([['X_variance_%i_%i' % (n, q) for q in range(self.input_dim)] for n in range(self.num_data)], [])
-    #    n1var = self._get_latent_param_names()
-    #    kernel_names = self._get_kernel_names()
-    #    return list(itertools.chain(n1var, *kernel_names))
-
-    #def _get_print_names(self):
-    #    return list(itertools.chain(*self._get_kernel_names()))
-
-    def _get_params(self):
-        """
-        return parameter list containing private and shared parameters as follows:
-
-        =================================================================
-        | mu | S | Z || theta1 | theta2 | .. | thetaN |
-        =================================================================
-        """
-        X = self.gref.X.ravel()
-        X_var = self.gref.X_variance.ravel()
-        Z = self.gref.Z.ravel()
-        thetas = [SparseGP._get_params(g)[g.Z.size:] for g in self.bgplvms]
-        params = np.hstack([X, X_var, Z, np.hstack(thetas)])
-        return params
-
-#     def _set_var_params(self, g, X, X_var, Z):
-#         g.X = X.reshape(self.num_data, self.input_dim)
-#         g.X_variance = X_var.reshape(self.num_data, self.input_dim)
-#         g.Z = Z.reshape(self.num_inducing, self.input_dim)
-#
-#     def _set_kern_params(self, g, p):
-#         g.kern._set_params(p[:g.kern.num_params])
-#         g.likelihood._set_params(p[g.kern.num_params:])
-
-    def _set_params(self, x):
-        start = 0; end = self.NQ
-        X = x[start:end]
-        start = end; end += start
-        X_var = x[start:end]
-        start = end; end += self.MQ
-        Z = x[start:end]
-        thetas = x[end:]
-
-        # set params for all:
-        for g, s, e in itertools.izip(self.bgplvms, self.nparams, self.nparams[1:]):
-            g._set_params(np.hstack([X, X_var, Z, thetas[s:e]]))
-#             self._set_var_params(g, X, X_var, Z)
-#             self._set_kern_params(g, thetas[s:e].copy())
-#             g._compute_kernel_matrices()
-#             if self.auto_scale_factor:
-#                 g.scale_factor = np.sqrt(g.psi2.sum(0).mean() * g.likelihood.precision)
-# #                 self.scale_factor = np.sqrt(self.psi2.sum(0).mean() * self.likelihood.precision)
-#             g._computations()
-
-
-    def update_likelihood_approximation(self): # TODO: object oriented vs script base
-        for bgplvm in self.bgplvms:
-            bgplvm.update_likelihood_approximation()
-
-    def log_likelihood(self):
-        ll = -self.gref.KL_divergence()
-        for g in self.bgplvms:
-            ll += SparseGP.log_likelihood(g)
-        return ll
-
-    def _log_likelihood_gradients(self):
-        dLdmu, dLdS = reduce(lambda a, b: [a[0] + b[0], a[1] + b[1]], (g.dL_dmuS() for g in self.bgplvms))
-        dKLmu, dKLdS = self.gref.dKL_dmuS()
-        dLdmu -= dKLmu
-        dLdS -= dKLdS
-        dLdmuS = np.hstack((dLdmu.flatten(), dLdS.flatten())).flatten()
-        dldzt1 = reduce(lambda a, b: a + b, (SparseGP._log_likelihood_gradients(g)[:self.MQ] for g in self.bgplvms))
-
-        return np.hstack((dLdmuS,
-                             dldzt1,
-                np.hstack([np.hstack([g.dL_dtheta(),
-                                            g.likelihood._gradients(\
-                                                partial=g.partial_for_likelihood)]) \
-                              for g in self.bgplvms])))
-
-
-
     def _handle_plotting(self, fignum, axes, plotf, sharex=False, sharey=False):
         if axes is None:
             fig = pylab.figure(num=fignum)

GPy/models/sparse_gp_coregionalized_regression.py

@@ -0,0 +1,66 @@
+# Copyright (c) 2012 - 2014 the GPy Austhors (see AUTHORS.txt)
+# Licensed under the BSD 3-clause license (see LICENSE.txt)
+
+import numpy as np
+from ..core import SparseGP
+from ..inference.latent_function_inference import VarDTC
+from .. import likelihoods
+from .. import kern
+from .. import util
+
+class SparseGPCoregionalizedRegression(SparseGP):
+    """
+    Sparse Gaussian Process model for heteroscedastic multioutput regression
+
+    This is a thin wrapper around the SparseGP class, with a set of sensible defaults
+
+    :param X_list: list of input observations corresponding to each output
+    :type X_list: list of numpy arrays
+    :param Y_list: list of observed values related to the different noise models
+    :type Y_list: list of numpy arrays
+    :param Z_list: list of inducing inputs (optional)
+    :type Z_list: empty list | list of numpy arrays
+    :param kernel: a GPy kernel, defaults to RBF ** Coregionalized
+    :type kernel: None | GPy.kernel defaults
+    :likelihoods_list: a list of likelihoods, defaults to list of Gaussian likelihoods
+    :type likelihoods_list: None | a list GPy.likelihoods
+    :param num_inducing: number of inducing inputs, defaults to 10 per output (ignored if Z_list is not empty)
+    :type num_inducing: integer | list of integers
+
+    :param name: model name
+    :type name: string
+    :param W_rank: number tuples of the corregionalization parameters 'W' (see coregionalize kernel documentation)
+    :type W_rank: integer
+    :param kernel_name: name of the kernel
+    :type kernel_name: string
+    """
+
+    def __init__(self, X_list, Y_list, Z_list=[], kernel=None, likelihoods_list=None, num_inducing=10, X_variance=None, name='SGPCR',W_rank=1,kernel_name='X'):
+
+        #Input and Output
+        X,Y,self.output_index = util.multioutput.build_XY(X_list,Y_list)
+        Ny = len(Y_list)
+
+        #Kernel
+        if kernel is None:
+            kernel = util.multioutput.ICM(input_dim=X.shape[1]-1, num_outputs=Ny, kernel=GPy.kern.rbf(X.shape[1]-1), W_rank=1,name=kernel_name)
+
+        #Likelihood
+        likelihood = util.multioutput.build_likelihood(Y_list,self.output_index,likelihoods_list)
+
+        #Inducing inputs list
+        if len(Z_list):
+            assert len(Z_list) == self.output_dim, 'Number of outputs do not match length of inducing inputs list.'
+        else:
+            if isinstance(num_inducing,np.int):
+                num_inducing = [num_inducing] * Ny
+            num_inducing = np.asarray(num_inducing)
+            assert num_inducing.size == Ny, 'Number of outputs do not match length of inducing inputs list.'
+            for ni,Xi in zip(num_inducing,X_list):
+                i = np.random.permutation(Xi.shape[0])[:ni]
+                Z_list.append(Xi[i].copy())
+
+        Z, _, Iz = util.multioutput.build_XY(Z_list)
+
+        super(SparseGPCoregionalizedRegression, self).__init__(X, Y, Z, kernel, likelihood, inference_method=VarDTC(), Y_metadata={'output_index':self.output_index})
+        self['.*inducing'][:,-1].fix()

GPy/models/sparse_gp_regression.py

@@ -45,10 +45,10 @@ class SparseGPRegression(SparseGP):
             assert Z.shape[1] == input_dim
 
         likelihood = likelihoods.Gaussian()
-        
+
         if not (X_variance is None):
             X = NormalPosterior(X,X_variance)
-        
+
         SparseGP.__init__(self, X, Y, Z, kernel, likelihood, inference_method=VarDTC())
 
     def _getstate(self):

GPy/models/ss_gplvm.py

@@ -61,7 +61,7 @@ class SSGPLVM(SparseGP):
         super(SSGPLVM, self).parameters_changed()
         self._log_marginal_likelihood -= self.variational_prior.KL_divergence(self.X)
 
-        self.X.mean.gradient, self.X.variance.gradient, self.X.binary_prob.gradient = self.kern.gradients_qX_expectations(variational_posterior=self.X, Z=self.Z, **self.grad_dict)
+        self.X.mean.gradient, self.X.variance.gradient, self.X.binary_prob.gradient = self.kern.gradients_qX_expectations(variational_posterior=self.X, Z=self.Z, dL_dpsi0=self.grad_dict['dL_dpsi0'], dL_dpsi1=self.grad_dict['dL_dpsi1'], dL_dpsi2=self.grad_dict['dL_dpsi2'])
 
         # update for the KL divergence
         self.variational_prior.update_gradients_KL(self.X)

GPy/plotting/matplot_dep/models_plots.py

@@ -6,13 +6,15 @@ import numpy as np
 import Tango
 from base_plots import gpplot, x_frame1D, x_frame2D
 from ...util.misc import param_to_array
+from ...models.gp_coregionalized_regression import GPCoregionalizedRegression
+from ...models.sparse_gp_coregionalized_regression import SparseGPCoregionalizedRegression
 
 
 def plot_fit(model, plot_limits=None, which_data_rows='all',
         which_data_ycols='all', fixed_inputs=[],
         levels=20, samples=0, fignum=None, ax=None, resolution=None,
         plot_raw=False,
-        linecol=Tango.colorsHex['darkBlue'],fillcol=Tango.colorsHex['lightBlue']):
+        linecol=Tango.colorsHex['darkBlue'],fillcol=Tango.colorsHex['lightBlue'], Y_metadata=None):
     """
     Plot the posterior of the GP.
       - In one dimension, the function is plotted with a shaded region identifying two standard deviations.
@@ -56,8 +58,8 @@ def plot_fit(model, plot_limits=None, which_data_rows='all',
     if ax is None:
         fig = pb.figure(num=fignum)
         ax = fig.add_subplot(111)
-    
-    if hasattr(model, 'has_uncertain_inputs') and model.has_uncertain_inputs(): 
+
+    if hasattr(model, 'has_uncertain_inputs') and model.has_uncertain_inputs():
         X = model.X.mean
         X_variance = param_to_array(model.X.variance)
     else:
@@ -68,7 +70,7 @@ def plot_fit(model, plot_limits=None, which_data_rows='all',
     #work out what the inputs are for plotting (1D or 2D)
     fixed_dims = np.array([i for i,v in fixed_inputs])
     free_dims = np.setdiff1d(np.arange(model.input_dim),fixed_dims)
-
+    plots = {}
     #one dimensional plotting
     if len(free_dims) == 1:
 
@@ -84,25 +86,30 @@ def plot_fit(model, plot_limits=None, which_data_rows='all',
             m, v = model._raw_predict(Xgrid)
             lower = m - 2*np.sqrt(v)
             upper = m + 2*np.sqrt(v)
-            Y = Y
         else:
-            m, v, lower, upper = model.predict(Xgrid)
-            Y = Y
+            if isinstance(model,GPCoregionalizedRegression) or isinstance(model,SparseGPCoregionalizedRegression):
+                meta = {'output_index': Xgrid[:,-1:].astype(np.int)}
+            else:
+                meta = None
+            m, v = model.predict(Xgrid, full_cov=False, Y_metadata=meta)
+            lower, upper = model.predict_quantiles(Xgrid, Y_metadata=meta)
+
+
         for d in which_data_ycols:
-            gpplot(Xnew, m[:, d], lower[:, d], upper[:, d], ax=ax, edgecol=linecol, fillcol=fillcol)
-            ax.plot(X[which_data_rows,free_dims], Y[which_data_rows, d], 'kx', mew=1.5)
+            plots['gpplot'] = gpplot(Xnew, m[:, d], lower[:, d], upper[:, d], ax=ax, edgecol=linecol, fillcol=fillcol)
+            plots['dataplot'] = ax.plot(X[which_data_rows,free_dims], Y[which_data_rows, d], 'kx', mew=1.5)
 
         #optionally plot some samples
         if samples: #NOTE not tested with fixed_inputs
             Ysim = model.posterior_samples(Xgrid, samples)
             for yi in Ysim.T:
-                ax.plot(Xnew, yi[:,None], Tango.colorsHex['darkBlue'], linewidth=0.25)
+                plots['posterior_samples'] = ax.plot(Xnew, yi[:,None], Tango.colorsHex['darkBlue'], linewidth=0.25)
                 #ax.plot(Xnew, yi[:,None], marker='x', linestyle='--',color=Tango.colorsHex['darkBlue']) #TODO apply this line for discrete outputs.
 
 
         #add error bars for uncertain (if input uncertainty is being modelled)
         if hasattr(model,"has_uncertain_inputs") and model.has_uncertain_inputs():
-            ax.errorbar(X[which_data_rows, free_dims].flatten(), Y[which_data_rows, which_data_ycols].flatten(),
+            plots['xerrorbar'] = ax.errorbar(X[which_data_rows, free_dims].flatten(), Y[which_data_rows, which_data_ycols].flatten(),
                         xerr=2 * np.sqrt(X_variance[which_data_rows, free_dims].flatten()),
                         ecolor='k', fmt=None, elinewidth=.5, alpha=.5)
 
@@ -118,7 +125,7 @@ def plot_fit(model, plot_limits=None, which_data_rows='all',
             #Zu = model.Z[:,free_dims] * model._Xscale[:,free_dims] + model._Xoffset[:,free_dims]
             Zu = Z[:,free_dims]
             z_height = ax.get_ylim()[0]
-            ax.plot(Zu, np.zeros_like(Zu) + z_height, 'r|', mew=1.5, markersize=12)
+            plots['inducing_inputs'] = ax.plot(Zu, np.zeros_like(Zu) + z_height, 'r|', mew=1.5, markersize=12)
 
 
 
@@ -137,14 +144,12 @@ def plot_fit(model, plot_limits=None, which_data_rows='all',
         #predict on the frame and plot
         if plot_raw:
             m, _ = model._raw_predict(Xgrid)
-            Y = Y
         else:
-            m, _, _, _ = model.predict(Xgrid)
-            Y = Y
+            m, _ = model.predict(Xgrid)
         for d in which_data_ycols:
             m_d = m[:,d].reshape(resolution, resolution).T
-            ax.contour(x, y, m_d, levels, vmin=m.min(), vmax=m.max(), cmap=pb.cm.jet)
-            ax.scatter(X[which_data_rows, free_dims[0]], X[which_data_rows, free_dims[1]], 40, Y[which_data_rows, d], cmap=pb.cm.jet, vmin=m.min(), vmax=m.max(), linewidth=0.)
+            plots['contour'] = ax.contour(x, y, m_d, levels, vmin=m.min(), vmax=m.max(), cmap=pb.cm.jet)
+            plots['dataplot'] = ax.scatter(X[which_data_rows, free_dims[0]], X[which_data_rows, free_dims[1]], 40, Y[which_data_rows, d], cmap=pb.cm.jet, vmin=m.min(), vmax=m.max(), linewidth=0.)
 
         #set the limits of the plot to some sensible values
         ax.set_xlim(xmin[0], xmax[0])
@@ -157,11 +162,11 @@ def plot_fit(model, plot_limits=None, which_data_rows='all',
         if hasattr(model,"Z"):
             #Zu = model.Z[:,free_dims] * model._Xscale[:,free_dims] + model._Xoffset[:,free_dims]
             Zu = Z[:,free_dims]
-            ax.plot(Zu[:,free_dims[0]], Zu[:,free_dims[1]], 'wo')
+            plots['inducing_inputs'] = ax.plot(Zu[:,free_dims[0]], Zu[:,free_dims[1]], 'wo')
 
     else:
         raise NotImplementedError, "Cannot define a frame with more than two input dimensions"
-
+    return plots
 
 def plot_fit_f(model, *args, **kwargs):
     """

GPy/testing/bgplvm_tests.py

@@ -1,85 +0,0 @@
-# Copyright (c) 2012, Nicolo Fusi
-# Licensed under the BSD 3-clause license (see LICENSE.txt)
-
-import unittest
-import numpy as np
-import GPy
-from ..models import BayesianGPLVM
-
-class BGPLVMTests(unittest.TestCase):
-    def test_bias_kern(self):
-        N, num_inducing, input_dim, D = 10, 3, 2, 4
-        X = np.random.rand(N, input_dim)
-        k = GPy.kern.RBF(input_dim) + GPy.kern.White(input_dim, 0.00001)
-        K = k.K(X)
-        Y = np.random.multivariate_normal(np.zeros(N),K,input_dim).T
-        Y -= Y.mean(axis=0)
-        k = GPy.kern.bias(input_dim) + GPy.kern.White(input_dim, 0.00001)
-        m = BayesianGPLVM(Y, input_dim, kernel=k, num_inducing=num_inducing)
-        m.randomize()
-        self.assertTrue(m.checkgrad())
-
-    def test_linear_kern(self):
-        N, num_inducing, input_dim, D = 10, 3, 2, 4
-        X = np.random.rand(N, input_dim)
-        k = GPy.kern.RBF(input_dim) + GPy.kern.White(input_dim, 0.00001)
-        K = k.K(X)
-        Y = np.random.multivariate_normal(np.zeros(N),K,input_dim).T
-        Y -= Y.mean(axis=0)
-        k = GPy.kern.Linear(input_dim) + GPy.kern.White(input_dim, 0.00001)
-        m = BayesianGPLVM(Y, input_dim, kernel=k, num_inducing=num_inducing)
-        m.randomize()
-        self.assertTrue(m.checkgrad())
-
-    def test_rbf_kern(self):
-        N, num_inducing, input_dim, D = 10, 3, 2, 4
-        X = np.random.rand(N, input_dim)
-        k = GPy.kern.RBF(input_dim) + GPy.kern.White(input_dim, 0.00001)
-        K = k.K(X)
-        Y = np.random.multivariate_normal(np.zeros(N),K,input_dim).T
-        Y -= Y.mean(axis=0)
-        k = GPy.kern.RBF(input_dim) + GPy.kern.White(input_dim, 0.00001)
-        m = BayesianGPLVM(Y, input_dim, kernel=k, num_inducing=num_inducing)
-        m.randomize()
-        self.assertTrue(m.checkgrad())
-
-    def test_rbf_bias_kern(self):
-        N, num_inducing, input_dim, D = 10, 3, 2, 4
-        X = np.random.rand(N, input_dim)
-        k = GPy.kern.RBF(input_dim) +  GPy.kern.Bias(input_dim) + GPy.kern.White(input_dim, 0.00001)
-        K = k.K(X)
-        Y = np.random.multivariate_normal(np.zeros(N),K,input_dim).T
-        Y -= Y.mean(axis=0)
-        k = GPy.kern.RBF(input_dim) + GPy.kern.Bias(input_dim) + GPy.kern.White(input_dim, 0.00001)
-        m = BayesianGPLVM(Y, input_dim, kernel=k, num_inducing=num_inducing)
-        m.randomize()
-        self.assertTrue(m.checkgrad())
-
-    def test_rbf_line_kern(self):
-        N, num_inducing, input_dim, D = 10, 3, 2, 4
-        X = np.random.rand(N, input_dim)
-        k = GPy.kern.RBF(input_dim) +  GPy.kern.Linear(input_dim) + GPy.kern.White(input_dim, 0.00001)
-        K = k.K(X)
-        Y = np.random.multivariate_normal(np.zeros(N),K,input_dim).T
-        Y -= Y.mean(axis=0)
-        k = GPy.kern.RBF(input_dim) + GPy.kern.Bias(input_dim) + GPy.kern.White(input_dim, 0.00001)
-        m = BayesianGPLVM(Y, input_dim, kernel=k, num_inducing=num_inducing)
-        m.randomize()
-        self.assertTrue(m.checkgrad())
-
-    def test_linear_bias_kern(self):
-        N, num_inducing, input_dim, D = 30, 5, 4, 30
-        X = np.random.rand(N, input_dim)
-        k = GPy.kern.Linear(input_dim) +  GPy.kern.Bias(input_dim) + GPy.kern.White(input_dim, 0.00001)
-        K = k.K(X)
-        Y = np.random.multivariate_normal(np.zeros(N),K,input_dim).T
-        Y -= Y.mean(axis=0)
-        k = GPy.kern.Linear(input_dim) + GPy.kern.Bias(input_dim) + GPy.kern.White(input_dim, 0.00001)
-        m = BayesianGPLVM(Y, input_dim, kernel=k, num_inducing=num_inducing)
-        m.randomize()
-        self.assertTrue(m.checkgrad())
-
-
-if __name__ == "__main__":
-    print "Running unit tests, please be (very) patient..."
-    unittest.main()

GPy/testing/index_operations_tests.py

@@ -17,24 +17,33 @@ class Test(unittest.TestCase):
         self.param_index.add(one, [3])
         self.param_index.add(two, [0,5])
         self.param_index.add(three, [2,4,7])
+        self.view = ParameterIndexOperationsView(self.param_index, 2, 6)
+
+    def test_clear(self):
+        self.param_index.clear()
+        self.assertDictEqual(self.param_index._properties, {})
 
     def test_remove(self):
         self.param_index.remove(three, np.r_[3:10])
         self.assertListEqual(self.param_index[three].tolist(), [2])
         self.param_index.remove(one, [1])
-        self.assertListEqual(self.param_index[one].tolist(), [3])        
+        self.assertListEqual(self.param_index[one].tolist(), [3])
+        self.assertListEqual(self.param_index.remove('not in there', []).tolist(), [])
+        self.param_index.remove(one, [9])
+        self.assertListEqual(self.param_index[one].tolist(), [3])
+        self.assertListEqual(self.param_index.remove('not in there', [2,3,4]).tolist(), [])
 
     def test_shift_left(self):
-        self.param_index.shift_left(1, 2)
+        self.view.shift_left(0, 2)
         self.assertListEqual(self.param_index[three].tolist(), [2,5])
         self.assertListEqual(self.param_index[two].tolist(), [0,3])
-        self.assertListEqual(self.param_index[one].tolist(), [1])        
+        self.assertListEqual(self.param_index[one].tolist(), [])
 
     def test_shift_right(self):
-        self.param_index.shift_right(5, 2)
+        self.view.shift_right(3, 2)
         self.assertListEqual(self.param_index[three].tolist(), [2,4,9])
         self.assertListEqual(self.param_index[two].tolist(), [0,7])
-        self.assertListEqual(self.param_index[one].tolist(), [3])        
+        self.assertListEqual(self.param_index[one].tolist(), [3])
 
     def test_index_view(self):
         #=======================================================================
@@ -44,17 +53,17 @@ class Test(unittest.TestCase):
         #                   three     three          three
         # view:             [0    1    2    3    4    5    ]
         #=======================================================================
-        view = ParameterIndexOperationsView(self.param_index, 2, 6)
-        self.assertSetEqual(set(view.properties()), set([one, two, three]))
-        for v,p in zip(view.properties_for(np.r_[:6]), self.param_index.properties_for(np.r_[2:2+6])):
+        self.view = ParameterIndexOperationsView(self.param_index, 2, 6)
+        self.assertSetEqual(set(self.view.properties()), set([one, two, three]))
+        for v,p in zip(self.view.properties_for(np.r_[:6]), self.param_index.properties_for(np.r_[2:2+6])):
             self.assertEqual(v, p)
-        self.assertSetEqual(set(view[two]), set([3]))
+        self.assertSetEqual(set(self.view[two]), set([3]))
         self.assertSetEqual(set(self.param_index[two]), set([0, 5]))
-        view.add(two, np.array([0]))
-        self.assertSetEqual(set(view[two]), set([0,3]))
+        self.view.add(two, np.array([0]))
+        self.assertSetEqual(set(self.view[two]), set([0,3]))
         self.assertSetEqual(set(self.param_index[two]), set([0, 2, 5]))
-        view.clear()
-        for v,p in zip(view.properties_for(np.r_[:6]), self.param_index.properties_for(np.r_[2:2+6])):
+        self.view.clear()
+        for v,p in zip(self.view.properties_for(np.r_[:6]), self.param_index.properties_for(np.r_[2:2+6])):
             self.assertEqual(v, p)
             self.assertEqual(v, [])
         param_index = ParameterIndexOperations()
@@ -62,11 +71,17 @@ class Test(unittest.TestCase):
         param_index.add(two, [0,5])
         param_index.add(three, [2,4,7])
         view2 = ParameterIndexOperationsView(param_index, 2, 6)
-        view.update(view2)
+        self.view.update(view2)
         for [i,v],[i2,v2] in zip(sorted(param_index.items()), sorted(self.param_index.items())):
             self.assertEqual(i, i2)
             self.assertTrue(np.all(v == v2))
-        
+
+    def test_misc(self):
+        for k,v in self.param_index.copy()._properties.iteritems():
+            self.assertListEqual(self.param_index[k].tolist(), v.tolist())
+        self.assertEqual(self.param_index.size, 6)
+        self.assertEqual(self.view.size, 5)
+
 if __name__ == "__main__":
     #import sys;sys.argv = ['', 'Test.test_index_view']
     unittest.main()

GPy/testing/kernel_tests.py

@@ -6,7 +6,9 @@ import numpy as np
 import GPy
 import sys
 
-verbose = True
+verbose = 0
+
+
 
 class Kern_check_model(GPy.core.Model):
     """
@@ -31,9 +33,10 @@ class Kern_check_model(GPy.core.Model):
         self.X2 = X2
         self.dL_dK = dL_dK
 
-    def is_positive_definite(self):
+    def is_positive_semi_definite(self):
         v = np.linalg.eig(self.kernel.K(self.X))[0]
-        if any(v<-10*sys.float_info.epsilon):
+        if any(v.real<=-1e-10):
+            print v.real.min()
             return False
         else:
             return True
@@ -87,11 +90,11 @@ class Kern_check_dKdiag_dX(Kern_check_dK_dX):
         return (np.diag(self.dL_dK)*self.kernel.Kdiag(self.X)).sum()
 
     def parameters_changed(self):
-        self.X.gradient =  self.kernel.gradients_X_diag(self.dL_dK, self.X)
+        self.X.gradient =  self.kernel.gradients_X_diag(self.dL_dK.diagonal(), self.X)
 
 
 
-def kern_test(kern, X=None, X2=None, output_ind=None, verbose=False):
+def check_kernel_gradient_functions(kern, X=None, X2=None, output_ind=None, verbose=False, fixed_X_dims=None):
     """
     This function runs on kernels to check the correctness of their
     implementation. It checks that the covariance function is positive definite
@@ -106,18 +109,18 @@ def kern_test(kern, X=None, X2=None, output_ind=None, verbose=False):
 
     """
     pass_checks = True
-    if X==None:
+    if X is None:
         X = np.random.randn(10, kern.input_dim)
         if output_ind is not None:
             X[:, output_ind] = np.random.randint(kern.output_dim, X.shape[0])
-    if X2==None:
+    if X2 is None:
         X2 = np.random.randn(20, kern.input_dim)
         if output_ind is not None:
             X2[:, output_ind] = np.random.randint(kern.output_dim, X2.shape[0])
 
     if verbose:
         print("Checking covariance function is positive definite.")
-    result = Kern_check_model(kern, X=X).is_positive_definite()
+    result = Kern_check_model(kern, X=X).is_positive_semi_definite()
     if result and verbose:
         print("Check passed.")
     if not result:
@@ -161,7 +164,10 @@ def kern_test(kern, X=None, X2=None, output_ind=None, verbose=False):
     if verbose:
         print("Checking gradients of K(X, X) wrt X.")
     try:
-        result = Kern_check_dK_dX(kern, X=X, X2=None).checkgrad(verbose=verbose)
+        testmodel = Kern_check_dK_dX(kern, X=X, X2=None)
+        if fixed_X_dims is not None:
+            testmodel.X[:,fixed_X_dims].fix()
+        result = testmodel.checkgrad(verbose=verbose)
     except NotImplementedError:
         result=True
         if verbose:
@@ -170,14 +176,17 @@ def kern_test(kern, X=None, X2=None, output_ind=None, verbose=False):
         print("Check passed.")
     if not result:
         print("Gradient of K(X, X) wrt X failed for " + kern.name + " covariance function. Gradient values as follows:")
-        Kern_check_dK_dX(kern, X=X, X2=None).checkgrad(verbose=True)
+        testmodel.checkgrad(verbose=True)
         pass_checks = False
         return False
 
     if verbose:
         print("Checking gradients of K(X, X2) wrt X.")
     try:
-        result = Kern_check_dK_dX(kern, X=X, X2=X2).checkgrad(verbose=verbose)
+        testmodel = Kern_check_dK_dX(kern, X=X, X2=X2)
+        if fixed_X_dims is not None:
+            testmodel.X[:,fixed_X_dims].fix()
+        result = testmodel.checkgrad(verbose=verbose)
     except NotImplementedError:
         result=True
         if verbose:
@@ -185,8 +194,8 @@ def kern_test(kern, X=None, X2=None, output_ind=None, verbose=False):
     if result and verbose:
         print("Check passed.")
     if not result:
-        print("Gradient of K(X, X) wrt X failed for " + kern.name + " covariance function. Gradient values as follows:")
-        Kern_check_dK_dX(kern, X=X, X2=X2).checkgrad(verbose=True)
+        print("Gradient of K(X, X2) wrt X failed for " + kern.name + " covariance function. Gradient values as follows:")
+        testmodel.checkgrad(verbose=True)
         pass_checks = False
         return False
 
@@ -210,27 +219,137 @@ def kern_test(kern, X=None, X2=None, output_ind=None, verbose=False):
 
 
 
-class KernelTestsContinuous(unittest.TestCase):
+class KernelGradientTestsContinuous(unittest.TestCase):
     def setUp(self):
-        self.X = np.random.randn(100,2)
-        self.X2 = np.random.randn(110,2)
+        self.N, self.D = 100, 5
+        self.X = np.random.randn(self.N,self.D)
+        self.X2 = np.random.randn(self.N+10,self.D)
 
         continuous_kerns = ['RBF', 'Linear']
         self.kernclasses = [getattr(GPy.kern, s) for s in continuous_kerns]
 
     def test_Matern32(self):
-        k = GPy.kern.Matern32(2)
-        self.assertTrue(kern_test(k, X=self.X, X2=self.X2, verbose=verbose))
+        k = GPy.kern.Matern32(self.D)
+        k.randomize()
+        self.assertTrue(check_kernel_gradient_functions(k, X=self.X, X2=self.X2, verbose=verbose))
+
+    def test_Prod(self):
+        k = GPy.kern.Matern32(2, active_dims=[2,3]) * GPy.kern.RBF(2, active_dims=[0,4]) + GPy.kern.Linear(self.D)
+        k.randomize()
+        self.assertTrue(check_kernel_gradient_functions(k, X=self.X, X2=self.X2, verbose=verbose))
+
+    def test_Add(self):
+        k = GPy.kern.Matern32(2, active_dims=[2,3]) + GPy.kern.RBF(2, active_dims=[0,4]) + GPy.kern.Linear(self.D)
+        k.randomize()
+        self.assertTrue(check_kernel_gradient_functions(k, X=self.X, X2=self.X2, verbose=verbose))
 
     def test_Matern52(self):
-        k = GPy.kern.Matern52(2)
-        self.assertTrue(kern_test(k, X=self.X, X2=self.X2, verbose=verbose))
+        k = GPy.kern.Matern52(self.D)
+        k.randomize()
+        self.assertTrue(check_kernel_gradient_functions(k, X=self.X, X2=self.X2, verbose=verbose))
 
-    #TODO: turn off grad checkingwrt X for indexed kernels liek coregionalize
+    def test_RBF(self):
+        k = GPy.kern.RBF(self.D)
+        k.randomize()
+        self.assertTrue(check_kernel_gradient_functions(k, X=self.X, X2=self.X2, verbose=verbose))
+
+    def test_Linear(self):
+        k = GPy.kern.Linear(self.D)
+        k.randomize()
+        self.assertTrue(check_kernel_gradient_functions(k, X=self.X, X2=self.X2, verbose=verbose))
+
+#TODO: turn off grad checkingwrt X for indexed kernels like coregionalize
+# class KernelGradientTestsContinuous1D(unittest.TestCase):
+#     def setUp(self):
+#         self.N, self.D = 100, 1
+#         self.X = np.random.randn(self.N,self.D)
+#         self.X2 = np.random.randn(self.N+10,self.D)
+#
+#         continuous_kerns = ['RBF', 'Linear']
+#         self.kernclasses = [getattr(GPy.kern, s) for s in continuous_kerns]
+#
+#     def test_PeriodicExponential(self):
+#         k = GPy.kern.PeriodicExponential(self.D)
+#         k.randomize()
+#         self.assertTrue(check_kernel_gradient_functions(k, X=self.X, X2=self.X2, verbose=verbose))
+#
+#     def test_PeriodicMatern32(self):
+#         k = GPy.kern.PeriodicMatern32(self.D)
+#         k.randomize()
+#         self.assertTrue(check_kernel_gradient_functions(k, X=self.X, X2=self.X2, verbose=verbose))
+#
+#     def test_PeriodicMatern52(self):
+#         k = GPy.kern.PeriodicMatern52(self.D)
+#         k.randomize()
+#         self.assertTrue(check_kernel_gradient_functions(k, X=self.X, X2=self.X2, verbose=verbose))
 
 
+class KernelTestsMiscellaneous(unittest.TestCase):
+    def setUp(self):
+        N, D = 100, 10
+        self.X = np.linspace(-np.pi, +np.pi, N)[:,None] * np.ones(D)
+        self.rbf = GPy.kern.RBF(2, active_dims=slice(0,4,2))
+        self.linear = GPy.kern.Linear(2, active_dims=(3,9))
+        self.matern = GPy.kern.Matern32(3, active_dims=np.array([2,4,9]))
+        self.sumkern = self.rbf + self.linear
+        self.sumkern += self.matern
+        self.sumkern.randomize()
 
+    def test_active_dims(self):
+        self.assertEqual(self.sumkern.input_dim, 10)
+        self.assertEqual(self.sumkern.active_dims, slice(0, 10, 1))
+
+    def test_which_parts(self):
+        self.assertTrue(np.allclose(self.sumkern.K(self.X, which_parts=[self.linear, self.matern]), self.linear.K(self.X)+self.matern.K(self.X)))
+        self.assertTrue(np.allclose(self.sumkern.K(self.X, which_parts=[self.linear, self.rbf]), self.linear.K(self.X)+self.rbf.K(self.X)))
+        self.assertTrue(np.allclose(self.sumkern.K(self.X, which_parts=self.sumkern.parts[0]), self.rbf.K(self.X)))
+
+class KernelTestsNonContinuous(unittest.TestCase):
+    def setUp(self):
+        N0 = 3
+        N1 = 9
+        N2 = 4
+        N = N0+N1+N2
+        self.D = 3
+        self.X = np.random.randn(N, self.D+1)
+        indices = np.random.random_integers(0, 2, size=N)
+        self.X[indices==0, -1] = 0
+        self.X[indices==1, -1] = 1
+        self.X[indices==2, -1] = 2
+        #self.X = self.X[self.X[:, -1].argsort(), :]
+        self.X2 = np.random.randn((N0+N1)*2, self.D+1)
+        self.X2[:(N0*2), -1] = 0
+        self.X2[(N0*2):, -1] = 1
+
+    def test_IndependentOutputs(self):
+        k = GPy.kern.RBF(self.D)
+        kern = GPy.kern.IndependentOutputs(k, -1, 'ind_single')
+        self.assertTrue(check_kernel_gradient_functions(kern, X=self.X, X2=self.X2, verbose=verbose, fixed_X_dims=-1))
+        k = [GPy.kern.RBF(1, active_dims=[1], name='rbf1'), GPy.kern.RBF(self.D, name='rbf012'), GPy.kern.RBF(2, active_dims=[0,2], name='rbf02')]
+        kern = GPy.kern.IndependentOutputs(k, -1, name='ind_split')
+        self.assertTrue(check_kernel_gradient_functions(kern, X=self.X, X2=self.X2, verbose=verbose, fixed_X_dims=-1))
 
 if __name__ == "__main__":
     print "Running unit tests, please be (very) patient..."
-    unittest.main()
+    #unittest.main()
+    np.random.seed(0)
+    N0 = 3
+    N1 = 9
+    N2 = 4
+    N = N0+N1+N2
+    D = 3
+    X = np.random.randn(N, D+1)
+    indices = np.random.random_integers(0, 2, size=N)
+    X[indices==0, -1] = 0
+    X[indices==1, -1] = 1
+    X[indices==2, -1] = 2
+    #X = X[X[:, -1].argsort(), :]
+    X2 = np.random.randn((N0+N1)*2, D+1)
+    X2[:(N0*2), -1] = 0
+    X2[(N0*2):, -1] = 1
+    k = [GPy.kern.RBF(1, active_dims=[1], name='rbf1'), GPy.kern.RBF(D, name='rbf012'), GPy.kern.RBF(2, active_dims=[0,2], name='rbf02')]
+    kern = GPy.kern.IndependentOutputs(k, -1, name='ind_split')
+    assert(check_kernel_gradient_functions(kern, X=X, X2=X2, verbose=verbose, fixed_X_dims=-1))
+    k = GPy.kern.RBF(D)
+    kern = GPy.kern.IndependentOutputs(k, -1, 'ind_single')
+    assert(check_kernel_gradient_functions(kern, X=X, X2=X2, verbose=verbose, fixed_X_dims=-1))

GPy/testing/likelihood_tests.py

@@ -1,11 +1,11 @@
 import numpy as np
 import unittest
 import GPy
-from ..models import GradientChecker
+from GPy.models import GradientChecker
 import functools
 import inspect
-from ..likelihoods import link_functions
-from ..core.parameterization import Param
+from GPy.likelihoods import link_functions
+from GPy.core.parameterization import Param
 from functools import partial
 #np.random.seed(300)
 #np.random.seed(7)
@@ -541,7 +541,8 @@ class TestNoiseModels(object):
             #import ipdb; ipdb.set_trace()
             #NOTE this test appears to be stochastic for some likelihoods (student t?)
             # appears to all be working in test mode right now...
-
+        #if isinstance(model, GPy.likelihoods.StudentT):
+        #    import ipdb;ipdb.set_trace()
         assert m.checkgrad(step=step)
 
     ###########
@@ -664,12 +665,11 @@ class LaplaceTests(unittest.TestCase):
             print m1
             print m2
 
-        m2.parameters_changed()
-        #m2._set_params(m1._get_params())
+        m2[:] = m1[:]
 
         #Predict for training points to get posterior mean and variance
-        post_mean, post_var, _, _ = m1.predict(X)
-        post_mean_approx, post_var_approx, _, _ = m2.predict(X)
+        post_mean, post_var = m1.predict(X)
+        post_mean_approx, post_var_approx, = m2.predict(X)
 
         if debug:
             import pylab as pb
@@ -701,8 +701,8 @@ class LaplaceTests(unittest.TestCase):
         np.testing.assert_almost_equal(m1.log_likelihood(), m2.log_likelihood(), decimal=2)
         #Check marginals are the same with random
         m1.randomize()
-        #m2._set_params(m1._get_params())
-        m2.parameters_changed()
+        m2[:] = m1[:]
+
         np.testing.assert_almost_equal(m1.log_likelihood(), m2.log_likelihood(), decimal=2)
 
         #Check they are checkgradding

GPy/testing/mrd_tests.py

@@ -1,32 +0,0 @@
-# Copyright (c) 2013, Max Zwiessele
-# Licensed under the BSD 3-clause license (see LICENSE.txt)
-'''
-Created on 10 Apr 2013
-
-@author: maxz
-'''
-
-import unittest
-import numpy as np
-import GPy
-
-class MRDTests(unittest.TestCase):
-
-    def test_gradients(self):
-        num_m = 3
-        N, num_inducing, input_dim, D = 20, 8, 6, 20
-        X = np.random.rand(N, input_dim)
-
-        k = GPy.kern.linear(input_dim) + GPy.kern.bias(input_dim) + GPy.kern.white(input_dim)
-        K = k.K(X)
-
-        Ylist = [np.random.multivariate_normal(np.zeros(N), K, input_dim).T for _ in range(num_m)]
-        likelihood_list = [GPy.likelihoods.Gaussian(Y) for Y in Ylist]
-
-        m = GPy.models.MRD(likelihood_list, input_dim=input_dim, kernels=k, num_inducing=num_inducing)
-
-        self.assertTrue(m.checkgrad())
-
-if __name__ == "__main__":
-    print "Running unit tests, please be (very) patient..."
-    unittest.main()

GPy/testing/observable_tests.py

@@ -8,7 +8,7 @@ from GPy.core.parameterization.parameterized import Parameterized
 from GPy.core.parameterization.param import Param
 import numpy
 
-# One trigger in init 
+# One trigger in init
 _trigger_start = -1
 
 class ParamTestParent(Parameterized):
@@ -21,11 +21,9 @@ class ParameterizedTest(Parameterized):
     params_changed_count = _trigger_start
     def parameters_changed(self):
         self.params_changed_count += 1
-    def _set_params(self, params, trigger_parent=True):
-        Parameterized._set_params(self, params, trigger_parent=trigger_parent)
 
 class Test(unittest.TestCase):
-    
+
     def setUp(self):
         self.parent = ParamTestParent('test parent')
         self.par = ParameterizedTest('test model')
@@ -41,12 +39,12 @@ class Test(unittest.TestCase):
 
         self.parent.add_parameter(self.par)
         self.parent.add_parameter(self.par2)
-        
+
         self._observer_triggered = None
         self._trigger_count = 0
         self._first = None
         self._second = None
-        
+
     def _trigger(self, which):
         self._observer_triggered = float(which)
         self._trigger_count += 1
@@ -54,18 +52,18 @@ class Test(unittest.TestCase):
             self._second = self._trigger
         else:
             self._first = self._trigger
-    
+
     def _trigger_priority(self, which):
         if self._first is not None:
             self._second = self._trigger_priority
         else:
             self._first = self._trigger_priority
-            
+
     def test_observable(self):
         self.par.add_observer(self, self._trigger, -1)
         self.assertEqual(self.par.params_changed_count, 0, 'no params changed yet')
         self.assertEqual(self.par.params_changed_count, self.parent.parent_changed_count, 'parent should be triggered as often as param')
-        
+
         self.p[0,1] = 3 # trigger observers
         self.assertEqual(self._observer_triggered, 3, 'observer should have triggered')
         self.assertEqual(self._trigger_count, 1, 'observer should have triggered once')
@@ -78,14 +76,14 @@ class Test(unittest.TestCase):
         self.assertEqual(self._trigger_count, 1, 'observer should have triggered once')
         self.assertEqual(self.par.params_changed_count, 2, 'params changed second')
         self.assertEqual(self.par.params_changed_count, self.parent.parent_changed_count, 'parent should be triggered as often as param')
-        
+
         self.par.add_observer(self, self._trigger, -1)
         self.p[2,1] = 4
         self.assertEqual(self._observer_triggered, 4, 'observer should have triggered')
         self.assertEqual(self._trigger_count, 2, 'observer should have triggered once')
         self.assertEqual(self.par.params_changed_count, 3, 'params changed second')
         self.assertEqual(self.par.params_changed_count, self.parent.parent_changed_count, 'parent should be triggered as often as param')
-        
+
         self.par.remove_observer(self, self._trigger)
         self.p[0,1] = 3
         self.assertEqual(self._observer_triggered, 4, 'observer should not have triggered')
@@ -99,7 +97,7 @@ class Test(unittest.TestCase):
         self.par._trigger_params_changed()
         self.assertEqual(self.par.params_changed_count, 1, 'now params changed')
         self.assertEqual(self.parent.parent_changed_count, self.par.params_changed_count)
-        
+
         self.par._param_array_[:] = 2
         self.par._trigger_params_changed()
         self.assertEqual(self.par.params_changed_count, 2, 'now params changed')
@@ -125,13 +123,13 @@ class Test(unittest.TestCase):
 
         self.par.remove_observer(self)
         self._first = self._second = None
-        
+
         self.par.add_observer(self, self._trigger, 1)
         self.par.add_observer(self, self._trigger_priority, 0)
         self.par.notify_observers(0)
         self.assertEqual(self._first, self._trigger, 'priority should be second')
         self.assertEqual(self._second, self._trigger_priority, 'priority should be second')
-        
+
 
 if __name__ == "__main__":
     #import sys;sys.argv = ['', 'Test.testName']

GPy/testing/parameterized_tests.py

@@ -7,8 +7,24 @@ import unittest
 import GPy
 import numpy as np
 from GPy.core.parameterization.parameter_core import HierarchyError
+from GPy.core.parameterization.array_core import ObsAr
 
-class Test(unittest.TestCase):
+class ArrayCoreTest(unittest.TestCase):
+    def setUp(self):
+        self.X = np.random.normal(1,1, size=(100,10))
+        self.obsX = ObsAr(self.X)
+
+    def test_init(self):
+        X = ObsAr(self.X)
+        X2 = ObsAr(X)
+        self.assertIs(X, X2, "no new Observable array, when Observable is given")
+
+    def test_slice(self):
+        t1 = self.X[2:78]
+        t2 = self.obsX[2:78]
+        self.assertListEqual(t1.tolist(), t2.tolist(), "Slicing should be the exact same, as in ndarray")
+
+class ParameterizedTest(unittest.TestCase):
 
     def setUp(self):
         self.rbf = GPy.kern.RBF(1)
@@ -16,94 +32,112 @@ class Test(unittest.TestCase):
         from GPy.core.parameterization import Param
         from GPy.core.parameterization.transformations import Logistic
         self.param = Param('param', np.random.rand(25,2), Logistic(0, 1))
-        
+
         self.test1 = GPy.core.Parameterized("test model")
-        self.test1.add_parameter(self.white)
-        self.test1.add_parameter(self.rbf, 0)
-        self.test1.add_parameter(self.param)
-        
+        self.test1.kern = self.rbf+self.white
+        self.test1.add_parameter(self.test1.kern)
+        self.test1.add_parameter(self.param, 0)
+
         x = np.linspace(-2,6,4)[:,None]
         y = np.sin(x)
         self.testmodel = GPy.models.GPRegression(x,y)
-        
+
     def test_add_parameter(self):
         self.assertEquals(self.rbf._parent_index_, 0)
         self.assertEquals(self.white._parent_index_, 1)
+        self.assertEquals(self.param._parent_index_, 0)
         pass
-    
+
     def test_fixes(self):
         self.white.fix(warning=False)
-        self.test1.remove_parameter(self.test1.param)
+        self.test1.remove_parameter(self.param)
         self.assertTrue(self.test1._has_fixes())
-
         from GPy.core.parameterization.transformations import FIXED, UNFIXED
         self.assertListEqual(self.test1._fixes_.tolist(),[UNFIXED,UNFIXED,FIXED])
-
-        self.test1.add_parameter(self.white, 0)
+        self.test1.kern.add_parameter(self.white, 0)
         self.assertListEqual(self.test1._fixes_.tolist(),[FIXED,UNFIXED,UNFIXED])
-        
+        self.test1.kern.rbf.fix()
+        self.assertListEqual(self.test1._fixes_.tolist(),[FIXED]*3)
+
     def test_remove_parameter(self):
         from GPy.core.parameterization.transformations import FIXED, UNFIXED, __fixed__, Logexp
         self.white.fix()
-        self.test1.remove_parameter(self.white)
+        self.test1.kern.remove_parameter(self.white)
         self.assertIs(self.test1._fixes_,None)
-        
+
         self.assertListEqual(self.white._fixes_.tolist(), [FIXED])
         self.assertEquals(self.white.constraints._offset, 0)
         self.assertIs(self.test1.constraints, self.rbf.constraints._param_index_ops)
-        self.assertIs(self.test1.constraints, self.param.constraints._param_index_ops)        
-        
+        self.assertIs(self.test1.constraints, self.param.constraints._param_index_ops)
+
         self.test1.add_parameter(self.white, 0)
         self.assertIs(self.test1.constraints, self.white.constraints._param_index_ops)
         self.assertIs(self.test1.constraints, self.rbf.constraints._param_index_ops)
-        self.assertIs(self.test1.constraints, self.param.constraints._param_index_ops)        
+        self.assertIs(self.test1.constraints, self.param.constraints._param_index_ops)
         self.assertListEqual(self.test1.constraints[__fixed__].tolist(), [0])
         self.assertIs(self.white._fixes_,None)
         self.assertListEqual(self.test1._fixes_.tolist(),[FIXED] + [UNFIXED] * 52)
-        
+
         self.test1.remove_parameter(self.white)
         self.assertIs(self.test1._fixes_,None)
         self.assertListEqual(self.white._fixes_.tolist(), [FIXED])
         self.assertIs(self.test1.constraints, self.rbf.constraints._param_index_ops)
         self.assertIs(self.test1.constraints, self.param.constraints._param_index_ops)
-        self.assertListEqual(self.test1.constraints[Logexp()].tolist(), [0,1])
-        
+        self.assertListEqual(self.test1.constraints[Logexp()].tolist(), range(self.param.size, self.param.size+self.rbf.size))
+
+    def test_remove_parameter_param_array_grad_array(self):
+        val = self.test1.kern._param_array_.copy()
+        self.test1.kern.remove_parameter(self.white)
+        self.assertListEqual(self.test1.kern._param_array_.tolist(), val[:2].tolist())
+
     def test_add_parameter_already_in_hirarchy(self):
-        self.assertRaises(HierarchyError, self.test1.add_parameter, self.white._parameters_[0])        
-        
+        self.assertRaises(HierarchyError, self.test1.add_parameter, self.white._parameters_[0])
+
     def test_default_constraints(self):
         self.assertIs(self.rbf.variance.constraints._param_index_ops, self.rbf.constraints._param_index_ops)
         self.assertIs(self.test1.constraints, self.rbf.constraints._param_index_ops)
         self.assertListEqual(self.rbf.constraints.indices()[0].tolist(), range(2))
         from GPy.core.parameterization.transformations import Logexp
-        kern = self.rbf+self.white
+        kern = self.test1.kern
+        self.test1.remove_parameter(kern)
         self.assertListEqual(kern.constraints[Logexp()].tolist(), range(3))
 
     def test_constraints(self):
         self.rbf.constrain(GPy.transformations.Square(), False)
-        self.assertListEqual(self.test1.constraints[GPy.transformations.Square()].tolist(), range(2))
-        self.assertListEqual(self.test1.constraints[GPy.transformations.Logexp()].tolist(), [2])
-        
-        self.test1.remove_parameter(self.rbf)
+        self.assertListEqual(self.test1.constraints[GPy.transformations.Square()].tolist(), range(self.param.size, self.param.size+self.rbf.size))
+        self.assertListEqual(self.test1.constraints[GPy.transformations.Logexp()].tolist(), [self.param.size+self.rbf.size])
+
+        self.test1.kern.remove_parameter(self.rbf)
         self.assertListEqual(self.test1.constraints[GPy.transformations.Square()].tolist(), [])
 
     def test_constraints_views(self):
-        self.assertEqual(self.white.constraints._offset, 2)
-        self.assertEqual(self.rbf.constraints._offset, 0)
-        self.assertEqual(self.param.constraints._offset, 3)
+        self.assertEqual(self.white.constraints._offset, self.param.size+self.rbf.size)
+        self.assertEqual(self.rbf.constraints._offset, self.param.size)
+        self.assertEqual(self.param.constraints._offset, 0)
 
     def test_fixing_randomize(self):
-        self.white.fix(warning=False)
-        val = float(self.test1.white.variance)
+        self.white.fix(warning=True)
+        val = float(self.white.variance)
         self.test1.randomize()
         self.assertEqual(val, self.white.variance)
 
+    def test_fixing_randomize_parameter_handling(self):
+        self.rbf.fix(warning=True)
+        val = float(self.rbf.variance)
+        self.test1.kern.randomize()
+        self.assertEqual(val, self.rbf.variance)
+
     def test_fixing_optimize(self):
         self.testmodel.kern.lengthscale.fix()
         val = float(self.testmodel.kern.lengthscale)
         self.testmodel.randomize()
         self.assertEqual(val, self.testmodel.kern.lengthscale)
 
+    def test_printing(self):
+        print self.test1
+        print self.param
+        print self.test1['']
+
 if __name__ == "__main__":
     #import sys;sys.argv = ['', 'Test.test_add_parameter']
     unittest.main()

GPy/testing/prior_tests.py

@@ -15,7 +15,7 @@ class PriorTests(unittest.TestCase):
         X, y = X[:, None], y[:, None]
         m = GPy.models.GPRegression(X, y)
         lognormal = GPy.priors.LogGaussian(1, 2)
-        m.set_prior('rbf', lognormal)
+        m.rbf.set_prior(lognormal)
         m.randomize()
         self.assertTrue(m.checkgrad())
 
@@ -28,7 +28,7 @@ class PriorTests(unittest.TestCase):
         X, y = X[:, None], y[:, None]
         m = GPy.models.GPRegression(X, y)
         Gamma = GPy.priors.Gamma(1, 1)
-        m.set_prior('rbf', Gamma)
+        m.rbf.set_prior(Gamma)
         m.randomize()
         self.assertTrue(m.checkgrad())
 
@@ -41,16 +41,9 @@ class PriorTests(unittest.TestCase):
         X, y = X[:, None], y[:, None]
         m = GPy.models.GPRegression(X, y)
         gaussian = GPy.priors.Gaussian(1, 1)
-        success = False
-
         # setting a Gaussian prior on non-negative parameters
         # should raise an assertionerror.
-        try:
-            m.set_prior('rbf', gaussian)
-        except AssertionError:
-            success = True
-
-        self.assertTrue(success)
+        self.assertRaises(AssertionError, m.rbf.set_prior, gaussian)
 
 
 if __name__ == "__main__":

GPy/testing/psi_stat_expectation_tests.py

@@ -12,6 +12,7 @@ import numpy
 from GPy.kern import RBF
 from GPy.kern import Linear
 from copy import deepcopy
+from GPy.core.parameterization.variational import NormalPosterior
 
 __test__ = lambda: 'deep' in sys.argv
 # np.random.seed(0)
@@ -28,53 +29,21 @@ def ard(p):
 class Test(unittest.TestCase):
     input_dim = 9
     num_inducing = 13
-    N = 300
+    N = 1000
     Nsamples = 1e6
 
     def setUp(self):
-        i_s_dim_list = [2,4,3]
-        indices = numpy.cumsum(i_s_dim_list).tolist()
-        input_slices = [slice(a,b) for a,b in zip([None]+indices, indices)]
-        #input_slices[2] = deepcopy(input_slices[1])
-        input_slice_kern = GPy.kern.kern(9,
-                                         [
-                                          RBF(i_s_dim_list[0], np.random.rand(), np.random.rand(i_s_dim_list[0]), ARD=True),
-                                          RBF(i_s_dim_list[1], np.random.rand(), np.random.rand(i_s_dim_list[1]), ARD=True),
-                                          Linear(i_s_dim_list[2], np.random.rand(i_s_dim_list[2]), ARD=True)
-                                          ],
-                                         input_slices = input_slices
-                                         )
         self.kerns = (
-#                     input_slice_kern,
-#                       (GPy.kern.rbf(self.input_dim, ARD=True) +
-#                        GPy.kern.linear(self.input_dim, ARD=True) +
-#                        GPy.kern.bias(self.input_dim) +
-#                        GPy.kern.white(self.input_dim)),
-                    (#GPy.kern.rbf(self.input_dim, np.random.rand(), np.random.rand(self.input_dim), ARD=True)
-                     GPy.kern.Linear(self.input_dim, np.random.rand(self.input_dim), ARD=True)
-                     +GPy.kern.RBF(self.input_dim, np.random.rand(), np.random.rand(self.input_dim), ARD=True)
-#                      +GPy.kern.bias(self.input_dim)
-#                      +GPy.kern.white(self.input_dim)),
-                    ),
-#                     (GPy.kern.rbf(self.input_dim, np.random.rand(), np.random.rand(self.input_dim), ARD=True) +
-#                      GPy.kern.bias(self.input_dim, np.random.rand())),
-#         (GPy.kern.rbf(self.input_dim, np.random.rand(), np.random.rand(self.input_dim), ARD=True)
-#          +GPy.kern.rbf(self.input_dim, np.random.rand(), np.random.rand(self.input_dim), ARD=True)
-#          #+GPy.kern.bias(self.input_dim, np.random.rand())
-#          #+GPy.kern.white(self.input_dim, np.random.rand())),
-#         ),
-#                     GPy.kern.white(self.input_dim, np.random.rand())),
-#                     GPy.kern.rbf(self.input_dim), GPy.kern.rbf(self.input_dim, ARD=True),
-#                       GPy.kern.linear(self.input_dim, ARD=False), GPy.kern.linear(self.input_dim, ARD=True),
-#                       GPy.kern.linear(self.input_dim) + GPy.kern.bias(self.input_dim),
-#                     GPy.kern.rbf(self.input_dim) + GPy.kern.bias(self.input_dim),
-#                       GPy.kern.linear(self.input_dim) + GPy.kern.bias(self.input_dim) + GPy.kern.white(self.input_dim),
-#                       GPy.kern.rbf(self.input_dim) + GPy.kern.bias(self.input_dim) + GPy.kern.white(self.input_dim),
-#                       GPy.kern.bias(self.input_dim), GPy.kern.white(self.input_dim),
+                      #GPy.kern.RBF([0,1,2], ARD=True)+GPy.kern.Bias(self.input_dim)+GPy.kern.White(self.input_dim),
+                      #GPy.kern.RBF(self.input_dim)+GPy.kern.Bias(self.input_dim)+GPy.kern.White(self.input_dim),
+                      #GPy.kern.Linear(self.input_dim) + GPy.kern.Bias(self.input_dim) + GPy.kern.White(self.input_dim),
+                      #GPy.kern.Linear(self.input_dim, ARD=True) + GPy.kern.Bias(self.input_dim) + GPy.kern.White(self.input_dim),
+                      GPy.kern.Linear([1,3,6,7], ARD=True) + GPy.kern.RBF([0,5,8], ARD=True) + GPy.kern.White(self.input_dim),
                       )
-        self.q_x_mean = np.random.randn(self.input_dim)
-        self.q_x_variance = np.exp(np.random.randn(self.input_dim))
+        self.q_x_mean = np.random.randn(self.input_dim)[None]
+        self.q_x_variance = np.exp(.5*np.random.randn(self.input_dim))[None]
         self.q_x_samples = np.random.randn(self.Nsamples, self.input_dim) * np.sqrt(self.q_x_variance) + self.q_x_mean
+        self.q_x = NormalPosterior(self.q_x_mean, self.q_x_variance)
         self.Z = np.random.randn(self.num_inducing, self.input_dim)
         self.q_x_mean.shape = (1, self.input_dim)
         self.q_x_variance.shape = (1, self.input_dim)
@@ -114,8 +83,9 @@ class Test(unittest.TestCase):
 
     def test_psi2(self):
         for kern in self.kerns:
+            kern.randomize()
             Nsamples = int(np.floor(self.Nsamples/self.N))
-            psi2 = kern.psi2(self.Z, self.q_x_mean, self.q_x_variance)
+            psi2 = kern.psi2(self.Z, self.q_x)
             K_ = np.zeros((self.num_inducing, self.num_inducing))
             diffs = []
             for i, q_x_sample_stripe in enumerate(np.array_split(self.q_x_samples, self.Nsamples / Nsamples)):
@@ -130,8 +100,8 @@ class Test(unittest.TestCase):
                 pylab.figure(msg)
                 pylab.plot(diffs, marker='x', mew=.2)
 #                 print msg, np.allclose(psi2.squeeze(), K_, rtol=1e-1, atol=.1)
-                self.assertTrue(np.allclose(psi2.squeeze(), K_),
-                                            #rtol=1e-1, atol=.1),
+                self.assertTrue(np.allclose(psi2.squeeze(), K_,
+                                            atol=.1, rtol=1),
                                 msg=msg + ": not matching")
 #                 sys.stdout.write(".")
             except:

GPy/testing/psi_stat_gradient_tests.py

@@ -11,6 +11,7 @@ import itertools
 from GPy.core import Model
 from GPy.core.parameterization.param import Param
 from GPy.core.parameterization.transformations import Logexp
+from GPy.core.parameterization.variational import NormalPosterior
 
 class PsiStatModel(Model):
     def __init__(self, which, X, X_variance, Z, num_inducing, kernel):
@@ -18,23 +19,24 @@ class PsiStatModel(Model):
         self.which = which
         self.X = Param("X", X)
         self.X_variance = Param('X_variance', X_variance, Logexp())
+        self.q = NormalPosterior(self.X, self.X_variance)
         self.Z = Param("Z", Z)
         self.N, self.input_dim = X.shape
         self.num_inducing, input_dim = Z.shape
         assert self.input_dim == input_dim, "shape missmatch: Z:{!s} X:{!s}".format(Z.shape, X.shape)
         self.kern = kernel
-        self.psi_ = self.kern.__getattribute__(self.which)(self.Z, self.X, self.X_variance)
-        self.add_parameters(self.X, self.X_variance, self.Z, self.kern)
+        self.psi_ = self.kern.__getattribute__(self.which)(self.Z, self.q)
+        self.add_parameters(self.q, self.Z, self.kern)
 
     def log_likelihood(self):
         return self.kern.__getattribute__(self.which)(self.Z, self.X, self.X_variance).sum()
 
     def parameters_changed(self):
-        psimu, psiS = self.kern.__getattribute__("d" + self.which + "_dmuS")(numpy.ones_like(self.psi_), self.Z, self.X, self.X_variance)
+        psimu, psiS = self.kern.__getattribute__("d" + self.which + "_dmuS")(numpy.ones_like(self.psi_), self.Z, self.q)
         self.X.gradient = psimu
         self.X_variance.gradient = psiS
         #psimu, psiS = numpy.ones(self.N * self.input_dim), numpy.ones(self.N * self.input_dim)
-        try: psiZ = self.kern.__getattribute__("d" + self.which + "_dZ")(numpy.ones_like(self.psi_), self.Z, self.X, self.X_variance)
+        try: psiZ = self.kern.__getattribute__("d" + self.which + "_dZ")(numpy.ones_like(self.psi_), self.Z, self.q)
         except AttributeError: psiZ = numpy.zeros_like(self.Z)
         self.Z.gradient = psiZ
         #psiZ = numpy.ones(self.num_inducing * self.input_dim)
@@ -176,6 +178,6 @@ if __name__ == "__main__":
             +GPy.kern.White(input_dim)
             )
             )
-        m2.ensure_default_constraints()
+        #m2.ensure_default_constraints()
     else:
         unittest.main()

GPy/testing/unit_tests.py

@@ -34,7 +34,7 @@ class GradientTests(unittest.TestCase):
         model_fit = getattr(GPy.models, model_type)
 
         # noise = GPy.kern.White(dimension)
-        kern = kern #  + noise
+        kern = kern  #  + noise
         if uncertain_inputs:
             m = model_fit(X, Y, kernel=kern, X_variance=np.random.rand(X.shape[0], X.shape[1]))
         else:
@@ -60,13 +60,14 @@ class GradientTests(unittest.TestCase):
 
     def test_GPRegression_mlp_1d(self):
         ''' Testing the GP regression with mlp kernel with white kernel on 1d data '''
-        mlp = GPy.kern.mlp(1)
+        mlp = GPy.kern.MLP(1)
         self.check_model(mlp, model_type='GPRegression', dimension=1)
 
-    def test_GPRegression_poly_1d(self):
-        ''' Testing the GP regression with polynomial kernel with white kernel on 1d data '''
-        mlp = GPy.kern.Poly(1, degree=5)
-        self.check_model(mlp, model_type='GPRegression', dimension=1)
+    #TODO:
+    #def test_GPRegression_poly_1d(self):
+    #    ''' Testing the GP regression with polynomial kernel with white kernel on 1d data '''
+    #    mlp = GPy.kern.Poly(1, degree=5)
+    #    self.check_model(mlp, model_type='GPRegression', dimension=1)
 
     def test_GPRegression_matern52_1D(self):
         ''' Testing the GP regression with matern52 kernel on 1d data '''
@@ -163,14 +164,14 @@ class GradientTests(unittest.TestCase):
         rbflin = GPy.kern.RBF(2) + GPy.kern.Linear(2)
         self.check_model(rbflin, model_type='SparseGPRegression', dimension=2)
 
-    #@unittest.expectedFailure
+    # @unittest.expectedFailure
     def test_SparseGPRegression_rbf_linear_white_kern_2D_uncertain_inputs(self):
         ''' Testing the sparse GP regression with rbf, linear kernel on 2d data with uncertain inputs'''
         rbflin = GPy.kern.RBF(2) + GPy.kern.Linear(2)
         raise unittest.SkipTest("This is not implemented yet!")
         self.check_model(rbflin, model_type='SparseGPRegression', dimension=2, uncertain_inputs=1)
 
-    #@unittest.expectedFailure
+    # @unittest.expectedFailure
     def test_SparseGPRegression_rbf_linear_white_kern_1D_uncertain_inputs(self):
         ''' Testing the sparse GP regression with rbf, linear kernel on 1d data with uncertain inputs'''
         rbflin = GPy.kern.RBF(1) + GPy.kern.Linear(1)
@@ -202,7 +203,7 @@ class GradientTests(unittest.TestCase):
         X = np.hstack([np.random.normal(5, 2, N / 2), np.random.normal(10, 2, N / 2)])[:, None]
         Y = np.hstack([np.ones(N / 2), np.zeros(N / 2)])[:, None]
         kernel = GPy.kern.RBF(1)
-        m = GPy.models.GPClassification(X,Y,kernel=kernel)
+        m = GPy.models.GPClassification(X, Y, kernel=kernel)
         m.update_likelihood_approximation()
         self.assertTrue(m.checkgrad())
 
@@ -212,11 +213,11 @@ class GradientTests(unittest.TestCase):
         Y = np.hstack([np.ones(N / 2), np.zeros(N / 2)])[:, None]
         Z = np.linspace(0, 15, 4)[:, None]
         kernel = GPy.kern.RBF(1)
-        m = GPy.models.SparseGPClassification(X,Y,kernel=kernel,Z=Z)
-        #distribution = GPy.likelihoods.likelihood_functions.Bernoulli()
-        #likelihood = GPy.likelihoods.EP(Y, distribution)
-        #m = GPy.core.SparseGP(X, likelihood, kernel, Z)
-        #m.ensure_default_constraints()
+        m = GPy.models.SparseGPClassification(X, Y, kernel=kernel, Z=Z)
+        # distribution = GPy.likelihoods.likelihood_functions.Bernoulli()
+        # likelihood = GPy.likelihoods.EP(Y, distribution)
+        # m = GPy.core.SparseGP(X, likelihood, kernel, Z)
+        # m.ensure_default_constraints()
         m.update_likelihood_approximation()
         self.assertTrue(m.checkgrad())
 
@@ -224,8 +225,8 @@ class GradientTests(unittest.TestCase):
         N = 20
         X = np.hstack([np.random.rand(N / 2) + 1, np.random.rand(N / 2) - 1])[:, None]
         k = GPy.kern.RBF(1) + GPy.kern.White(1)
-        Y = np.hstack([np.ones(N/2),np.zeros(N/2)])[:,None]
-        m = GPy.models.FITCClassification(X, Y, kernel = k)
+        Y = np.hstack([np.ones(N / 2), np.zeros(N / 2)])[:, None]
+        m = GPy.models.FITCClassification(X, Y, kernel=k)
         m.update_likelihood_approximation()
         self.assertTrue(m.checkgrad())
 
@@ -238,7 +239,7 @@ class GradientTests(unittest.TestCase):
         Y = np.vstack((Y1, Y2))
 
         k1 = GPy.kern.RBF(1)
-        m = GPy.models.GPMultioutputRegression(X_list=[X1,X2],Y_list=[Y1,Y2],kernel_list=[k1])
+        m = GPy.models.GPMultioutputRegression(X_list=[X1, X2], Y_list=[Y1, Y2], kernel_list=[k1])
         m.constrain_fixed('.*rbf_var', 1.)
         self.assertTrue(m.checkgrad())
 
@@ -251,7 +252,7 @@ class GradientTests(unittest.TestCase):
         Y = np.vstack((Y1, Y2))
 
         k1 = GPy.kern.RBF(1)
-        m = GPy.models.SparseGPMultioutputRegression(X_list=[X1,X2],Y_list=[Y1,Y2],kernel_list=[k1])
+        m = GPy.models.SparseGPMultioutputRegression(X_list=[X1, X2], Y_list=[Y1, Y2], kernel_list=[k1])
         m.constrain_fixed('.*rbf_var', 1.)
         self.assertTrue(m.checkgrad())
 

GPy/util/__init__.py

@@ -14,6 +14,7 @@ import subarray_and_sorting
 import caching
 import diag
 import initialization
+import multioutput
 
 try:
     import sympy

GPy/util/caching.py

@@ -9,24 +9,27 @@ class Cacher(object):
 
     """
 
-    def __init__(self, operation, limit=5, ignore_args=()):
+    def __init__(self, operation, limit=5, ignore_args=(), force_kwargs=()):
         self.limit = int(limit)
         self.ignore_args = ignore_args
+        self.force_kwargs = force_kwargs
         self.operation=operation
         self.cached_inputs = []
         self.cached_outputs = []
         self.inputs_changed = []
 
-    def __call__(self, *args):
+    def __call__(self, *args, **kw):
         """
         A wrapper function for self.operation,
         """
 
         #ensure that specified arguments are ignored
+        items = sorted(kw.items(), key=lambda x: x[0])
+        oa_all = args + tuple(a for _,a in items)
         if len(self.ignore_args) != 0:
-            oa = [a for i,a in enumerate(args) if i not in self.ignore_args]
+            oa = [a for i,a in itertools.chain(enumerate(args), items) if i not in self.ignore_args and i not in self.force_kwargs]
         else:
-            oa = args
+            oa = oa_all
 
         # this makes sure we only add an observer once, and that None can be in args
         observable_args = []
@@ -37,36 +40,45 @@ class Cacher(object):
         #make sure that all the found argument really are observable:
         #otherswise don't cache anything, pass args straight though
         if not all([isinstance(arg, Observable) for arg in observable_args]):
-            return self.operation(*args)
+            return self.operation(*args, **kw)
 
+        if len(self.force_kwargs) != 0:
+            # check if there are force args, which force reloading
+            for k in self.force_kwargs:
+                if k in kw and kw[k] is not None:
+                    return self.operation(*args, **kw)
         # TODO: WARNING !!! Cache OFFSWITCH !!! WARNING
-        # return self.operation(*args)
+        #return self.operation(*args)
 
         #if the result is cached, return the cached computation
         state = [all(a is b for a, b in itertools.izip_longest(args, cached_i)) for cached_i in self.cached_inputs]
-        if any(state):
-            i = state.index(True)
-            if self.inputs_changed[i]:
-                #(elements of) the args have changed since we last computed: update
-                self.cached_outputs[i] = self.operation(*args)
-                self.inputs_changed[i] = False
-            return self.cached_outputs[i]
-        else:
-            #first time we've seen these arguments: compute
+        try:
+            if any(state):
+                i = state.index(True)
+                if self.inputs_changed[i]:
+                    #(elements of) the args have changed since we last computed: update
+                    self.cached_outputs[i] = self.operation(*args, **kw)
+                    self.inputs_changed[i] = False
+                return self.cached_outputs[i]
+            else:
+                #first time we've seen these arguments: compute
 
-            #first make sure the depth limit isn't exceeded
-            if len(self.cached_inputs) == self.limit:
-                args_ = self.cached_inputs.pop(0)
-                [a.remove_observer(self, self.on_cache_changed) for a in args_ if a is not None]
-                self.inputs_changed.pop(0)
-                self.cached_outputs.pop(0)
-
-            #compute
-            self.cached_inputs.append(args)
-            self.cached_outputs.append(self.operation(*args))
-            self.inputs_changed.append(False)
-            [a.add_observer(self, self.on_cache_changed) for a in observable_args]
-            return self.cached_outputs[-1]#Max says return.
+                #first make sure the depth limit isn't exceeded
+                if len(self.cached_inputs) == self.limit:
+                    args_ = self.cached_inputs.pop(0)
+                    [a.remove_observer(self, self.on_cache_changed) for a in args_ if a is not None]
+                    self.inputs_changed.pop(0)
+                    self.cached_outputs.pop(0)
+                #compute
+                self.cached_inputs.append(oa_all)
+                self.cached_outputs.append(self.operation(*args, **kw))
+                self.inputs_changed.append(False)
+                [a.add_observer(self, self.on_cache_changed) for a in observable_args]
+                return self.cached_outputs[-1]#return
+        except:
+            raise
+        finally:
+            self.reset()
 
     def on_cache_changed(self, arg):
         """
@@ -76,7 +88,7 @@ class Cacher(object):
         """
         self.inputs_changed = [any([a is arg for a in args]) or old_ic for args, old_ic in zip(self.cached_inputs, self.inputs_changed)]
 
-    def reset(self, obj):
+    def reset(self):
         """
         Totally reset the cache
         """
@@ -90,15 +102,16 @@ class Cache_this(object):
     """
     A decorator which can be applied to bound methods in order to cache them
     """
-    def __init__(self, limit=5, ignore_args=()):
+    def __init__(self, limit=5, ignore_args=(), force_kwargs=()):
         self.limit = limit
         self.ignore_args = ignore_args
+        self.force_args = force_kwargs
         self.c = None
     def __call__(self, f):
-        def f_wrap(*args):
+        def f_wrap(*args, **kw):
             if self.c is None:
-                self.c = Cacher(f, self.limit, ignore_args=self.ignore_args)
-            return self.c(*args)
+                self.c = Cacher(f, self.limit, ignore_args=self.ignore_args, force_kwargs=self.force_args)
+            return self.c(*args, **kw)
         f_wrap._cacher = self
-        f_wrap.__doc__ = "**cached**\n\n" + (f.__doc__ or "")
+        f_wrap.__doc__ = "**cached**" + (f.__doc__ or "")
         return f_wrap

GPy/util/data_resources.json

@@ -32,6 +32,33 @@
       "details":"Artificially generated data of silhouettes given poses. Note that the data does not display a left/right ambiguity because across the entire data set one of the arms sticks out more the the other, disambiguating the pose as to which way the individual is facing.",
       "size":1
    },
+   "football_data":{
+      "files":[
+         [
+	     "E0.csv", "E1.csv", "E2.csv", "E3.csv"
+         ]
+      ],
+      "citation":"",
+      "license":null,
+      "urls":[
+          "http://www.football-data.co.uk/mmz4281/"
+      ],
+      "details":"Results of English football matches since 1993/94 season.",
+      "size":1
+   },
+   "google_trends":{
+      "files":[
+         [
+         ]
+      ],
+      "citation":"",
+      "license":null,
+      "urls":[
+          "http://www.google.com/trends/"
+      ],
+      "details":"Google trends results.",
+      "size":0
+   },
    "osu_accad":{
       "files":[
          [

GPy/util/datasets.py

@@ -1,5 +1,8 @@
+import csv
 import os
+import copy
 import numpy as np
+import pylab as pb
 import GPy
 import scipy.io
 import cPickle as pickle
@@ -7,6 +10,8 @@ import zipfile
 import tarfile
 import datetime
 import json
+import re
+
 ipython_available=True
 try:
     import IPython
@@ -32,11 +37,18 @@ neil_url = 'http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/'
 # Read data resources from json file.
 # Don't do this when ReadTheDocs is scanning as it breaks things
 on_rtd = os.environ.get('READTHEDOCS', None) == 'True' #Checks if RTD is scanning
+
 if not (on_rtd):
     path = os.path.join(os.path.dirname(__file__), 'data_resources.json')
     json_data=open(path).read()
     data_resources = json.loads(json_data)
 
+if not (on_rtd):
+    path = os.path.join(os.path.dirname(__file__), 'football_teams.json')
+    json_data=open(path).read()
+    football_dict = json.loads(json_data)
+
+    
 
 def prompt_user(prompt):
     """Ask user for agreeing to data set licenses."""
@@ -274,8 +286,76 @@ def della_gatta_TRP63_gene_expression(data_set='della_gatta', gene_number=None):
             Y = Y[:, None]
     return data_details_return({'X': X, 'Y': Y, 'gene_number' : gene_number}, data_set)
 
+    
 
+def football_data(season='1314', data_set='football_data'):
+    """Football data from English games since 1993. This downloads data from football-data.co.uk for the given season. """
+    def league2num(string):
+        league_dict = {'E0':0, 'E1':1, 'E2': 2, 'E3': 3, 'EC':4}
+        return league_dict[string]
 
+    def football2num(string):
+        if football_dict.has_key(string):
+            return football_dict[string]
+        else:
+            football_dict[string] = len(football_dict)+1
+            return len(football_dict)+1
+
+    data_set_season = data_set + '_' + season
+    data_resources[data_set_season] = copy.deepcopy(data_resources[data_set])
+    data_resources[data_set_season]['urls'][0]+=season + '/'
+    start_year = int(year[0:2])
+    end_year = int(year[2:4])
+    files = ['E0.csv', 'E1.csv', 'E2.csv', 'E3.csv']
+    if start_year>4 and start_year < 93:
+        files += ['EC.csv']
+    data_resources[data_set_season]['files'] = [files]
+    if not data_available(data_set_season):
+        download_data(data_set_season)
+    for file in reversed(files):
+        filename = os.path.join(data_path, data_set_season, file)
+        # rewrite files removing blank rows.
+        writename = os.path.join(data_path, data_set_season, 'temp.csv')
+        input = open(filename, 'rb')
+        output = open(writename, 'wb')
+        writer = csv.writer(output)
+        for row in csv.reader(input):
+            if any(field.strip() for field in row):
+                writer.writerow(row)
+        input.close()
+        output.close()
+        table = np.loadtxt(writename,skiprows=1, usecols=(0, 1, 2, 3, 4, 5), converters = {0: league2num, 1: pb.datestr2num, 2:football2num, 3:football2num}, delimiter=',')
+        X = table[:, :4]
+        Y = table[:, 4:]
+    return data_details_return({'X': X, 'Y': Y}, data_set)
+
+# This will be for downloading google trends data.
+def google_trends(query_terms=['big data', 'machine learning', 'data science'], data_set='google_trends'):
+    """Data downloaded from Google trends for given query terms."""
+    # Inspired by this notebook:
+    # http://nbviewer.ipython.org/github/sahuguet/notebooks/blob/master/GoogleTrends%20meet%20Notebook.ipynb
+
+    # quote the query terms.
+    for i, element in enumerate(query_terms):
+        query_terms[i] = urllib2.quote(element)
+    query = 'http://www.google.com/trends/fetchComponent?q=%s&cid=TIMESERIES_GRAPH_0&export=3' % ",".join(query_terms)
+
+    data = urllib2.urlopen(query).read()
+
+    # In the notebook they did some data cleaning: remove Javascript header+footer, and translate new Date(....,..,..) into YYYY-MM-DD.
+    header = """// Data table response\ngoogle.visualization.Query.setResponse("""
+    data = data[len(header):-2]
+    data = re.sub('new Date\((\d+),(\d+),(\d+)\)', (lambda m: '"%s-%02d-%02d"' % (m.group(1).strip(), 1+int(m.group(2)), int(m.group(3)))), data)
+    timeseries = json.loads(data)
+    #import pandas as pd
+    columns = [k['label'] for k in timeseries['table']['cols']]
+    rows = map(lambda x: [k['v'] for k in x['c']], timeseries['table']['rows'])
+    terms = len(columns)-1
+    X = np.asarray([(pb.datestr2num(row[0]), i) for i in range(terms) for row in rows ])
+    Y = np.asarray([[row[i+1]] for i in range(terms) for row in rows ])
+    output_info = columns[1:]
+    return data_details_return({'X': X, 'Y': Y, 'query_terms': output_info, 'info': "Data downloaded from google trends with query terms: " + ', '.join(output_info) + '.'}, data_set)
+    
 # The data sets
 def oil(data_set='three_phase_oil_flow'):
     """The three phase oil data from Bishop and James (1993)."""

GPy/util/football_teams.json

@@ -0,0 +1 @@
+{"Canvey Island": 94, "Crewe": 21, "Fleetwood Town": 134, "Wrexham": 89, "Barnet": 69, "Ipswich": 29, "Rochdale": 84, "Bristol Rvs": 70, "Liverpool": 10, "Chelsea": 20, "York": 113, "Newcastle": 18, "QPR": 28, "Middlesboro": 116, "Tranmere": 68, "Bury": 72, "Luton": 24, "AFC Wimbledon": 126, "West Ham": 15, "Braintree Town": 135, "Bournemouth": 58, "Hayes & Yeading": 130, "Rushden & D": 81, "Weymouth": 120, "Chesterfield": 48, "Exeter": 104, "Barnsley": 45, "Aldershot": 95, "Gateshead": 129, "Hartlepool": 55, "Newport County": 132, "Crystal Palace": 23, "Ebbsfleet": 123, "Wigan": 19, "Shrewsbury": 83, "Hereford": 105, "Stevenage": 111, "Grimsby": 73, "Crawley Town": 114, "Morecambe": 109, "Oldham": 61, "Aston Villa": 1, "Bristol City": 51, "Gravesend": 103, "Huddersfield": 60, "Reading": 33, "Nuneaton Town": 140, "AFC Telford United": 137, "Wycombe": 91, "Leeds": 43, "Colchester": 54, "Rotherham": 63, "Southport": 100, "Southampton": 37, "Darlington": 82, "Blackburn": 16, "Bath City": 133, "Yeovil": 62, "Leyton Orient": 75, "Forest Green": 101, "Chester": 80, "Halifax": 110, "Portsmouth": 11, "Woking": 108, "Histon": 125, "Man City": 7, "Northampton": 78, "Arsenal": 17, "Charlton": 14, "Middlesbrough": 9, "Watford": 41, "Nott'm Forest": 59, "Eastbourne Borough": 131, "Hull": 27, "Barrow": 127, "Doncaster": 52, "Carlisle": 92, "Gillingham": 53, "Accrington": 93, "Dartford": 139, "Altrincham": 112, "Scarborough": 106, "Northwich": 117, "Farsley": 124, "Tamworth": 96, "St. Albans": 119, "Alfreton Town": 136, "Mansfield": 86, "Macclesfield": 76, "Torquay": 87, "Brighton": 26, "Bradford": 56, "Lincoln": 77, "Brentford": 49, "Everton": 3, "Cambridge": 102, "Sheffield United": 35, "Stockport": 85, "Bolton": 2, "Southend": 65, "Cheltenham": 71, "Walsall": 64, "Preston": 42, "Peterboro": 79, "Birmingham": 6, "Boston": 90, "Burton": 97, "West Brom": 8, "Man United": 4, "Stafford Rangers": 118, "Wimbledon": 115, "Scunthorpe": 50, "Kidderminster": 107, "Millwall": 44, "Swansea": 67, "Norwich": 31, "Burnley": 22, "Sunderland": 13, "Sheffield Weds": 40, "Fulham": 5, "Dag and Red": 99, "Oxford": 74, "Stoke": 39, "Tottenham": 12, "Kettering Town": 128, "Coventry": 32, "Wolves": 38, "Port Vale": 66, "Milton Keynes Dons": 57, "Plymouth": 34, "Derby": 25, "Notts County": 88, "Leicester": 36, "Droylsden": 121, "Blackpool": 47, "Salisbury": 122, "Cardiff": 30, "Grays": 98, "Swindon": 46, "Hyde United": 138}

GPy/util/multioutput.py

@@ -1,12 +1,17 @@
 import numpy as np
 import warnings
-from .. import kern
+import GPy
 
-def build_XY(input_list,output_list=None,index=None):
+
+def get_slices(input_list):
     num_outputs = len(input_list)
     _s = [0] + [ _x.shape[0] for _x in input_list ]
     _s = np.cumsum(_s)
     slices = [slice(a,b) for a,b in zip(_s[:-1],_s[1:])]
+    return slices
+
+def build_XY(input_list,output_list=None,index=None):
+    num_outputs = len(input_list)
     if output_list is not None:
         assert num_outputs == len(output_list)
         Y = np.vstack(output_list)
@@ -15,42 +20,84 @@ def build_XY(input_list,output_list=None,index=None):
 
     if index is not None:
         assert len(index) == num_outputs
-        I = np.vstack( [j*np.ones((_x.shape[0],1)) for _x,j in zip(input_list,index)] )
+        I = np.hstack( [np.repeat(j,_x.shape[0]) for _x,j in zip(input_list,index)] )
     else:
-        I = np.vstack( [j*np.ones((_x.shape[0],1)) for _x,j in zip(input_list,range(num_outputs))] )
+        I = np.hstack( [np.repeat(j,_x.shape[0]) for _x,j in zip(input_list,range(num_outputs))] )
 
     X = np.vstack(input_list)
-    X = np.hstack([X,I])
-    return X,Y,slices
+    X = np.hstack([X,I[:,None]])
 
-def build_lcm(input_dim, num_outputs, CK = [], NC = [], W_columns=1,W=None,kappa=None):
-    #TODO build_icm or build_lcm
+    return X,Y,I[:,None]#slices
+
+def build_likelihood(Y_list,noise_index,likelihoods_list=None):
+    Ny = len(Y_list)
+    if likelihoods_list is None:
+       likelihoods_list = [GPy.likelihoods.Gaussian(name="Gaussian_noise_%s" %j) for y,j in zip(Y_list,range(Ny))]
+    else:
+        assert len(likelihoods_list) == Ny
+    #likelihood = GPy.likelihoods.mixed_noise.MixedNoise(likelihoods_list=likelihoods_list, noise_index=noise_index)
+    likelihood = GPy.likelihoods.mixed_noise.MixedNoise(likelihoods_list=likelihoods_list)
+    return likelihood
+
+
+def ICM(input_dim, num_outputs, kernel, W_rank=1,W=None,kappa=None,name='X'):
     """
-    Builds a kernel for a linear coregionalization model
+    Builds a kernel for an Intrinsic Coregionalization Model
+
+    :input_dim: Input dimensionality (does not include dimension of indices)
+    :num_outputs: Number of outputs
+    :param kernel: kernel that will be multiplied by the coregionalize kernel (matrix B).
+    :type kernel: a GPy kernel
+    :param W_rank: number tuples of the corregionalization parameters 'W'
+    :type W_rank: integer
+    """
+    if kernel.input_dim <> input_dim:
+        kernel.input_dim = input_dim
+        warnings.warn("kernel's input dimension overwritten to fit input_dim parameter.")
+
+    K = kernel.prod(GPy.kern.Coregionalize(1, num_outputs, active_dims=[input_dim], rank=W_rank,W=W,kappa=kappa,name='B'),name=name)
+    #K = kernel * GPy.kern.Coregionalize(1, num_outputs, active_dims=[input_dim], rank=W_rank,W=W,kappa=kappa,name='B')
+    #K = kernel ** GPy.kern.Coregionalize(input_dim, num_outputs,W_rank,W,kappa, name= 'B')
+    K['.*variance'] = 1.
+    K['.*variance'].fix()
+    return K
+
+
+def LCM(input_dim, num_outputs, kernels_list, W_rank=1,name='X'):
+    """
+    Builds a kernel for an Linear Coregionalization Model
+
+    :input_dim: Input dimensionality (does not include dimension of indices)
+    :num_outputs: Number of outputs
+    :param kernel: kernel that will be multiplied by the coregionalize kernel (matrix B).
+    :type kernel: a GPy kernel
+    :param W_rank: number tuples of the corregionalization parameters 'W'
+    :type W_rank: integer
+    """
+    Nk = len(kernels_list)
+    K = ICM(input_dim,num_outputs,kernels_list[0],W_rank,name='%s%s' %(name,0))
+    j = 1
+    for kernel in kernels_list[1:]:
+        K += ICM(input_dim,num_outputs,kernel,W_rank,name='%s%s' %(name,j))
+    return K
+
+
+def Private(input_dim, num_outputs, kernel, output, kappa=None,name='X'):
+    """
+    Builds a kernel for an Intrinsic Coregionalization Model
 
     :input_dim: Input dimensionality
     :num_outputs: Number of outputs
-    :param CK: List of coregionalized kernels (i.e., this will be multiplied by a coregionalize kernel).
-    :param K: List of kernels that will be added up together with CK, but won't be multiplied by a coregionalize kernel
-    :param W_columns: number tuples of the corregionalization parameters 'coregion_W'
-    :type W_columns: integer
+    :param kernel: kernel that will be multiplied by the coregionalize kernel (matrix B).
+    :type kernel: a GPy kernel
+    :param W_rank: number tuples of the corregionalization parameters 'W'
+    :type W_rank: integer
     """
-
-    for k in CK:
-        if k.input_dim <> input_dim:
-            k.input_dim = input_dim
-            warnings.warn("kernel's input dimension overwritten to fit input_dim parameter.")
-
-    for k in NC:
-        if k.input_dim <> input_dim + 1:
-            k.input_dim = input_dim + 1
-            warnings.warn("kernel's input dimension overwritten to fit input_dim parameter.")
-
-    kernel = CK[0].prod(kern.Coregionalize(num_outputs,W_columns,W,kappa),tensor=True)
-    for k in CK[1:]:
-        k_coreg = kern.Coregionalize(num_outputs,W_columns,W,kappa)
-        kernel += k.prod(k_coreg,tensor=True)
-    for k in NC:
-        kernel += k
-
-    return kernel
+    K = ICM(input_dim,num_outputs,kernel,W_rank=1,kappa=kappa,name=name)
+    K.B.W.fix(0)
+    _range = range(num_outputs)
+    _range.pop(output)
+    for j in _range:
+        K.B.kappa[j] = 0
+        K.B.kappa[j].fix()
+    return K