diff --git a/GPy/kern/src/kern.py b/GPy/kern/src/kern.py index 6a7aea19..c08489e2 100644 --- a/GPy/kern/src/kern.py +++ b/GPy/kern/src/kern.py @@ -209,9 +209,6 @@ class Kern(Parameterized): dtheta = self.psicomp.psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior)[0] self.gradient[:] = dtheta - def reset_gradients(self): - raise NotImplementedError - def gradients_Z_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior, psi0=None, psi1=None, psi2=None): """ diff --git a/GPy/kern/src/kernel_slice_operations.py b/GPy/kern/src/kernel_slice_operations.py index 50a1e2e8..327436e7 100644 --- a/GPy/kern/src/kernel_slice_operations.py +++ b/GPy/kern/src/kernel_slice_operations.py @@ -97,17 +97,17 @@ def _slice_Kdiag(f): def _slice_update_gradients_full(f): @wraps(f) - def wrap(self, dL_dK, X, X2=None, *a, **kw): + def wrap(self, dL_dK, X, X2=None): with _Slice_wrap(self, X, X2) as s: - ret = f(self, dL_dK, s.X, s.X2, *a, **kw) + ret = f(self, dL_dK, s.X, s.X2) return ret return wrap def _slice_update_gradients_diag(f): @wraps(f) - def wrap(self, dL_dKdiag, X, *a, **kw): + def wrap(self, dL_dKdiag, X): with _Slice_wrap(self, X, None) as s: - ret = f(self, dL_dKdiag, s.X, *a, **kw) + ret = f(self, dL_dKdiag, s.X) return ret return wrap diff --git a/GPy/kern/src/multioutput_kern.py b/GPy/kern/src/multioutput_kern.py index c196e2cd..7c499092 100644 --- a/GPy/kern/src/multioutput_kern.py +++ b/GPy/kern/src/multioutput_kern.py @@ -4,7 +4,39 @@ from functools import reduce, partial from .independent_outputs import index_to_slices from paramz.caching import Cache_this +class ZeroKern(Kern): + def __init__(self): + super(ZeroKern, self).__init__(1, None, name='ZeroKern',useGPU=False) + + def K(self, X ,X2=None): + if X2 is None: + X2 = X + return np.zeros((X.shape[0],X2.shape[0])) + + def update_gradients_full(self,dL_dK, X, X2=None): + return np.zeros(dL_dK.shape) + + def gradients_X(self,dL_dK, X, X2=None): + return np.zeros((X.shape[0],X.shape[1])) + class MultioutputKern(CombinationKernel): + """ + Multioutput kernel is a meta class for combining different kernels for multioutput GPs. + + As an example let us have inputs x1 for output 1 with covariance k1 and x2 for output 2 with covariance k2. + In addition, we need to define the cross covariances k12(x1,x2) and k21(x2,x1). Then the kernel becomes: + k([x1,x2],[x1,x2]) = [k1(x1,x1) k12(x1, x2); k21(x2, x1), k2(x2,x2)] + + For the kernel, the kernels of outputs are given as list in param "kernels" and cross covariances are + given in param "cross_covariances" as a dictionary of tuples (i,j) as keys. If no cross covariance is given, + it defaults to zero, as in k12(x1,x2)=0. + + In the cross covariance dictionary, the value needs to be a struct with elements + -'kernel': a member of Kernel class that stores the hyper parameters to be updated when optimizing the GP + -'K': function defining the cross covariance + -'update_gradients_full': a function to be used for updating gradients + -'gradients_X': gives a gradient of the cross covariance with respect to the first input + """ def __init__(self, kernels, cross_covariances={}, name='MultioutputKern'): #kernels contains a list of kernels as input, if not isinstance(kernels, list): @@ -30,13 +62,14 @@ class MultioutputKern(CombinationKernel): unique=True for j in range(0,nl): if i==j or (kernels[i] is kernels[j]): - covariance[i][j] = {'K': kernels[i].K, 'update_gradients_full': kernels[i].update_gradients_full, 'gradients_X': kernels[i].gradients_X} + covariance[i][j] = {'kern': kernels[i], 'K': kernels[i].K, 'update_gradients_full': kernels[i].update_gradients_full, 'gradients_X': kernels[i].gradients_X} if i>j: unique=False elif cross_covariances.get((i,j)) is not None: #cross covariance is given covariance[i][j] = cross_covariances.get((i,j)) - else: # zero matrix - covariance[i][j] = {'K': lambda x, x2: np.zeros((x.shape[0],x2.shape[0])), 'update_gradients_full': lambda dl_dk, x, x2, reset: np.zeros(dl_dk.shape), 'gradients_X': lambda dl_dk, x, x2: np.zeros((x.shape[0],x.shape[1]))} + else: # zero covariance structure + kern = ZeroKern() + covariance[i][j] = {'kern': kern, 'K': kern.K, 'update_gradients_full': kern.update_gradients_full, 'gradients_X': kern.gradients_X} if unique is True: linked.append(i) self.covariance = covariance @@ -59,24 +92,33 @@ class MultioutputKern(CombinationKernel): target = np.zeros(X.shape[0]) [[np.copyto(target[s], kern.Kdiag(X[s])) for s in slices_i] for kern, slices_i in zip(kerns, slices)] return target - + + def update_gradients_full_wrapper(self, cov_struct, dL_dK, X, X2): + gradient = cov_struct['kern'].gradient.copy() + cov_struct['update_gradients_full'](dL_dK, X, X2) + cov_struct['kern'].gradient += gradient + + def update_gradients_diag_wrapper(self, kern, dL_dKdiag, X): + gradient = kern.gradient.copy() + kern.update_gradients_diag(dL_dKdiag, X) + kern.gradient += gradient + def reset_gradients(self): for kern in self.kern: kern.reset_gradients() - def update_gradients_full(self,dL_dK, X, X2=None, reset=True): - if reset: self.reset_gradients() + def update_gradients_full(self,dL_dK, X, X2=None): + self.reset_gradients() slices = index_to_slices(X[:,self.index_dim]) if X2 is not None: slices2 = index_to_slices(X2[:,self.index_dim]) - [[[[ self.covariance[i][j]['update_gradients_full'](dL_dK[slices[i][k],slices2[j][l]], X[slices[i][k],:], X2[slices2[j][l],:], False) for k in range(len(slices[i]))] for l in range(len(slices2[j]))] for i in range(len(slices))] for j in range(len(slices2))] + [[[[ self.update_gradients_full_wrapper(self.covariance[i][j], dL_dK[slices[i][k],slices2[j][l]], X[slices[i][k],:], X2[slices2[j][l],:]) for k in range(len(slices[i]))] for l in range(len(slices2[j]))] for i in range(len(slices))] for j in range(len(slices2))] else: - [[[[ self.covariance[i][j]['update_gradients_full'](dL_dK[slices[i][k],slices[j][l]], X[slices[i][k],:], X[slices[j][l],:] , False) for k in range(len(slices[i]))] for l in range(len(slices[j]))] for i in range(len(slices))] for j in range(len(slices))] + [[[[ self.update_gradients_full_wrapper(self.covariance[i][j], dL_dK[slices[i][k],slices[j][l]], X[slices[i][k],:], X[slices[j][l],:]) for k in range(len(slices[i]))] for l in range(len(slices[j]))] for i in range(len(slices))] for j in range(len(slices))] - def update_gradients_diag(self, dL_dKdiag, X, reset=True): - if reset: self.reset_gradients() + def update_gradients_diag(self, dL_dKdiag, X): + self.reset_gradients() slices = index_to_slices(X[:,self.index_dim]) - kerns = itertools.repeat(self.kern) if self.single_kern else self.kern - [[ self.kern[i].update_gradients_diag(dL_dKdiag[slices[i][k]], X[slices[i][k],:], False) for k in range(len(slices[i]))] for i in range(len(slices))] + [[ self.update_gradients_diag_wrapper(self.covariance[i][i]['kern'], dL_dKdiag[slices[i][k]], X[slices[i][k],:]) for k in range(len(slices[i]))] for i in range(len(slices))] def gradients_X(self,dL_dK, X, X2=None): slices = index_to_slices(X[:,self.index_dim]) diff --git a/GPy/kern/src/rbf.py b/GPy/kern/src/rbf.py index 479e1357..0b6730d8 100644 --- a/GPy/kern/src/rbf.py +++ b/GPy/kern/src/rbf.py @@ -107,10 +107,10 @@ class RBF(Stationary): def gradients_qX_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): return self.psicomp.psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior)[3:] - def update_gradients_diag(self, dL_dKdiag, X, reset=True): - super(RBF,self).update_gradients_diag(dL_dKdiag, X, reset=reset) + def update_gradients_diag(self, dL_dKdiag, X): + super(RBF,self).update_gradients_diag(dL_dKdiag, X) if self.use_invLengthscale: self.inv_l.gradient =self.lengthscale.gradient*(self.lengthscale**3/-2.) - def update_gradients_full(self, dL_dK, X, X2=None, reset=True): - super(RBF,self).update_gradients_full(dL_dK, X, X2, reset=reset) + def update_gradients_full(self, dL_dK, X, X2=None): + super(RBF,self).update_gradients_full(dL_dK, X, X2) if self.use_invLengthscale: self.inv_l.gradient =self.lengthscale.gradient*(self.lengthscale**3/-2.) diff --git a/GPy/kern/src/stationary.py b/GPy/kern/src/stationary.py index cd09a4a2..81129a75 100644 --- a/GPy/kern/src/stationary.py +++ b/GPy/kern/src/stationary.py @@ -178,7 +178,7 @@ class Stationary(Kern): else: self.lengthscale.gradient = np.zeros(self.input_dim) - def update_gradients_diag(self, dL_dKdiag, X, reset=True): + def update_gradients_diag(self, dL_dKdiag, X): """ Given the derivative of the objective with respect to the diagonal of the covariance matrix, compute the derivative wrt the parameters of @@ -186,9 +186,8 @@ class Stationary(Kern): See also update_gradients_full """ - if reset: self.reset_gradients() - self.variance.gradient += np.sum(dL_dKdiag) - self.lengthscale.gradient += 0. + self.variance.gradient = np.sum(dL_dKdiag) + self.lengthscale.gradient = 0. def update_gradients_full(self, dL_dK, X, X2=None, reset=True): """ @@ -196,8 +195,7 @@ class Stationary(Kern): (dL_dK), compute the gradient wrt the parameters of this kernel, and store in the parameters object as e.g. self.variance.gradient """ - if reset: self.reset_gradients() - self.variance.gradient += np.sum(self.K(X, X2)* dL_dK)/self.variance + self.variance.gradient = np.sum(self.K(X, X2)* dL_dK)/self.variance #now the lengthscale gradient(s) dL_dr = self.dK_dr_via_X(X, X2) * dL_dK @@ -206,12 +204,12 @@ class Stationary(Kern): tmp = dL_dr*self._inv_dist(X, X2) if X2 is None: X2 = X if config.getboolean('cython', 'working'): - self.lengthscale.gradient += self._lengthscale_grads_cython(tmp, X, X2) + self.lengthscale.gradient = self._lengthscale_grads_cython(tmp, X, X2) else: - self.lengthscale.gradient += self._lengthscale_grads_pure(tmp, X, X2) + self.lengthscale.gradient = self._lengthscale_grads_pure(tmp, X, X2) else: r = self._scaled_dist(X, X2) - self.lengthscale.gradient += -np.sum(dL_dr*r)/self.lengthscale + self.lengthscale.gradient = -np.sum(dL_dr*r)/self.lengthscale def update_gradients_direct(self, dL_dVar, dL_dLen): diff --git a/GPy/testing/kernel_tests.py b/GPy/testing/kernel_tests.py index f903c9aa..e1c9d934 100644 --- a/GPy/testing/kernel_tests.py +++ b/GPy/testing/kernel_tests.py @@ -494,7 +494,6 @@ class KernelGradientTestsContinuous(unittest.TestCase): X2t,_,_ = GPy.util.multioutput.build_XY([self.X2, self.X2]) self.assertTrue(check_kernel_gradient_functions(k, X=Xt, X2=X2t, verbose=verbose, fixed_X_dims=-1)) - def test_Precomputed(self): Xall = np.concatenate([self.X, self.X2]) cov = np.dot(Xall, Xall.T)