From b90a8672321864dbb6d9fbfbd96dfed30c459612 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Wed, 2 Apr 2014 11:43:32 +0100 Subject: [PATCH] [GPU] update gradients rest --- GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py | 76 ++++++++++++++++++--- GPy/kern/_src/rbf.py | 68 ++++++++++-------- 2 files changed, 106 insertions(+), 38 deletions(-) diff --git a/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py b/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py index a21a15e3..d8c84df4 100644 --- a/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py +++ b/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py @@ -246,7 +246,7 @@ try: dpsi2_dgamma[IDX_NMMQ(n,m1,m2,q)] = var2*neq*(psi2exp1_c/denom_sqrt - psi2exp2_c); dpsi2_dmu[IDX_NMMQ(n,m1,m2,q)] = var2*neq*(-2.0*psi2_common*muZ*psi2exp1_c); dpsi2_dS[IDX_NMMQ(n,m1,m2,q)] = var2*neq*(psi2_common*(2.0*muZ*muZ/(2.0*S_c+l_c)-1.0)*psi2exp1_c); - dpsi2_dZ[IDX_NMMQ(n,m1,m2,q)] = var2*neq*(psi2_common*(dZ*denom/-2.0+muZ)*psi2exp1_c-gamma1*Z1_c/l_c*psi2exp2_c)*2.0; + dpsi2_dZ[IDX_NMMQ(n,m1,m2,q)] = var2*neq*(psi2_common*(dZ*denom/-2.0+muZ)*psi2exp1_c-gamma1*Z2_c/l_c*psi2exp2_c)*2.0; return var2*neq*(psi2_common*(S_c/l_c+dZ*dZ*denom/(4.0*l_c)+muZ*muZ/(2.0*S_c+l_c))*psi2exp1_c+gamma1*Z2/(2.0*l_c)*psi2exp2_c)*l_sqrt_c*2.0; } """) @@ -411,7 +411,7 @@ class PSICOMP_SSRBF(object): dpsi2_dl_gpu = self.gpuCache['dpsi2_dl_gpu'] psi1_comb_gpu = self.gpuCache['psi1_neq_gpu'] psi2_comb_gpu = self.gpuCache['psi2_neq_gpu'] - grad_dl_gpu = self.gpuCache['grad_l_gpu'] + grad_l_gpu = self.gpuCache['grad_l_gpu'] # variance variance.gradient = gpuarray.sum(dL_dpsi0).get() \ @@ -420,22 +420,78 @@ class PSICOMP_SSRBF(object): # lengscale if ARD: - grad_dl_gpu.fill(0.) + grad_l_gpu.fill(0.) linalg_gpu.mul_bcast(psi1_comb_gpu, dL_dpsi1, dpsi1_dl_gpu, dL_dpsi1.size) - linalg_gpu.sum_axis(grad_dl_gpu, psi1_comb_gpu, 1, N*M) + linalg_gpu.sum_axis(grad_l_gpu, psi1_comb_gpu, 1, N*M) linalg_gpu.mul_bcast(psi2_comb_gpu, dL_dpsi2, dpsi2_dl_gpu, dL_dpsi2.size) - linalg_gpu.sum_axis(grad_dl_gpu, psi2_comb_gpu, 1, N*M*M) - lengthscale.gradient = grad_dl_gpu.get() + linalg_gpu.sum_axis(grad_l_gpu, psi2_comb_gpu, 1, N*M*M) + lengthscale.gradient = grad_l_gpu.get() else: linalg_gpu.mul_bcast(psi1_comb_gpu, dL_dpsi1, dpsi1_dl_gpu, dL_dpsi1.size) linalg_gpu.mul_bcast(psi2_comb_gpu, dL_dpsi2, dpsi2_dl_gpu, dL_dpsi2.size) lengthscale.gradient = gpuarray.sum(psi1_comb_gpu).get() + gpuarray.sum(psi2_comb_gpu).get() - def gradients_Z_expectations(self, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, mu, S, gamma): - pass + def gradients_Z_expectations(self, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior): + mu = variational_posterior.mean + S = variational_posterior.variance + gamma = variational_posterior.binary_prob + self._psiDercomputations(variance, lengthscale, Z, mu, S, gamma) + N, M, Q = mu.shape[0],Z.shape[0], mu.shape[1] + + dpsi1_dZ_gpu = self.gpuCache['dpsi1_dZ_gpu'] + dpsi2_dZ_gpu = self.gpuCache['dpsi2_dZ_gpu'] + psi1_comb_gpu = self.gpuCache['psi1_neq_gpu'] + psi2_comb_gpu = self.gpuCache['psi2_neq_gpu'] + grad_Z_gpu = self.gpuCache['grad_Z_gpu'] + + grad_Z_gpu.fill(0.) + linalg_gpu.mul_bcast(psi1_comb_gpu, dL_dpsi1, dpsi1_dZ_gpu, dL_dpsi1.size) + linalg_gpu.sum_axis(grad_Z_gpu, psi1_comb_gpu, 1, N) + linalg_gpu.mul_bcast(psi2_comb_gpu, dL_dpsi2, dpsi2_dZ_gpu, dL_dpsi2.size) + linalg_gpu.sum_axis(grad_Z_gpu, psi2_comb_gpu, 1, N*M) + return grad_Z_gpu.get() - def gradients_qX_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, mu, S, gamma): - pass + def gradients_qX_expectations(self, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior): + mu = variational_posterior.mean + S = variational_posterior.variance + gamma = variational_posterior.binary_prob + self._psiDercomputations(variance, lengthscale, Z, mu, S, gamma) + N, M, Q = mu.shape[0],Z.shape[0], mu.shape[1] + + dpsi1_dmu_gpu = self.gpuCache['dpsi1_dmu_gpu'] + dpsi2_dmu_gpu = self.gpuCache['dpsi2_dmu_gpu'] + dpsi1_dS_gpu = self.gpuCache['dpsi1_dS_gpu'] + dpsi2_dS_gpu = self.gpuCache['dpsi2_dS_gpu'] + dpsi1_dgamma_gpu = self.gpuCache['dpsi1_dgamma_gpu'] + dpsi2_dgamma_gpu = self.gpuCache['dpsi2_dgamma_gpu'] + psi1_comb_gpu = self.gpuCache['psi1_neq_gpu'] + psi2_comb_gpu = self.gpuCache['psi2_neq_gpu'] + grad_mu_gpu = self.gpuCache['grad_mu_gpu'] + grad_S_gpu = self.gpuCache['grad_S_gpu'] + grad_gamma_gpu = self.gpuCache['grad_gamma_gpu'] + + # mu gradients + grad_mu_gpu.fill(0.) + linalg_gpu.mul_bcast(psi1_comb_gpu, dL_dpsi1, dpsi1_dmu_gpu, dL_dpsi1.size) + linalg_gpu.sum_axis(grad_mu_gpu, psi1_comb_gpu, N, M) + linalg_gpu.mul_bcast(psi2_comb_gpu, dL_dpsi2, dpsi2_dmu_gpu, dL_dpsi2.size) + linalg_gpu.sum_axis(grad_mu_gpu, psi2_comb_gpu, N, M*M) + + # S gradients + grad_S_gpu.fill(0.) + linalg_gpu.mul_bcast(psi1_comb_gpu, dL_dpsi1, dpsi1_dS_gpu, dL_dpsi1.size) + linalg_gpu.sum_axis(grad_S_gpu, psi1_comb_gpu, N, M) + linalg_gpu.mul_bcast(psi2_comb_gpu, dL_dpsi2, dpsi2_dS_gpu, dL_dpsi2.size) + linalg_gpu.sum_axis(grad_S_gpu, psi2_comb_gpu, N, M*M) + + # gamma gradients + grad_gamma_gpu.fill(0.) + linalg_gpu.mul_bcast(psi1_comb_gpu, dL_dpsi1, dpsi1_dgamma_gpu, dL_dpsi1.size) + linalg_gpu.sum_axis(grad_gamma_gpu, psi1_comb_gpu, N, M) + linalg_gpu.mul_bcast(psi2_comb_gpu, dL_dpsi2, dpsi2_dgamma_gpu, dL_dpsi2.size) + linalg_gpu.sum_axis(grad_gamma_gpu, psi2_comb_gpu, N, M*M) + + return grad_mu_gpu.get(), grad_S_gpu.get(), grad_gamma_gpu.get() @Cache_this(limit=1) def _Z_distances(Z): diff --git a/GPy/kern/_src/rbf.py b/GPy/kern/_src/rbf.py index 22966448..39d36cf3 100644 --- a/GPy/kern/_src/rbf.py +++ b/GPy/kern/_src/rbf.py @@ -73,36 +73,33 @@ class RBF(Stationary): def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): # Spike-and-Slab GPLVM if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - dL_dpsi0_gpu = gpuarray.to_gpu(np.asfortranarray(dL_dpsi0)) - dL_dpsi1_gpu = gpuarray.to_gpu(np.asfortranarray(dL_dpsi1)) - dL_dpsi2_gpu = gpuarray.to_gpu(np.asfortranarray(dL_dpsi2)) - self.psicomp.update_gradients_expectations(dL_dpsi0_gpu, dL_dpsi1_gpu, dL_dpsi2_gpu, self.variance, self.lengthscale, Z, variational_posterior) - vg = self.variance.gradient.copy() - lg = self.lengthscale.gradient.copy() - - _, _dpsi1_dvariance, _, _, _, _, _dpsi1_dlengthscale = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) - _, _dpsi2_dvariance, _, _, _, _, _dpsi2_dlengthscale = ssrbf_psi_comp._psi2computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) - - #contributions from psi0: - self.variance.gradient = np.sum(dL_dpsi0) - - #from psi1 - self.variance.gradient += np.sum(dL_dpsi1 * _dpsi1_dvariance) - if self.ARD: - self.lengthscale.gradient = (dL_dpsi1[:,:,None]*_dpsi1_dlengthscale).reshape(-1,self.input_dim).sum(axis=0) + if self.useGPU: + dL_dpsi0_gpu = gpuarray.to_gpu(np.asfortranarray(dL_dpsi0)) + dL_dpsi1_gpu = gpuarray.to_gpu(np.asfortranarray(dL_dpsi1)) + dL_dpsi2_gpu = gpuarray.to_gpu(np.asfortranarray(dL_dpsi2)) + self.psicomp.update_gradients_expectations(dL_dpsi0_gpu, dL_dpsi1_gpu, dL_dpsi2_gpu, self.variance, self.lengthscale, Z, variational_posterior) else: - self.lengthscale.gradient = (dL_dpsi1[:,:,None]*_dpsi1_dlengthscale).sum() - - #from psi2 - self.variance.gradient += (dL_dpsi2 * _dpsi2_dvariance).sum() - if self.ARD: - self.lengthscale.gradient += (dL_dpsi2[:,:,:,None] * _dpsi2_dlengthscale).reshape(-1,self.input_dim).sum(axis=0) - else: - self.lengthscale.gradient += (dL_dpsi2[:,:,:,None] * _dpsi2_dlengthscale).sum() - print np.abs(vg-self.variance.gradient) - print np.abs(lg-self.lengthscale.gradient) - + _, _dpsi1_dvariance, _, _, _, _, _dpsi1_dlengthscale = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) + _, _dpsi2_dvariance, _, _, _, _, _dpsi2_dlengthscale = ssrbf_psi_comp._psi2computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) + + #contributions from psi0: + self.variance.gradient = np.sum(dL_dpsi0) + + #from psi1 + self.variance.gradient += np.sum(dL_dpsi1 * _dpsi1_dvariance) + if self.ARD: + self.lengthscale.gradient = (dL_dpsi1[:,:,None]*_dpsi1_dlengthscale).reshape(-1,self.input_dim).sum(axis=0) + else: + self.lengthscale.gradient = (dL_dpsi1[:,:,None]*_dpsi1_dlengthscale).sum() + + #from psi2 + self.variance.gradient += (dL_dpsi2 * _dpsi2_dvariance).sum() + if self.ARD: + self.lengthscale.gradient += (dL_dpsi2[:,:,:,None] * _dpsi2_dlengthscale).reshape(-1,self.input_dim).sum(axis=0) + else: + self.lengthscale.gradient += (dL_dpsi2[:,:,:,None] * _dpsi2_dlengthscale).sum() + elif isinstance(variational_posterior, variational.NormalPosterior): l2 = self.lengthscale**2 if l2.size != self.input_dim: @@ -141,6 +138,12 @@ class RBF(Stationary): def gradients_Z_expectations(self, dL_dpsi1, dL_dpsi2, Z, variational_posterior): # Spike-and-Slab GPLVM if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): + dL_dpsi1_gpu = gpuarray.to_gpu(np.asfortranarray(dL_dpsi1)) + dL_dpsi2_gpu = gpuarray.to_gpu(np.asfortranarray(dL_dpsi2)) + gZ = self.psicomp.gradients_Z_expectations(dL_dpsi1_gpu, dL_dpsi2_gpu, self.variance, self.lengthscale, Z, variational_posterior) + + + _, _, _, _, _, _dpsi1_dZ, _ = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) _, _, _, _, _, _dpsi2_dZ, _ = ssrbf_psi_comp._psi2computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) @@ -150,6 +153,8 @@ class RBF(Stationary): #psi2 grad += (dL_dpsi2[:, :, :, None] * _dpsi2_dZ).sum(axis=0).sum(axis=1) + print np.abs(gZ - grad).max() + return grad elif isinstance(variational_posterior, variational.NormalPosterior): @@ -174,6 +179,11 @@ class RBF(Stationary): def gradients_qX_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): # Spike-and-Slab GPLVM if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): + dL_dpsi1_gpu = gpuarray.to_gpu(np.asfortranarray(dL_dpsi1)) + dL_dpsi2_gpu = gpuarray.to_gpu(np.asfortranarray(dL_dpsi2)) + gmu,gS,gg = self.psicomp.gradients_qX_expectations(dL_dpsi1_gpu, dL_dpsi2_gpu, self.variance, self.lengthscale, Z, variational_posterior) + + ndata = variational_posterior.mean.shape[0] _, _, _dpsi1_dgamma, _dpsi1_dmu, _dpsi1_dS, _, _ = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) @@ -191,6 +201,8 @@ class RBF(Stationary): if self.group_spike_prob: grad_gamma[:] = grad_gamma.mean(axis=0) + + print np.abs(gmu-grad_mu).max(),np.abs(gS-grad_S).max(),np.abs(gg-grad_gamma).max() return grad_mu, grad_S, grad_gamma