From 9bb5c1af1f1ddd2cfe5c25833a485d63aa7457a0 Mon Sep 17 00:00:00 2001 From: James Hensman Date: Tue, 6 May 2014 16:36:01 +0100 Subject: [PATCH 01/55] proper whitespace --- GPy/models/bayesian_gplvm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/GPy/models/bayesian_gplvm.py b/GPy/models/bayesian_gplvm.py index 03cd361c..0a2c2396 100644 --- a/GPy/models/bayesian_gplvm.py +++ b/GPy/models/bayesian_gplvm.py @@ -70,7 +70,7 @@ class BayesianGPLVM(SparseGP): if isinstance(self.inference_method, VarDTC_GPU): update_gradients(self) return - + super(BayesianGPLVM, self).parameters_changed() self._log_marginal_likelihood -= self.variational_prior.KL_divergence(self.X) @@ -82,7 +82,7 @@ class BayesianGPLVM(SparseGP): def plot_latent(self, labels=None, which_indices=None, resolution=50, ax=None, marker='o', s=40, fignum=None, plot_inducing=True, legend=True, - plot_limits=None, + plot_limits=None, aspect='auto', updates=False, **kwargs): import sys assert "matplotlib" in sys.modules, "matplotlib package has not been imported." @@ -159,7 +159,7 @@ class BayesianGPLVM(SparseGP): from ..plotting.matplot_dep import dim_reduction_plots return dim_reduction_plots.plot_steepest_gradient_map(self,*args,**kwargs) - + def latent_cost_and_grad(mu_S, kern, Z, dL_dpsi0, dL_dpsi1, dL_dpsi2): """ From ddbf15d76386c2277c8155112337e4d34c73d00d Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Thu, 8 May 2014 13:08:16 +0100 Subject: [PATCH 02/55] [GPU] --- .../latent_function_inference/var_dtc_gpu.py | 111 +++++++++++------- .../var_dtc_parallel.py | 56 +++++---- GPy/kern/_src/psi_comp/ssrbf_psi_comp.py | 6 +- GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py | 29 +++-- GPy/util/caching.py | 3 +- GPy/util/linalg_gpu.py | 3 + 6 files changed, 122 insertions(+), 86 deletions(-) diff --git a/GPy/inference/latent_function_inference/var_dtc_gpu.py b/GPy/inference/latent_function_inference/var_dtc_gpu.py index 9b2da1c9..a06c6c80 100644 --- a/GPy/inference/latent_function_inference/var_dtc_gpu.py +++ b/GPy/inference/latent_function_inference/var_dtc_gpu.py @@ -15,7 +15,7 @@ try: import scikits.cuda.linalg as culinalg import pycuda.gpuarray as gpuarray from scikits.cuda import cublas - from ...util.linalg_gpu import logDiagSum, strideSum, mul_bcast, sum_axis, outer_prod, mul_bcast_first, join_prod + from ...util.linalg_gpu import logDiagSum, strideSum, mul_bcast, sum_axis, outer_prod, mul_bcast_first, join_prod, traceDot except: pass @@ -69,14 +69,14 @@ class VarDTC_GPU(object): # inference_minibatch 'dL_dpsi0_gpu' :gpuarray.empty((self.batchsize,),np.float64,order='F'), 'dL_dpsi1_gpu' :gpuarray.empty((self.batchsize,num_inducing),np.float64,order='F'), - 'dL_dpsi2_gpu' :gpuarray.empty((self.batchsize,num_inducing,num_inducing),np.float64,order='F'), + 'dL_dpsi2_gpu' :gpuarray.empty((num_inducing,num_inducing),np.float64,order='F'), 'dL_dthetaL_gpu' :gpuarray.empty((self.batchsize,),np.float64,order='F'), 'betapsi1_gpu' :gpuarray.empty((self.batchsize,num_inducing),np.float64,order='F'), 'thetaL_t_gpu' :gpuarray.empty((self.batchsize,),np.float64,order='F'), 'betaYT2_gpu' :gpuarray.empty((output_dim,self.batchsize),np.float64,order='F'), 'psi0p_gpu' :gpuarray.empty((self.batchsize,),np.float64,order='F'), 'psi1p_gpu' :gpuarray.empty((self.batchsize,num_inducing),np.float64,order='F'), - 'psi2p_gpu' :gpuarray.empty((self.batchsize,num_inducing,num_inducing),np.float64,order='F'), + 'psi2p_gpu' :gpuarray.empty((num_inducing,num_inducing),np.float64,order='F'), } self.gpuCache['ones_gpu'].fill(1.0) @@ -136,6 +136,12 @@ class VarDTC_GPU(object): num_inducing, input_dim = Z.shape[0], Z.shape[1] num_data, output_dim = Y.shape + #see whether we've got a different noise variance for each datum + beta = 1./np.fmax(likelihood.variance, 1e-6) + het_noise = beta.size > 1 + if het_noise: + self.batchsize=0 + self._initGPUCache(kern, num_inducing, input_dim, output_dim, Y) if isinstance(X, VariationalPosterior): @@ -143,9 +149,7 @@ class VarDTC_GPU(object): else: uncertain_inputs = False - #see whether we've got a different noise variance for each datum - beta = 1./np.fmax(likelihood.variance, 1e-6) - het_noise = beta.size > 1 + trYYT = self._trYYT psi1Y_gpu = self.gpuCache['psi1Y_gpu'] @@ -218,7 +222,6 @@ class VarDTC_GPU(object): psi2_full = np.zeros((num_inducing,num_inducing),order='F') psi1Y_full = np.zeros((num_inducing,output_dim),order='F') # MxD psi0_full = 0 -# YRY_full = 0 for n_start in xrange(0,num_data,self.batchsize): n_end = min(self.batchsize+n_start, num_data) @@ -236,7 +239,6 @@ class VarDTC_GPU(object): beta_slice = beta[n_start:n_end] psi0_full += (beta_slice*psi0).sum() psi1Y_full += np.dot(psi1.T,beta_slice[:,None]*Y_slice) # MxD -# YRY_full += (beta_slice*np.square(Y_slice).sum(axis=-1)).sum() else: psi0_full += psi0.sum() psi1Y_full += np.dot(psi1.T,Y_slice) # MxD @@ -245,18 +247,17 @@ class VarDTC_GPU(object): if het_noise: psi2_full += np.einsum('n,nmo->mo',beta_slice,psi2) else: - psi2_full += psi2.sum(axis=0) + psi2_full += psi2 else: if het_noise: psi2_full += np.einsum('n,nm,no->mo',beta_slice,psi1,psi1) else: - psi2_full += tdot(psi1.T) + psi2_full += np.outer(psi1.T, psi1.T) if not het_noise: psi0_full *= beta psi1Y_full *= beta psi2_full *= beta -# YRY_full = trYYT*beta psi1Y_gpu.set(psi1Y_full) psi2_gpu.set(psi2_full) @@ -372,6 +373,17 @@ class VarDTC_GPU(object): post = Posterior(woodbury_inv=KmmInvPsi2P_gpu.get(), woodbury_vector=v_gpu.get(), K=Kmm_gpu.get(), mean=None, cov=None, K_chol=Lm_gpu.get()) + #====================================================================== + # Compute dL_dthetaL for uncertian input and non-heter noise + #====================================================================== + + if uncertain_inputs and not het_noise: + dL_dthetaL = (YRY_full*beta + beta*output_dim*psi0_full - num_data*output_dim*beta)/2. + dL_dthetaL += -beta*cublas.cublasDdot(self.cublas_handle,dL_dpsi2R_gpu.size, dL_dpsi2R_gpu.gpudata,1,psi2_gpu.gpudata,1) + dL_dthetaL += -beta*cublas.cublasDdot(self.cublas_handle,dL_dpsi2R_gpu.size, dL_dpsi2R_gpu.gpudata,1,psi2_gpu.gpudata,1) + dL_dthetaL += traceDot(v_gpu, psi1Y_gpu, num_inducing, output_dim) + self.midRes['dL_dthetaL'] = dL_dthetaL + return logL, dL_dKmm_gpu.get(), post def inference_minibatch(self, kern, X, Z, likelihood, Y): @@ -403,6 +415,8 @@ class VarDTC_GPU(object): nSlice = n_end-n_start X_slice = X[n_start:n_end] + if het_noise: + beta = beta[n_start] # nSlice==1 if kern.useGPU: if uncertain_inputs: @@ -413,8 +427,6 @@ class VarDTC_GPU(object): psi0p_gpu = kern.Kdiag(X_slice) psi1p_gpu = kern.K(X_slice, Z) psi2p_gpu = self.gpuCache['psi2p_gpu'] - if psi2p_gpu.shape[0] > nSlice: - psi2p_gpu = psi2p_gpu.ravel()[:nSlice*num_inducing*num_inducing].reshape(nSlice,num_inducing,num_inducing) else: if uncertain_inputs: psi0 = kern.psi0(Z, X_slice) @@ -430,7 +442,6 @@ class VarDTC_GPU(object): if psi0p_gpu.shape[0] > nSlice: psi0p_gpu = psi0p_gpu[:nSlice] psi1p_gpu = psi1p_gpu.ravel()[:nSlice*num_inducing].reshape(nSlice,num_inducing) - psi2p_gpu = psi2p_gpu.ravel()[:nSlice*num_inducing*num_inducing].reshape(nSlice,num_inducing,num_inducing) psi0p_gpu.set(np.asfortranarray(psi0)) psi1p_gpu.set(np.asfortranarray(psi1)) if uncertain_inputs: @@ -473,13 +484,13 @@ class VarDTC_GPU(object): # Compute dL_dpsi #====================================================================== - dL_dpsi0_gpu.fill(0.) - cublas.cublasDaxpy(self.cublas_handle, dL_dpsi0_gpu.size, output_dim/(-2.), beta_gpu_slice.gpudata, 1, dL_dpsi0_gpu.gpudata, 1) + dL_dpsi0_gpu.fill(-output_dim *beta/2.) cublas.cublasDgemm(self.cublas_handle, 'T', 'T', nSlice, num_inducing, output_dim, 1.0, betaYT_gpu_slice.gpudata, output_dim, v_gpu.gpudata, num_inducing, 0., dL_dpsi1_gpu.gpudata, nSlice) if uncertain_inputs: - outer_prod(dL_dpsi2_gpu,beta_gpu_slice,dL_dpsi2R_gpu,beta_gpu_slice.size) + cublas.cublasDcopy(self.cublas_handle, dL_dpsi2R_gpu.size, dL_dpsi2R_gpu.gpudata, 1, dL_dpsi2_gpu.gpudata, 1) + cublas.cublasDscal(self.cublas_handle, dL_dpsi2_gpu.szie, beta, dL_dpsi2_gpu.gpudata, 1) else: cublas.cublasDgemm(self.cublas_handle, 'N', 'N', nSlice, num_inducing, output_dim, 1.0, betapsi1_gpu.gpudata, nSlice, dL_dpsi2R_gpu.gpudata, num_inducing, 1.0, dL_dpsi1_gpu.gpudata, nSlice) @@ -487,34 +498,44 @@ class VarDTC_GPU(object): # Compute dL_dthetaL #====================================================================== - if not uncertain_inputs: - join_prod(psi2p_gpu,psi1p_gpu,psi1p_gpu,nSlice,num_inducing) + if uncertain_inputs and not het_noise: + if isEnd: + dL_dthetaL = self.midRes['dL_dthetaL'] + else: + dL_dthetaL = 0 + +# if not uncertain_inputs: +# cublas.cublasDgemm(self.cublas_handle, 'T', 'N', num_inducing, num_inducing, nSlice, beta, psi1p_gpu.gpudata, nSlice, psi1p_gpu.gpudata, nSlice, 1.0, psi2p_gpu.gpudata, num_inducing) +# join_prod(psi2p_gpu,psi1p_gpu,psi1p_gpu,nSlice,num_inducing) - mul_bcast_first(psi2R_gpu,dL_dpsi2R_gpu,psi2p_gpu,nSlice) - - - dL_dthetaL_gpu.fill(0.) - - cublas.cublasDcopy(self.cublas_handle, betaYT_gpu_slice.size, betaYT_gpu_slice.gpudata, 1, betaYT2_gpu.gpudata, 1) - mul_bcast(betaYT2_gpu,betaYT2_gpu,betaYT2_gpu,betaYT2_gpu.size) - cublas.cublasDscal(self.cublas_handle, betaYT2_gpu.size, 0.5, betaYT2_gpu.gpudata, 1) - sum_axis(dL_dthetaL_gpu, betaYT2_gpu, 1, output_dim) - - cublas.cublasDaxpy(self.cublas_handle, dL_dthetaL_gpu.size, output_dim/(-2.0), beta_gpu_slice.gpudata, 1, dL_dthetaL_gpu.gpudata, 1) - cublas.cublasDcopy(self.cublas_handle, beta_gpu_slice.size, beta_gpu_slice.gpudata, 1, thetaL_t_gpu.gpudata, 1) - mul_bcast(thetaL_t_gpu,thetaL_t_gpu,thetaL_t_gpu,thetaL_t_gpu.size) - mul_bcast(thetaL_t_gpu,thetaL_t_gpu,psi0p_gpu,thetaL_t_gpu.size) - cublas.cublasDaxpy(self.cublas_handle, dL_dthetaL_gpu.size, output_dim/2.0, thetaL_t_gpu.gpudata, 1, dL_dthetaL_gpu.gpudata, 1) - - thetaL_t_gpu.fill(0.) - sum_axis(thetaL_t_gpu, psi2R_gpu, nSlice, num_inducing*num_inducing) - mul_bcast(thetaL_t_gpu,thetaL_t_gpu,beta_gpu_slice,thetaL_t_gpu.size) - mul_bcast(thetaL_t_gpu,thetaL_t_gpu,beta_gpu_slice,thetaL_t_gpu.size) - cublas.cublasDaxpy(self.cublas_handle, dL_dthetaL_gpu.size, -1.0, thetaL_t_gpu.gpudata, 1, dL_dthetaL_gpu.gpudata, 1) - - cublas.cublasDgemm(self.cublas_handle, 'T', 'T', output_dim, nSlice, num_inducing, -1.0, v_gpu.gpudata, num_inducing, betapsi1_gpu.gpudata, nSlice, 0.0, betaYT2_gpu.gpudata, output_dim) - mul_bcast(betaYT2_gpu,betaYT2_gpu,betaYT_gpu_slice,betaYT2_gpu.size) - sum_axis(dL_dthetaL_gpu, betaYT2_gpu, 1, output_dim) +# psiR = cublas.cublasDdot(self.cublas_handle, dL_dpsi2R_gpu.size, dL_dpsi2R_gpu.gpudata, 1, psi2p_gpu.gpudata, 1) +# # mul_bcast_first(psi2R_gpu,dL_dpsi2R_gpu,psi2p_gpu,nSlice) +# +# dL_dthetaL_gpu.fill(0.) +# dL_dthetaL = 0. +# +# dL_dthetaL += cublas.cublasDdot(self.cublas_handle, betaYT_gpu_slice.size, betaYT_gpu_slice.gpudata,1,betaYT_gpu_slice.gpudata,1) +# +# cublas.cublasDcopy(self.cublas_handle, betaYT_gpu_slice.size, betaYT_gpu_slice.gpudata, 1, betaYT2_gpu.gpudata, 1) +# mul_bcast(betaYT2_gpu,betaYT2_gpu,betaYT2_gpu,betaYT2_gpu.size) +# cublas.cublasDscal(self.cublas_handle, betaYT2_gpu.size, 0.5, betaYT2_gpu.gpudata, 1) +# sum_axis(dL_dthetaL_gpu, betaYT2_gpu, 1, output_dim) +# +# cublas.cublasDaxpy(self.cublas_handle, dL_dthetaL_gpu.size, output_dim/(-2.0), beta_gpu_slice.gpudata, 1, dL_dthetaL_gpu.gpudata, 1) +# cublas.cublasDcopy(self.cublas_handle, beta_gpu_slice.size, beta_gpu_slice.gpudata, 1, thetaL_t_gpu.gpudata, 1) +# mul_bcast(thetaL_t_gpu,thetaL_t_gpu,thetaL_t_gpu,thetaL_t_gpu.size) +# mul_bcast(thetaL_t_gpu,thetaL_t_gpu,psi0p_gpu,thetaL_t_gpu.size) +# cublas.cublasDaxpy(self.cublas_handle, dL_dthetaL_gpu.size, output_dim/2.0, thetaL_t_gpu.gpudata, 1, dL_dthetaL_gpu.gpudata, 1) +# +# thetaL_t_gpu.fill(0.) +# sum_axis(thetaL_t_gpu, psi2R_gpu, nSlice, num_inducing*num_inducing) +# mul_bcast(thetaL_t_gpu,thetaL_t_gpu,beta_gpu_slice,thetaL_t_gpu.size) +# mul_bcast(thetaL_t_gpu,thetaL_t_gpu,beta_gpu_slice,thetaL_t_gpu.size) +# cublas.cublasDaxpy(self.cublas_handle, dL_dthetaL_gpu.size, -1.0, thetaL_t_gpu.gpudata, 1, dL_dthetaL_gpu.gpudata, 1) +# +# cublas.cublasDgemm(self.cublas_handle, 'T', 'T', output_dim, nSlice, num_inducing, -1.0, v_gpu.gpudata, num_inducing, betapsi1_gpu.gpudata, nSlice, 0.0, betaYT2_gpu.gpudata, output_dim) +# mul_bcast(betaYT2_gpu,betaYT2_gpu,betaYT_gpu_slice,betaYT2_gpu.size) +# sum_axis(dL_dthetaL_gpu, betaYT2_gpu, 1, output_dim) if kern.useGPU: dL_dpsi0 = dL_dpsi0_gpu @@ -540,6 +561,6 @@ class VarDTC_GPU(object): grad_dict = {'dL_dKdiag':dL_dpsi0, 'dL_dKnm':dL_dpsi1, 'dL_dthetaL':dL_dthetaL} - + return isEnd, (n_start,n_end), grad_dict diff --git a/GPy/inference/latent_function_inference/var_dtc_parallel.py b/GPy/inference/latent_function_inference/var_dtc_parallel.py index 7c347213..da7e67cb 100644 --- a/GPy/inference/latent_function_inference/var_dtc_parallel.py +++ b/GPy/inference/latent_function_inference/var_dtc_parallel.py @@ -116,7 +116,7 @@ class VarDTC_minibatch(object): if het_noise: psi2_full += beta_slice*np.outer(psi1,psi1) else: - psi2_full += np.outer(psi1,psi1) + psi2_full += np.outer(psi1.T,psi1.T) if not het_noise: psi0_full *= beta @@ -169,13 +169,16 @@ class VarDTC_minibatch(object): #====================================================================== # Compute the Posterior distribution of inducing points p(u|Y) #====================================================================== - -# phi_u_mean = np.dot(Kmm,v) -# LLInvKmm,_ = dtrtrs(LL,Kmm) -# # phi_u_var = np.einsum('ma,mb->ab',LLInvKmm,LLInvKmm) -# phi_u_var = Kmm - np.dot(LLInvKmm.T,LLInvKmm) - + post = Posterior(woodbury_inv=KmmInvPsi2P, woodbury_vector=v, K=Kmm, mean=None, cov=None, K_chol=Lm) + + #====================================================================== + # Compute dL_dthetaL for uncertian input and non-heter noise + #====================================================================== + + if uncertain_inputs and not het_noise: + dL_dthetaL = (YRY_full*beta + beta*output_dim*psi0_full - num_data*output_dim*beta)/2. - beta*(dL_dpsi2R*psi2_full).sum() - beta*(v.T*psi1Y_full).sum() + self.midRes['dL_dthetaL'] = dL_dthetaL return logL, dL_dKmm, post @@ -213,20 +216,21 @@ class VarDTC_minibatch(object): Y_slice = YYT_factor[n_start:n_end] X_slice = X[n_start:n_end] - if uncertain_inputs: - psi0 = kern.psi0(Z, X_slice) - psi1 = kern.psi1(Z, X_slice) - psi2 = kern.psi2(Z, X_slice) - else: + if not uncertain_inputs: psi0 = kern.Kdiag(X_slice) psi1 = kern.K(X_slice, Z) psi2 = None + betapsi1 = np.einsum('n,nm->nm',beta,psi1) + elif het_noise: + psi0 = kern.psi0(Z, X_slice) + psi1 = kern.psi1(Z, X_slice) + psi2 = kern.psi2(Z, X_slice) + betapsi1 = np.einsum('n,nm->nm',beta,psi1) if het_noise: beta = beta[n_start] # assuming batchsize==1 betaY = beta*Y_slice - betapsi1 = np.einsum('n,nm->nm',beta,psi1) #====================================================================== # Load Intermediate Results @@ -234,12 +238,12 @@ class VarDTC_minibatch(object): dL_dpsi2R = self.midRes['dL_dpsi2R'] v = self.midRes['v'] - + #====================================================================== # Compute dL_dpsi #====================================================================== - dL_dpsi0 = -0.5 * output_dim * (beta * np.ones((n_end-n_start,))) + dL_dpsi0 = -output_dim * (beta * np.ones((n_end-n_start,)))/2. dL_dpsi1 = np.dot(betaY,v.T) @@ -254,20 +258,22 @@ class VarDTC_minibatch(object): #====================================================================== if het_noise: - if uncertain_inputs: - psiR = np.einsum('mo,nmo->n',dL_dpsi2R,psi2) - else: - psiR = np.einsum('nm,no,mo->n',psi1,psi1,dL_dpsi2R) - - dL_dthetaL = ((np.square(betaY)).sum(axis=-1) + np.square(beta)*(output_dim*psi0)-output_dim*beta)/2. - np.square(beta)*psiR- (betaY*np.dot(betapsi1,v)).sum(axis=-1) - else: if uncertain_inputs: psiR = np.einsum('mo,mo->',dL_dpsi2R,psi2) else: psiR = np.einsum('nm,no,mo->',psi1,psi1,dL_dpsi2R) - dL_dthetaL = ((np.square(betaY)).sum() + beta*beta*output_dim*(psi0.sum())-num_slice*output_dim*beta)/2. - beta*beta*psiR- (betaY*np.dot(betapsi1,v)).sum() - + dL_dthetaL = ((np.square(betaY)).sum(axis=-1) + np.square(beta)*(output_dim*psi0)-output_dim*beta)/2. - np.square(beta)*psiR- (betaY*np.dot(betapsi1,v)).sum(axis=-1) + else: + if uncertain_inputs: + if isEnd: + dL_dthetaL = self.midRes['dL_dthetaL'] + else: + dL_dthetaL = 0. + else: + psiR = np.einsum('nm,no,mo->',psi1,psi1,dL_dpsi2R) + dL_dthetaL = ((np.square(betaY)).sum() + beta*beta*output_dim*(psi0.sum())-num_slice*output_dim*beta)/2. - beta*beta*psiR- (betaY*np.dot(betapsi1,v)).sum() + if uncertain_inputs: grad_dict = {'dL_dpsi0':dL_dpsi0, 'dL_dpsi1':dL_dpsi1, @@ -320,7 +326,7 @@ def update_gradients(model): dL_dthetaL[n_range[0]:n_range[1]] = grad_dict['dL_dthetaL'] else: dL_dthetaL += grad_dict['dL_dthetaL'] - + # Set the gradients w.r.t. kernel model.kern.gradient = kern_grad diff --git a/GPy/kern/_src/psi_comp/ssrbf_psi_comp.py b/GPy/kern/_src/psi_comp/ssrbf_psi_comp.py index 5e30b5c5..24e04ae4 100644 --- a/GPy/kern/_src/psi_comp/ssrbf_psi_comp.py +++ b/GPy/kern/_src/psi_comp/ssrbf_psi_comp.py @@ -6,7 +6,7 @@ The package for the psi statistics computation """ import numpy as np -from GPy.util.caching import Cache_this +from GPy.util.caching import Cache_this,Cacher @Cache_this(limit=1) def psicomputations(variance, lengthscale, Z, mu, S, gamma): @@ -88,7 +88,7 @@ def _psi2computations(variance, lengthscale, Z, mu, S, gamma): return _psi2 -def psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior): +def _psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior): ARD = (len(lengthscale)!=1) dvar_psi1, dl_psi1, dZ_psi1, dmu_psi1, dS_psi1, dgamma_psi1 = _psi1compDer(dL_dpsi1, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) @@ -211,3 +211,5 @@ def _psi2compDer(dL_dpsi2, variance, lengthscale, Z, mu, S, gamma): # _dpsi2_dlengthscale = 2.*lengthscale* _psi2_q * (_psi2_common*(S[:,None,None,:]/lengthscale2+_psi2_Zdist_sq*_psi2_denom+_psi2_mudist_sq)*_psi2_exp_dist_sq+(1-gamma[:,None,None,:])*_psi2_Z_sq_sum*0.5/lengthscale2*_psi2_exp_Z) # NxMxMxQ return _dL_dvariance, _dL_dlengthscale, _dL_dZ, _dL_dmu, _dL_dS, _dL_dgamma + +psiDerivativecomputations = Cacher(_psiDerivativecomputations, limit=1, ignore_args=(0,1,2,)) diff --git a/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py b/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py index f49dc52a..e220e0d0 100644 --- a/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py +++ b/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py @@ -58,19 +58,22 @@ try: __device__ double comp_psi2_element(double var, double *l, double *Z, double *mu, double *S, double *logGamma, double *log1Gamma, double *logpsi2denom, int N, int M, int Q, int idx) { // psi2 (n,m1,m2) - int m2 = idx/(M*N); - int m1 = (idx%(M*N))/N; - int n = idx%N; - - double psi2_exp=0; - for(int q=0;q Date: Fri, 9 May 2014 16:12:16 +0100 Subject: [PATCH 03/55] [GPU] varDTC_gpu ready --- .../latent_function_inference/var_dtc_gpu.py | 144 ++++++------------ .../var_dtc_parallel.py | 14 +- GPy/models/ss_gplvm.py | 3 +- GPy/util/linalg_gpu.py | 4 +- 4 files changed, 52 insertions(+), 113 deletions(-) diff --git a/GPy/inference/latent_function_inference/var_dtc_gpu.py b/GPy/inference/latent_function_inference/var_dtc_gpu.py index a06c6c80..47a90fd2 100644 --- a/GPy/inference/latent_function_inference/var_dtc_gpu.py +++ b/GPy/inference/latent_function_inference/var_dtc_gpu.py @@ -70,10 +70,6 @@ class VarDTC_GPU(object): 'dL_dpsi0_gpu' :gpuarray.empty((self.batchsize,),np.float64,order='F'), 'dL_dpsi1_gpu' :gpuarray.empty((self.batchsize,num_inducing),np.float64,order='F'), 'dL_dpsi2_gpu' :gpuarray.empty((num_inducing,num_inducing),np.float64,order='F'), - 'dL_dthetaL_gpu' :gpuarray.empty((self.batchsize,),np.float64,order='F'), - 'betapsi1_gpu' :gpuarray.empty((self.batchsize,num_inducing),np.float64,order='F'), - 'thetaL_t_gpu' :gpuarray.empty((self.batchsize,),np.float64,order='F'), - 'betaYT2_gpu' :gpuarray.empty((output_dim,self.batchsize),np.float64,order='F'), 'psi0p_gpu' :gpuarray.empty((self.batchsize,),np.float64,order='F'), 'psi1p_gpu' :gpuarray.empty((self.batchsize,num_inducing),np.float64,order='F'), 'psi2p_gpu' :gpuarray.empty((num_inducing,num_inducing),np.float64,order='F'), @@ -377,13 +373,12 @@ class VarDTC_GPU(object): # Compute dL_dthetaL for uncertian input and non-heter noise #====================================================================== - if uncertain_inputs and not het_noise: - dL_dthetaL = (YRY_full*beta + beta*output_dim*psi0_full - num_data*output_dim*beta)/2. - dL_dthetaL += -beta*cublas.cublasDdot(self.cublas_handle,dL_dpsi2R_gpu.size, dL_dpsi2R_gpu.gpudata,1,psi2_gpu.gpudata,1) - dL_dthetaL += -beta*cublas.cublasDdot(self.cublas_handle,dL_dpsi2R_gpu.size, dL_dpsi2R_gpu.gpudata,1,psi2_gpu.gpudata,1) - dL_dthetaL += traceDot(v_gpu, psi1Y_gpu, num_inducing, output_dim) - self.midRes['dL_dthetaL'] = dL_dthetaL - + if not het_noise: + dL_dthetaL = (YRY_full + output_dim*psi0_full - num_data*output_dim)/-2. + dL_dthetaL += cublas.cublasDdot(self.cublas_handle,dL_dpsi2R_gpu.size, dL_dpsi2R_gpu.gpudata,1,psi2_gpu.gpudata,1) + dL_dthetaL += cublas.cublasDdot(self.cublas_handle,v_gpu.size, v_gpu.gpudata,1,psi1Y_gpu.gpudata,1) + self.midRes['dL_dthetaL'] = -beta*dL_dthetaL + return logL, dL_dKmm_gpu.get(), post def inference_minibatch(self, kern, X, Z, likelihood, Y): @@ -419,22 +414,22 @@ class VarDTC_GPU(object): beta = beta[n_start] # nSlice==1 if kern.useGPU: - if uncertain_inputs: - psi0p_gpu = kern.psi0(Z, X_slice) - psi1p_gpu = kern.psi1(Z, X_slice) - psi2p_gpu = kern.psi2(Z, X_slice) - else: + if not uncertain_inputs: psi0p_gpu = kern.Kdiag(X_slice) psi1p_gpu = kern.K(X_slice, Z) psi2p_gpu = self.gpuCache['psi2p_gpu'] - else: - if uncertain_inputs: + elif het_noise: + psi0p_gpu = kern.psi0(Z, X_slice) + psi1p_gpu = kern.psi1(Z, X_slice) + psi2p_gpu = kern.psi2(Z, X_slice) + elif not uncertain_inputs or het_noise: + if not uncertain_inputs: + psi0 = kern.Kdiag(X_slice) + psi1 = kern.K(X_slice, Z) + elif het_noise: psi0 = kern.psi0(Z, X_slice) psi1 = kern.psi1(Z, X_slice) psi2 = kern.psi2(Z, X_slice) - else: - psi0 = kern.Kdiag(X_slice) - psi1 = kern.K(X_slice, Z) psi0p_gpu = self.gpuCache['psi0p_gpu'] psi1p_gpu = self.gpuCache['psi1p_gpu'] @@ -446,43 +441,23 @@ class VarDTC_GPU(object): psi1p_gpu.set(np.asfortranarray(psi1)) if uncertain_inputs: psi2p_gpu.set(np.asfortranarray(psi2)) - - #====================================================================== - # Prepare gpu memory - #====================================================================== - - dL_dpsi2R_gpu = self.gpuCache['dL_dpsi2R_gpu'] - v_gpu = self.gpuCache['v_gpu'] - betaYT_gpu = self.gpuCache['betaYT_gpu'] - beta_gpu = self.gpuCache['beta_gpu'] - dL_dpsi0_gpu = self.gpuCache['dL_dpsi0_gpu'] - dL_dpsi1_gpu = self.gpuCache['dL_dpsi1_gpu'] - dL_dpsi2_gpu = self.gpuCache['dL_dpsi2_gpu'] - dL_dthetaL_gpu = self.gpuCache['dL_dthetaL_gpu'] - psi2R_gpu = self.gpuCache['psi2_t_gpu'][:nSlice*num_inducing*num_inducing].reshape(nSlice,num_inducing,num_inducing) - betapsi1_gpu = self.gpuCache['betapsi1_gpu'] - thetaL_t_gpu = self.gpuCache['thetaL_t_gpu'] - betaYT2_gpu = self.gpuCache['betaYT2_gpu'] - - betaYT_gpu_slice = betaYT_gpu[:,n_start:n_end] - beta_gpu_slice = beta_gpu[n_start:n_end] - - # Adjust to the batch size - if dL_dpsi0_gpu.shape[0] > nSlice: - betaYT2_gpu = betaYT2_gpu[:,:nSlice] - dL_dpsi0_gpu = dL_dpsi0_gpu.ravel()[:nSlice] - dL_dpsi1_gpu = dL_dpsi1_gpu.ravel()[:nSlice*num_inducing].reshape(nSlice,num_inducing) - dL_dpsi2_gpu = dL_dpsi2_gpu.ravel()[:nSlice*num_inducing*num_inducing].reshape(nSlice,num_inducing,num_inducing) - dL_dthetaL_gpu = dL_dthetaL_gpu.ravel()[:nSlice] - psi2R_gpu = psi2R_gpu.ravel()[:nSlice*num_inducing*num_inducing].reshape(nSlice,num_inducing,num_inducing) - thetaL_t_gpu = thetaL_t_gpu.ravel()[:nSlice] - betapsi1_gpu = betapsi1_gpu.ravel()[:nSlice*num_inducing].reshape(nSlice,num_inducing) - - mul_bcast(betapsi1_gpu,beta_gpu_slice,psi1p_gpu,beta_gpu_slice.size) - + #====================================================================== # Compute dL_dpsi #====================================================================== + + dL_dpsi2R_gpu = self.gpuCache['dL_dpsi2R_gpu'] + v_gpu = self.gpuCache['v_gpu'] + dL_dpsi0_gpu = self.gpuCache['dL_dpsi0_gpu'] + dL_dpsi1_gpu = self.gpuCache['dL_dpsi1_gpu'] + dL_dpsi2_gpu = self.gpuCache['dL_dpsi2_gpu'] + betaYT_gpu = self.gpuCache['betaYT_gpu'] + betaYT_gpu_slice = betaYT_gpu[:,n_start:n_end] + + # Adjust to the batch size + if dL_dpsi0_gpu.shape[0] > nSlice: + dL_dpsi0_gpu = dL_dpsi0_gpu.ravel()[:nSlice] + dL_dpsi1_gpu = dL_dpsi1_gpu.ravel()[:nSlice*num_inducing].reshape(nSlice,num_inducing) dL_dpsi0_gpu.fill(-output_dim *beta/2.) @@ -490,52 +465,18 @@ class VarDTC_GPU(object): if uncertain_inputs: cublas.cublasDcopy(self.cublas_handle, dL_dpsi2R_gpu.size, dL_dpsi2R_gpu.gpudata, 1, dL_dpsi2_gpu.gpudata, 1) - cublas.cublasDscal(self.cublas_handle, dL_dpsi2_gpu.szie, beta, dL_dpsi2_gpu.gpudata, 1) + cublas.cublasDscal(self.cublas_handle, dL_dpsi2_gpu.size, beta, dL_dpsi2_gpu.gpudata, 1) else: - cublas.cublasDgemm(self.cublas_handle, 'N', 'N', nSlice, num_inducing, output_dim, 1.0, betapsi1_gpu.gpudata, nSlice, dL_dpsi2R_gpu.gpudata, num_inducing, 1.0, dL_dpsi1_gpu.gpudata, nSlice) - + cublas.cublasDgemm(self.cublas_handle, 'N', 'N', nSlice, num_inducing, output_dim, beta, psi1p_gpu.gpudata, nSlice, dL_dpsi2R_gpu.gpudata, num_inducing, 1.0, dL_dpsi1_gpu.gpudata, nSlice) + #====================================================================== # Compute dL_dthetaL #====================================================================== - - if uncertain_inputs and not het_noise: - if isEnd: - dL_dthetaL = self.midRes['dL_dthetaL'] - else: - dL_dthetaL = 0 - -# if not uncertain_inputs: -# cublas.cublasDgemm(self.cublas_handle, 'T', 'N', num_inducing, num_inducing, nSlice, beta, psi1p_gpu.gpudata, nSlice, psi1p_gpu.gpudata, nSlice, 1.0, psi2p_gpu.gpudata, num_inducing) -# join_prod(psi2p_gpu,psi1p_gpu,psi1p_gpu,nSlice,num_inducing) - -# psiR = cublas.cublasDdot(self.cublas_handle, dL_dpsi2R_gpu.size, dL_dpsi2R_gpu.gpudata, 1, psi2p_gpu.gpudata, 1) -# # mul_bcast_first(psi2R_gpu,dL_dpsi2R_gpu,psi2p_gpu,nSlice) -# -# dL_dthetaL_gpu.fill(0.) -# dL_dthetaL = 0. -# -# dL_dthetaL += cublas.cublasDdot(self.cublas_handle, betaYT_gpu_slice.size, betaYT_gpu_slice.gpudata,1,betaYT_gpu_slice.gpudata,1) -# -# cublas.cublasDcopy(self.cublas_handle, betaYT_gpu_slice.size, betaYT_gpu_slice.gpudata, 1, betaYT2_gpu.gpudata, 1) -# mul_bcast(betaYT2_gpu,betaYT2_gpu,betaYT2_gpu,betaYT2_gpu.size) -# cublas.cublasDscal(self.cublas_handle, betaYT2_gpu.size, 0.5, betaYT2_gpu.gpudata, 1) -# sum_axis(dL_dthetaL_gpu, betaYT2_gpu, 1, output_dim) -# -# cublas.cublasDaxpy(self.cublas_handle, dL_dthetaL_gpu.size, output_dim/(-2.0), beta_gpu_slice.gpudata, 1, dL_dthetaL_gpu.gpudata, 1) -# cublas.cublasDcopy(self.cublas_handle, beta_gpu_slice.size, beta_gpu_slice.gpudata, 1, thetaL_t_gpu.gpudata, 1) -# mul_bcast(thetaL_t_gpu,thetaL_t_gpu,thetaL_t_gpu,thetaL_t_gpu.size) -# mul_bcast(thetaL_t_gpu,thetaL_t_gpu,psi0p_gpu,thetaL_t_gpu.size) -# cublas.cublasDaxpy(self.cublas_handle, dL_dthetaL_gpu.size, output_dim/2.0, thetaL_t_gpu.gpudata, 1, dL_dthetaL_gpu.gpudata, 1) -# -# thetaL_t_gpu.fill(0.) -# sum_axis(thetaL_t_gpu, psi2R_gpu, nSlice, num_inducing*num_inducing) -# mul_bcast(thetaL_t_gpu,thetaL_t_gpu,beta_gpu_slice,thetaL_t_gpu.size) -# mul_bcast(thetaL_t_gpu,thetaL_t_gpu,beta_gpu_slice,thetaL_t_gpu.size) -# cublas.cublasDaxpy(self.cublas_handle, dL_dthetaL_gpu.size, -1.0, thetaL_t_gpu.gpudata, 1, dL_dthetaL_gpu.gpudata, 1) -# -# cublas.cublasDgemm(self.cublas_handle, 'T', 'T', output_dim, nSlice, num_inducing, -1.0, v_gpu.gpudata, num_inducing, betapsi1_gpu.gpudata, nSlice, 0.0, betaYT2_gpu.gpudata, output_dim) -# mul_bcast(betaYT2_gpu,betaYT2_gpu,betaYT_gpu_slice,betaYT2_gpu.size) -# sum_axis(dL_dthetaL_gpu, betaYT2_gpu, 1, output_dim) + if het_noise: + betaY = betaYT_gpu_slice.get() + dL_dthetaL = ((np.square(betaY)).sum(axis=-1) + np.square(beta)*(output_dim*psi0p_gpu.get())-output_dim*beta)/2. + dL_dthetaL += -beta*beta*cublas.cublasDdot(self.cublas_handle,dL_dpsi2R_gpu.size, dL_dpsi2R_gpu.gpudata,1,psi2p_gpu.gpudata,1) + dL_dthetaL += -beta*(betaY*np.dot(psi1p_gpu.get(),v_gpu.get())).sum(axis=-1) if kern.useGPU: dL_dpsi0 = dL_dpsi0_gpu @@ -548,10 +489,11 @@ class VarDTC_GPU(object): dL_dpsi2 = dL_dpsi2_gpu else: dL_dpsi2 = dL_dpsi2_gpu.get() - if het_noise: - dL_dthetaL = dL_dthetaL_gpu.get() - else: - dL_dthetaL = gpuarray.sum(dL_dthetaL_gpu).get() + if not het_noise: + if isEnd: + dL_dthetaL = self.midRes['dL_dthetaL'] + else: + dL_dthetaL = 0. if uncertain_inputs: grad_dict = {'dL_dpsi0':dL_dpsi0, 'dL_dpsi1':dL_dpsi1, diff --git a/GPy/inference/latent_function_inference/var_dtc_parallel.py b/GPy/inference/latent_function_inference/var_dtc_parallel.py index da7e67cb..f2a3c3fd 100644 --- a/GPy/inference/latent_function_inference/var_dtc_parallel.py +++ b/GPy/inference/latent_function_inference/var_dtc_parallel.py @@ -116,7 +116,7 @@ class VarDTC_minibatch(object): if het_noise: psi2_full += beta_slice*np.outer(psi1,psi1) else: - psi2_full += np.outer(psi1.T,psi1.T) + psi2_full += np.dot(psi1.T,psi1) if not het_noise: psi0_full *= beta @@ -176,7 +176,7 @@ class VarDTC_minibatch(object): # Compute dL_dthetaL for uncertian input and non-heter noise #====================================================================== - if uncertain_inputs and not het_noise: + if not het_noise: dL_dthetaL = (YRY_full*beta + beta*output_dim*psi0_full - num_data*output_dim*beta)/2. - beta*(dL_dpsi2R*psi2_full).sum() - beta*(v.T*psi1Y_full).sum() self.midRes['dL_dthetaL'] = dL_dthetaL @@ -265,14 +265,10 @@ class VarDTC_minibatch(object): dL_dthetaL = ((np.square(betaY)).sum(axis=-1) + np.square(beta)*(output_dim*psi0)-output_dim*beta)/2. - np.square(beta)*psiR- (betaY*np.dot(betapsi1,v)).sum(axis=-1) else: - if uncertain_inputs: - if isEnd: - dL_dthetaL = self.midRes['dL_dthetaL'] - else: - dL_dthetaL = 0. + if isEnd: + dL_dthetaL = self.midRes['dL_dthetaL'] else: - psiR = np.einsum('nm,no,mo->',psi1,psi1,dL_dpsi2R) - dL_dthetaL = ((np.square(betaY)).sum() + beta*beta*output_dim*(psi0.sum())-num_slice*output_dim*beta)/2. - beta*beta*psiR- (betaY*np.dot(betapsi1,v)).sum() + dL_dthetaL = 0. if uncertain_inputs: grad_dict = {'dL_dpsi0':dL_dpsi0, diff --git a/GPy/models/ss_gplvm.py b/GPy/models/ss_gplvm.py index dba12700..7eff3f89 100644 --- a/GPy/models/ss_gplvm.py +++ b/GPy/models/ss_gplvm.py @@ -42,7 +42,8 @@ class SSGPLVM(SparseGP): X_variance = np.random.uniform(0,.1,X.shape) gamma = np.empty_like(X, order='F') # The posterior probabilities of the binary variable in the variational approximation - gamma[:] = 0.5 + 0.01 * np.random.randn(X.shape[0], input_dim) + #gamma[:] = 0.5 + 0.01 * np.random.randn(X.shape[0], input_dim) + gamma[:] = 0.5 if group_spike: gamma[:] = gamma.mean(axis=0) diff --git a/GPy/util/linalg_gpu.py b/GPy/util/linalg_gpu.py index cd656dd3..cf6d483e 100644 --- a/GPy/util/linalg_gpu.py +++ b/GPy/util/linalg_gpu.py @@ -19,8 +19,8 @@ try: strideSum = ReductionKernel(np.float64, neutral="0", reduce_expr="a+b", map_expr="i%step==0?x[i]:0", arguments="double *x, int step") - # np.trace(np.dot(A,B)) (also equivalent to (A*B.T).sum() ) A - n x m, B - m x n - traceDot = ReductionKernel(np.float64, neutral="0", reduce_expr="a+b", map_expr="A[i]*B[(i%n)*m+i/n]", arguments="double *A, double *B, int n, int m") + # np.trace(np.dot(A,B)) (also equivalent to (A*B.T).sum() ) A - a1 x a2, B - a2 x a1 + traceDot = ReductionKernel(np.float64, neutral="0", reduce_expr="a+b", map_expr="A[i]*B[(i%a1)*a2+i/a1]", arguments="double *A, double *B, int a1, int a2") #======================================================================================= # Element-wise functions From d911766624eb7bc64733471a374674d090b30d43 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Wed, 14 May 2014 15:07:39 +0100 Subject: [PATCH 04/55] [mpi] add mpi into ssgplvm --- .../var_dtc_parallel.py | 144 +++++++--- GPy/models/ss_gplvm.py | 268 ++---------------- GPy/plotting/matplot_dep/variational_plots.py | 2 +- GPy/util/__init__.py | 1 + GPy/util/caching.py | 3 +- GPy/util/mpi.py | 35 +++ 6 files changed, 165 insertions(+), 288 deletions(-) create mode 100644 GPy/util/mpi.py diff --git a/GPy/inference/latent_function_inference/var_dtc_parallel.py b/GPy/inference/latent_function_inference/var_dtc_parallel.py index f2a3c3fd..ef298a9b 100644 --- a/GPy/inference/latent_function_inference/var_dtc_parallel.py +++ b/GPy/inference/latent_function_inference/var_dtc_parallel.py @@ -9,6 +9,11 @@ import numpy as np from ...util.misc import param_to_array log_2_pi = np.log(2*np.pi) +try: + from mpi4py import MPI +except: + pass + class VarDTC_minibatch(object): """ An object for inference when the likelihood is Gaussian, but we want to do sparse inference. @@ -20,9 +25,10 @@ class VarDTC_minibatch(object): """ const_jitter = 1e-6 - def __init__(self, batchsize, limit=1): + def __init__(self, batchsize, limit=1, mpi_comm=None): self.batchsize = batchsize + self.mpi_comm = mpi_comm # Cache functions from ...util.caching import Cacher @@ -51,42 +57,23 @@ class VarDTC_minibatch(object): else: return jitchol(tdot(Y)) - def inference_likelihood(self, kern, X, Z, likelihood, Y): - """ - The first phase of inference: - Compute: log-likelihood, dL_dKmm - - Cached intermediate results: Kmm, KmmInv, - """ - + def gatherPsiStat(self, kern, X, Z, Y, beta, uncertain_inputs, het_noise): + num_inducing = Z.shape[0] num_data, output_dim = Y.shape - if isinstance(X, VariationalPosterior): - uncertain_inputs = True - else: - uncertain_inputs = False - - #see whether we've got a different noise variance for each datum - beta = 1./np.fmax(likelihood.variance, 1e-6) - het_noise = beta.size > 1 - if het_noise: - self.batchsize = 1 - # VVT_factor is a matrix such that tdot(VVT_factor) = VVT...this is for efficiency! - #self.YYTfactor = beta*self.get_YYTfactor(Y) - YYT_factor = Y trYYT = self.get_trYYT(Y) - + psi2_full = np.zeros((num_inducing,num_inducing)) psi1Y_full = np.zeros((output_dim,num_inducing)) # DxM - psi0_full = 0 - YRY_full = 0 + psi0_full = 0. + YRY_full = 0. for n_start in xrange(0,num_data,self.batchsize): n_end = min(self.batchsize+n_start, num_data) - Y_slice = YYT_factor[n_start:n_end] + Y_slice = Y[n_start:n_end] X_slice = X[n_start:n_end] if uncertain_inputs: @@ -123,7 +110,47 @@ class VarDTC_minibatch(object): psi1Y_full *= beta psi2_full *= beta YRY_full = trYYT*beta + + if self.mpi_comm != None: + psi0_all = np.array(psi0_full) + psi1Y_all = psi1Y_full.copy() + psi2_all = psi2_full.copy() + YRY_all = np.array(YRY_full) + self.mpi_comm.Allreduce([psi0_full, MPI.DOUBLE], [psi0_all, MPI.DOUBLE]) + self.mpi_comm.Allreduce([psi1Y_full, MPI.DOUBLE], [psi1Y_all, MPI.DOUBLE]) + self.mpi_comm.Allreduce([psi2_full, MPI.DOUBLE], [psi2_all, MPI.DOUBLE]) + self.mpi_comm.Allreduce([YRY_full, MPI.DOUBLE], [YRY_all, MPI.DOUBLE]) + return psi0_all, psi1Y_all, psi2_all, YRY_all + return psi0_full, psi1Y_full, psi2_full, YRY_full + + def inference_likelihood(self, kern, X, Z, likelihood, Y): + """ + The first phase of inference: + Compute: log-likelihood, dL_dKmm + + Cached intermediate results: Kmm, KmmInv, + """ + + num_data, output_dim = Y.shape + if self.mpi_comm != None: + num_data_all = np.array(num_data,dtype=np.int32) + self.mpi_comm.Allreduce([np.int32(num_data), MPI.INT], [num_data_all, MPI.INT]) + num_data = num_data_all + + if isinstance(X, VariationalPosterior): + uncertain_inputs = True + else: + uncertain_inputs = False + + #see whether we've got a different noise variance for each datum + beta = 1./np.fmax(likelihood.variance, 1e-6) + het_noise = beta.size > 1 + if het_noise: + self.batchsize = 1 + + psi0_full, psi1Y_full, psi2_full, YRY_full = self.gatherPsiStat(kern, X, Z, Y, beta, uncertain_inputs, het_noise) + #====================================================================== # Compute Common Components #====================================================================== @@ -212,7 +239,6 @@ class VarDTC_minibatch(object): isEnd = False self.batch_pos = n_end - num_slice = n_end-n_start Y_slice = YYT_factor[n_start:n_end] X_slice = X[n_start:n_end] @@ -283,28 +309,32 @@ class VarDTC_minibatch(object): return isEnd, (n_start,n_end), grad_dict -def update_gradients(model): - model._log_marginal_likelihood, dL_dKmm, model.posterior = model.inference_method.inference_likelihood(model.kern, model.X, model.Z, model.likelihood, model.Y) +def update_gradients(model, mpi_comm=None): + if mpi_comm == None: + Y = model.Y + X = model.X + else: + Y = model.Y_local + X = model.X_local + + model._log_marginal_likelihood, dL_dKmm, model.posterior = model.inference_method.inference_likelihood(model.kern, X, model.Z, model.likelihood, Y) het_noise = model.likelihood.variance.size > 1 if het_noise: dL_dthetaL = np.empty((model.Y.shape[0],)) else: - dL_dthetaL = 0 - - #gradients w.r.t. kernel - model.kern.update_gradients_full(dL_dKmm, model.Z, None) + dL_dthetaL = np.float64(0.) + kern_grad = model.kern.gradient.copy() - - #gradients w.r.t. Z - model.Z.gradient = model.kern.gradients_X(dL_dKmm, model.Z) + kern_grad[:] = 0. + model.Z.gradient = 0. isEnd = False while not isEnd: - isEnd, n_range, grad_dict = model.inference_method.inference_minibatch(model.kern, model.X, model.Z, model.likelihood, model.Y) + isEnd, n_range, grad_dict = model.inference_method.inference_minibatch(model.kern, X, model.Z, model.likelihood, Y) if isinstance(model.X, VariationalPosterior): - X_slice = model.X[n_range[0]:n_range[1]] + X_slice = model.X[model.Y_range[0]+n_range[0]:model.Y_range[0]+n_range[1]] #gradients w.r.t. kernel model.kern.update_gradients_expectations(variational_posterior=X_slice, Z=model.Z, dL_dpsi0=grad_dict['dL_dpsi0'], dL_dpsi1=grad_dict['dL_dpsi1'], dL_dpsi2=grad_dict['dL_dpsi2']) @@ -322,14 +352,36 @@ def update_gradients(model): dL_dthetaL[n_range[0]:n_range[1]] = grad_dict['dL_dthetaL'] else: dL_dthetaL += grad_dict['dL_dthetaL'] - - # Set the gradients w.r.t. kernel - model.kern.gradient = kern_grad - - # Update Log-likelihood - model._log_marginal_likelihood -= model.variational_prior.KL_divergence(model.X) - # update for the KL divergence - model.variational_prior.update_gradients_KL(model.X) + # Gather the gradients from multiple MPI nodes + if mpi_comm != None: + if het_noise: + assert False, "Not implemented!" + kern_grad_all = kern_grad.copy() + Z_grad_all = model.Z.gradient.copy() + mpi_comm.Allreduce([kern_grad, MPI.DOUBLE], [kern_grad_all, MPI.DOUBLE]) + mpi_comm.Allreduce([model.Z.gradient, MPI.DOUBLE], [Z_grad_all, MPI.DOUBLE]) + kern_grad = kern_grad_all + model.Z.gradient = Z_grad_all + + #gradients w.r.t. kernel + model.kern.update_gradients_full(dL_dKmm, model.Z, None) + model.kern.gradient += kern_grad + + #gradients w.r.t. Z + model.Z.gradient += model.kern.gradients_X(dL_dKmm, model.Z) + + # Update Log-likelihood + KL_div = model.variational_prior.KL_divergence(X) + # update for the KL divergence + model.variational_prior.update_gradients_KL(X) + + if mpi_comm != None: + KL_div_all = np.array(KL_div) + mpi_comm.Allreduce([np.float64(KL_div), MPI.DOUBLE], [KL_div_all, MPI.DOUBLE]) + KL_div = KL_div_all + [mpi_comm.Allgatherv([pp.copy(), MPI.DOUBLE], [pa, (model.Y_list*pa.shape[-1], None), MPI.DOUBLE]) for pp,pa in zip(model.get_X_gradients(X),model.get_X_gradients(model.X))] + model._log_marginal_likelihood -= KL_div + # dL_dthetaL model.likelihood.update_gradients(dL_dthetaL) diff --git a/GPy/models/ss_gplvm.py b/GPy/models/ss_gplvm.py index 7eff3f89..cc57b191 100644 --- a/GPy/models/ss_gplvm.py +++ b/GPy/models/ss_gplvm.py @@ -28,8 +28,10 @@ class SSGPLVM(SparseGP): """ def __init__(self, Y, input_dim, X=None, X_variance=None, init='PCA', num_inducing=10, - Z=None, kernel=None, inference_method=None, likelihood=None, name='Spike-and-Slab GPLVM', group_spike=False, **kwargs): + Z=None, kernel=None, inference_method=None, likelihood=None, name='Spike-and-Slab GPLVM', group_spike=False, mpi_comm=None, **kwargs): + self.mpi_comm = mpi_comm + if X == None: from ..util.initialization import initialize_latent X, fracs = initialize_latent(init, input_dim, Y) @@ -52,19 +54,25 @@ class SSGPLVM(SparseGP): Z = np.random.permutation(X.copy())[:num_inducing] assert Z.shape[1] == X.shape[1] + pi = np.empty((input_dim)) + pi[:] = 0.5 + + if mpi_comm != None: + mpi_comm.Bcast(X, root=0) + mpi_comm.Bcast(fracs, root=0) + mpi_comm.Bcast(X_variance, root=0) + mpi_comm.Bcast(gamma, root=0) + mpi_comm.Bcast(Z, root=0) + mpi_comm.Bcast(pi, root=0) + if likelihood is None: likelihood = Gaussian() if kernel is None: kernel = kern.RBF(input_dim, lengthscale=fracs, ARD=True) # + kern.white(input_dim) - - pi = np.empty((input_dim)) - pi[:] = 0.5 + self.variational_prior = SpikeAndSlabPrior(pi=pi) # the prior probability of the latent binary variable b - X = np.asfortranarray(X) - X_variance = np.asfortranarray(X_variance) - gamma = np.asfortranarray(gamma) X = SpikeAndSlabPosterior(X, X_variance, gamma) if group_spike: @@ -75,13 +83,25 @@ class SSGPLVM(SparseGP): self.add_parameter(self.X, index=0) self.add_parameter(self.variational_prior) + if mpi_comm != None: + from ..util.mpi import divide_data + Y_start, Y_end, Y_list = divide_data(Y.shape[0], mpi_comm) + self.Y_local = self.Y[Y_start:Y_end] + self.X_local = self.X[Y_start:Y_end] + self.Y_range = (Y_start, Y_end) + self.Y_list = np.array(Y_list) + def set_X_gradients(self, X, X_grad): """Set the gradients of the posterior distribution of X in its specific form.""" X.mean.gradient, X.variance.gradient, X.binary_prob.gradient = X_grad + + def get_X_gradients(self, X): + """Get the gradients of the posterior distribution of X in its specific form.""" + return X.mean.gradient, X.variance.gradient, X.binary_prob.gradient def parameters_changed(self): if isinstance(self.inference_method, VarDTC_GPU) or isinstance(self.inference_method, VarDTC_minibatch): - update_gradients(self) + update_gradients(self, mpi_comm=self.mpi_comm) return super(SSGPLVM, self).parameters_changed() @@ -105,235 +125,3 @@ class SSGPLVM(SparseGP): return dim_reduction_plots.plot_latent(self, plot_inducing=plot_inducing, *args, **kwargs) - def do_test_latents(self, Y): - """ - Compute the latent representation for a set of new points Y - - Notes: - This will only work with a univariate Gaussian likelihood (for now) - """ - assert not self.likelihood.is_heteroscedastic - N_test = Y.shape[0] - input_dim = self.Z.shape[1] - means = np.zeros((N_test, input_dim)) - covars = np.zeros((N_test, input_dim)) - - dpsi0 = -0.5 * self.output_dim * self.likelihood.precision - dpsi2 = self.dL_dpsi2[0][None, :, :] # TODO: this may change if we ignore het. likelihoods - V = self.likelihood.precision * Y - - #compute CPsi1V - if self.Cpsi1V is None: - psi1V = np.dot(self.psi1.T, self.likelihood.V) - tmp, _ = linalg.dtrtrs(self._Lm, np.asfortranarray(psi1V), lower=1, trans=0) - tmp, _ = linalg.dpotrs(self.LB, tmp, lower=1) - self.Cpsi1V, _ = linalg.dtrtrs(self._Lm, tmp, lower=1, trans=1) - - dpsi1 = np.dot(self.Cpsi1V, V.T) - - start = np.zeros(self.input_dim * 2) - - for n, dpsi1_n in enumerate(dpsi1.T[:, :, None]): - args = (self.kern, self.Z, dpsi0, dpsi1_n.T, dpsi2) - xopt, fopt, neval, status = SCG(f=latent_cost, gradf=latent_grad, x=start, optargs=args, display=False) - - mu, log_S = xopt.reshape(2, 1, -1) - means[n] = mu[0].copy() - covars[n] = np.exp(log_S[0]).copy() - - return means, covars - - def dmu_dX(self, Xnew): - """ - Calculate the gradient of the prediction at Xnew w.r.t Xnew. - """ - dmu_dX = np.zeros_like(Xnew) - for i in range(self.Z.shape[0]): - dmu_dX += self.kern.dK_dX(self.Cpsi1Vf[i:i + 1, :], Xnew, self.Z[i:i + 1, :]) - return dmu_dX - - def dmu_dXnew(self, Xnew): - """ - Individual gradient of prediction at Xnew w.r.t. each sample in Xnew - """ - dK_dX = np.zeros((Xnew.shape[0], self.num_inducing)) - ones = np.ones((1, 1)) - for i in range(self.Z.shape[0]): - dK_dX[:, i] = self.kern.dK_dX(ones, Xnew, self.Z[i:i + 1, :]).sum(-1) - return np.dot(dK_dX, self.Cpsi1Vf) - - def plot_steepest_gradient_map(self, fignum=None, ax=None, which_indices=None, labels=None, data_labels=None, data_marker='o', data_s=40, resolution=20, aspect='auto', updates=False, ** kwargs): - input_1, input_2 = significant_dims = most_significant_input_dimensions(self, which_indices) - - X = np.zeros((resolution ** 2, self.input_dim)) - indices = np.r_[:X.shape[0]] - if labels is None: - labels = range(self.output_dim) - - def plot_function(x): - X[:, significant_dims] = x - dmu_dX = self.dmu_dXnew(X) - argmax = np.argmax(dmu_dX, 1) - return dmu_dX[indices, argmax], np.array(labels)[argmax] - - if ax is None: - fig = pyplot.figure(num=fignum) - ax = fig.add_subplot(111) - - if data_labels is None: - data_labels = np.ones(self.num_data) - ulabels = [] - for lab in data_labels: - if not lab in ulabels: - ulabels.append(lab) - marker = itertools.cycle(list(data_marker)) - from GPy.util import Tango - for i, ul in enumerate(ulabels): - if type(ul) is np.string_: - this_label = ul - elif type(ul) is np.int64: - this_label = 'class %i' % ul - else: - this_label = 'class %i' % i - m = marker.next() - index = np.nonzero(data_labels == ul)[0] - x = self.X[index, input_1] - y = self.X[index, input_2] - ax.scatter(x, y, marker=m, s=data_s, color=Tango.nextMedium(), label=this_label) - - ax.set_xlabel('latent dimension %i' % input_1) - ax.set_ylabel('latent dimension %i' % input_2) - - from matplotlib.cm import get_cmap - from GPy.util.latent_space_visualizations.controllers.imshow_controller import ImAnnotateController - if not 'cmap' in kwargs.keys(): - kwargs.update(cmap=get_cmap('jet')) - controller = ImAnnotateController(ax, - plot_function, - tuple(self.X.min(0)[:, significant_dims]) + tuple(self.X.max(0)[:, significant_dims]), - resolution=resolution, - aspect=aspect, - **kwargs) - ax.legend() - ax.figure.tight_layout() - if updates: - pyplot.show() - clear = raw_input('Enter to continue') - if clear.lower() in 'yes' or clear == '': - controller.deactivate() - return controller.view - - def plot_X_1d(self, fignum=None, ax=None, colors=None): - """ - Plot latent space X in 1D: - - - if fig is given, create input_dim subplots in fig and plot in these - - if ax is given plot input_dim 1D latent space plots of X into each `axis` - - if neither fig nor ax is given create a figure with fignum and plot in there - - colors: - colors of different latent space dimensions input_dim - - """ - import pylab - if ax is None: - fig = pylab.figure(num=fignum, figsize=(8, min(12, (2 * self.X.shape[1])))) - if colors is None: - colors = pylab.gca()._get_lines.color_cycle - pylab.clf() - else: - colors = iter(colors) - plots = [] - x = np.arange(self.X.shape[0]) - for i in range(self.X.shape[1]): - if ax is None: - a = fig.add_subplot(self.X.shape[1], 1, i + 1) - elif isinstance(ax, (tuple, list)): - a = ax[i] - else: - raise ValueError("Need one ax per latent dimnesion input_dim") - a.plot(self.X, c='k', alpha=.3) - plots.extend(a.plot(x, self.X.T[i], c=colors.next(), label=r"$\mathbf{{X_{{{}}}}}$".format(i))) - a.fill_between(x, - self.X.T[i] - 2 * np.sqrt(self.X_variance.T[i]), - self.X.T[i] + 2 * np.sqrt(self.X_variance.T[i]), - facecolor=plots[-1].get_color(), - alpha=.3) - a.legend(borderaxespad=0.) - a.set_xlim(x.min(), x.max()) - if i < self.X.shape[1] - 1: - a.set_xticklabels('') - pylab.draw() - if ax is None: - fig.tight_layout(h_pad=.01) # , rect=(0, 0, 1, .95)) - return fig - - def getstate(self): - """ - Get the current state of the class, - here just all the indices, rest can get recomputed - """ - return SparseGP._getstate(self) + [self.init] - - def setstate(self, state): - self._const_jitter = None - self.init = state.pop() - SparseGP._setstate(self, state) - - -def latent_cost_and_grad(mu_S, kern, Z, dL_dpsi0, dL_dpsi1, dL_dpsi2): - """ - objective function for fitting the latent variables for test points - (negative log-likelihood: should be minimised!) - """ - mu, log_S = mu_S.reshape(2, 1, -1) - S = np.exp(log_S) - - psi0 = kern.psi0(Z, mu, S) - psi1 = kern.psi1(Z, mu, S) - psi2 = kern.psi2(Z, mu, S) - - lik = dL_dpsi0 * psi0 + np.dot(dL_dpsi1.flatten(), psi1.flatten()) + np.dot(dL_dpsi2.flatten(), psi2.flatten()) - 0.5 * np.sum(np.square(mu) + S) + 0.5 * np.sum(log_S) - - mu0, S0 = kern.dpsi0_dmuS(dL_dpsi0, Z, mu, S) - mu1, S1 = kern.dpsi1_dmuS(dL_dpsi1, Z, mu, S) - mu2, S2 = kern.dpsi2_dmuS(dL_dpsi2, Z, mu, S) - - dmu = mu0 + mu1 + mu2 - mu - # dS = S0 + S1 + S2 -0.5 + .5/S - dlnS = S * (S0 + S1 + S2 - 0.5) + .5 - return -lik, -np.hstack((dmu.flatten(), dlnS.flatten())) - -def latent_cost(mu_S, kern, Z, dL_dpsi0, dL_dpsi1, dL_dpsi2): - """ - objective function for fitting the latent variables (negative log-likelihood: should be minimised!) - This is the same as latent_cost_and_grad but only for the objective - """ - mu, log_S = mu_S.reshape(2, 1, -1) - S = np.exp(log_S) - - psi0 = kern.psi0(Z, mu, S) - psi1 = kern.psi1(Z, mu, S) - psi2 = kern.psi2(Z, mu, S) - - lik = dL_dpsi0 * psi0 + np.dot(dL_dpsi1.flatten(), psi1.flatten()) + np.dot(dL_dpsi2.flatten(), psi2.flatten()) - 0.5 * np.sum(np.square(mu) + S) + 0.5 * np.sum(log_S) - return -float(lik) - -def latent_grad(mu_S, kern, Z, dL_dpsi0, dL_dpsi1, dL_dpsi2): - """ - This is the same as latent_cost_and_grad but only for the grad - """ - mu, log_S = mu_S.reshape(2, 1, -1) - S = np.exp(log_S) - - mu0, S0 = kern.dpsi0_dmuS(dL_dpsi0, Z, mu, S) - mu1, S1 = kern.dpsi1_dmuS(dL_dpsi1, Z, mu, S) - mu2, S2 = kern.dpsi2_dmuS(dL_dpsi2, Z, mu, S) - - dmu = mu0 + mu1 + mu2 - mu - # dS = S0 + S1 + S2 -0.5 + .5/S - dlnS = S * (S0 + S1 + S2 - 0.5) + .5 - - return -np.hstack((dmu.flatten(), dlnS.flatten())) - - diff --git a/GPy/plotting/matplot_dep/variational_plots.py b/GPy/plotting/matplot_dep/variational_plots.py index 27cb4051..e97f001b 100644 --- a/GPy/plotting/matplot_dep/variational_plots.py +++ b/GPy/plotting/matplot_dep/variational_plots.py @@ -93,7 +93,7 @@ def plot_SpikeSlab(parameterized, fignum=None, ax=None, colors=None, side_by_sid a.set_xticklabels('') # binary prob plot a = fig.add_subplot(*sub2) - a.bar(x,gamma[:,i],bottom=0.,linewidth=0,align='center') + a.bar(x,gamma[:,i],bottom=0.,linewidth=0,width=1.0,align='center') a.set_xlim(x.min(), x.max()) a.set_ylim([0.,1.]) pb.draw() diff --git a/GPy/util/__init__.py b/GPy/util/__init__.py index 77312278..3a9fbdb8 100644 --- a/GPy/util/__init__.py +++ b/GPy/util/__init__.py @@ -16,6 +16,7 @@ import diag import initialization import multioutput import linalg_gpu +import mpi try: import sympy diff --git a/GPy/util/caching.py b/GPy/util/caching.py index 1738cad1..8c26629c 100644 --- a/GPy/util/caching.py +++ b/GPy/util/caching.py @@ -102,7 +102,8 @@ class Cacher(object): return Cacher(self.operation, self.limit, self.ignore_args, self.force_kwargs) def __getstate__(self, memo=None): - raise NotImplementedError, "Trying to pickle Cacher object with function {}, pickling functions not possible.".format(str(self.operation)) + return (self.limit) +# raise NotImplementedError, "Trying to pickle Cacher object with function {}, pickling functions not possible.".format(str(self.operation)) def __setstate__(self, memo=None): raise NotImplementedError, "Trying to pickle Cacher object with function {}, pickling functions not possible.".format(str(self.operation)) diff --git a/GPy/util/mpi.py b/GPy/util/mpi.py new file mode 100644 index 00000000..e3d7d928 --- /dev/null +++ b/GPy/util/mpi.py @@ -0,0 +1,35 @@ +""" +The tools for mpi +""" + +try: + import numpy as np + from mpi4py import MPI + numpy_to_MPI_typemap = { + np.dtype(np.float64) : MPI.DOUBLE, + np.dtype(np.float32) : MPI.FLOAT, + np.dtype(np.int) : MPI.INT, + np.dtype(np.int8) : MPI.CHAR, + np.dtype(np.uint8) : MPI.UNSIGNED_CHAR, + np.dtype(np.int32) : MPI.INT, + np.dtype(np.uint32) : MPI.UNSIGNED_INT, + } +except: + pass + +def divide_data(datanum, comm): + + residue = (datanum)%comm.size + datanum_list = np.empty((comm.size),dtype=np.int32) + for i in xrange(comm.size): + if i Date: Thu, 15 May 2014 11:43:29 +0100 Subject: [PATCH 05/55] [ssgplvm] linear kernel --- GPy/core/parameterization/parameter_core.py | 2 +- GPy/core/parameterization/variational.py | 4 +- .../var_dtc_parallel.py | 20 ++- GPy/kern/_src/linear.py | 62 ++------ GPy/kern/_src/psi_comp/linear_psi_comp.py | 136 ++++++++++++------ GPy/kern/_src/rbf.py | 7 +- GPy/models/ss_gplvm.py | 35 +++-- 7 files changed, 162 insertions(+), 104 deletions(-) diff --git a/GPy/core/parameterization/parameter_core.py b/GPy/core/parameterization/parameter_core.py index 68140763..d54cf208 100644 --- a/GPy/core/parameterization/parameter_core.py +++ b/GPy/core/parameterization/parameter_core.py @@ -169,7 +169,7 @@ class Pickleable(object): else: pickle.dump(self, f, protocol) - #=========================================================================== + #=========================================================================== # copy and pickling #=========================================================================== def copy(self): diff --git a/GPy/core/parameterization/variational.py b/GPy/core/parameterization/variational.py index 3730baed..4b2b2e4e 100644 --- a/GPy/core/parameterization/variational.py +++ b/GPy/core/parameterization/variational.py @@ -160,7 +160,7 @@ class SpikeAndSlabPosterior(VariationalPosterior): else: return super(VariationalPrior, self).__getitem__(s) - def plot(self, *args): + def plot(self, *args, **kwargs): """ Plot latent space X in 1D: @@ -169,4 +169,4 @@ class SpikeAndSlabPosterior(VariationalPosterior): import sys assert "matplotlib" in sys.modules, "matplotlib package has not been imported." from ...plotting.matplot_dep import variational_plots - return variational_plots.plot_SpikeSlab(self,*args) + return variational_plots.plot_SpikeSlab(self,*args, **kwargs) diff --git a/GPy/inference/latent_function_inference/var_dtc_parallel.py b/GPy/inference/latent_function_inference/var_dtc_parallel.py index ef298a9b..727d7e1c 100644 --- a/GPy/inference/latent_function_inference/var_dtc_parallel.py +++ b/GPy/inference/latent_function_inference/var_dtc_parallel.py @@ -29,6 +29,7 @@ class VarDTC_minibatch(object): self.batchsize = batchsize self.mpi_comm = mpi_comm + self.limit = limit # Cache functions from ...util.caching import Cacher @@ -37,6 +38,20 @@ class VarDTC_minibatch(object): self.midRes = {} self.batch_pos = 0 # the starting position of the current mini-batch + + def __getstate__(self): + # has to be overridden, as Cacher objects cannot be pickled. + return self.batchsize, self.limit + + def __setstate__(self, state): + # has to be overridden, as Cacher objects cannot be pickled. + self.batchsize, self.limit = state + self.mpi_comm = None + self.midRes = {} + self.batch_pos = 0 + from ...util.caching import Cacher + self.get_trYYT = Cacher(self._get_trYYT, self.limit) + self.get_YYTfactor = Cacher(self._get_YYTfactor, self.limit) def set_limit(self, limit): self.get_trYYT.limit = limit @@ -334,7 +349,10 @@ def update_gradients(model, mpi_comm=None): while not isEnd: isEnd, n_range, grad_dict = model.inference_method.inference_minibatch(model.kern, X, model.Z, model.likelihood, Y) if isinstance(model.X, VariationalPosterior): - X_slice = model.X[model.Y_range[0]+n_range[0]:model.Y_range[0]+n_range[1]] + if mpi_comm ==None: + X_slice = model.X[n_range[0]:n_range[1]] + else: + X_slice = model.X[model.Y_range[0]+n_range[0]:model.Y_range[0]+n_range[1]] #gradients w.r.t. kernel model.kern.update_gradients_expectations(variational_posterior=X_slice, Z=model.Z, dL_dpsi0=grad_dict['dL_dpsi0'], dL_dpsi1=grad_dict['dL_dpsi1'], dL_dpsi2=grad_dict['dL_dpsi2']) diff --git a/GPy/kern/_src/linear.py b/GPy/kern/_src/linear.py index 007649b0..6ef71724 100644 --- a/GPy/kern/_src/linear.py +++ b/GPy/kern/_src/linear.py @@ -52,6 +52,9 @@ class Linear(Kern): self.variances = Param('variances', variances, Logexp()) self.add_parameter(self.variances) + + def set_for_SpikeAndSlab(self): + self.psicomp = linear_psi_comp.PSICOMP_SSLinear() @Cache_this(limit=2) def K(self, X, X2=None): @@ -107,35 +110,20 @@ class Linear(Kern): def psi0(self, Z, variational_posterior): if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - gamma = variational_posterior.binary_prob - mu = variational_posterior.mean - S = variational_posterior.variance - - return np.einsum('q,nq,nq->n',self.variances,gamma,np.square(mu)+S) -# return (self.variances*gamma*(np.square(mu)+S)).sum(axis=1) + return self.psicomp.psicomputations(self.variances, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[0] else: return np.sum(self.variances * self._mu2S(variational_posterior), 1) def psi1(self, Z, variational_posterior): if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - gamma = variational_posterior.binary_prob - mu = variational_posterior.mean - return np.einsum('nq,q,mq,nq->nm',gamma,self.variances,Z,mu) -# return (self.variances*gamma*mu).sum(axis=1) + return self.psicomp.psicomputations(self.variances, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[1] else: return self.K(variational_posterior.mean, Z) #the variance, it does nothing @Cache_this(limit=1) def psi2(self, Z, variational_posterior): if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - gamma = variational_posterior.binary_prob - mu = variational_posterior.mean - S = variational_posterior.variance - mu2 = np.square(mu) - variances2 = np.square(self.variances) - tmp = np.einsum('nq,q,mq,nq->nm',gamma,self.variances,Z,mu) - return np.einsum('nq,q,mq,oq,nq->nmo',gamma,variances2,Z,Z,mu2+S)+\ - np.einsum('nm,no->nmo',tmp,tmp) - np.einsum('nq,q,mq,oq,nq->nmo',np.square(gamma),variances2,Z,Z,mu2) + return self.psicomp.psicomputations(self.variances, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[2] else: ZA = Z * self.variances ZAinner = self._ZAinner(variational_posterior, Z) @@ -143,17 +131,11 @@ class Linear(Kern): def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - gamma = variational_posterior.binary_prob - mu = variational_posterior.mean - S = variational_posterior.variance - mu2S = np.square(mu)+S - _dpsi2_dvariance, _, _, _, _ = linear_psi_comp._psi2computations(self.variances, Z, mu, S, gamma) - grad = np.einsum('n,nq,nq->q',dL_dpsi0,gamma,mu2S) + np.einsum('nm,nq,mq,nq->q',dL_dpsi1,gamma,Z,mu) +\ - np.einsum('nmo,nmoq->q',dL_dpsi2,_dpsi2_dvariance) + dL_dvar,_,_,_,_ = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variances, Z, variational_posterior) if self.ARD: - self.variances.gradient = grad + self.variances.gradient = dL_dvar else: - self.variances.gradient = grad.sum() + self.variances.gradient = dL_dvar.sum() else: #psi1 self.update_gradients_full(dL_dpsi1, variational_posterior.mean, Z) @@ -170,15 +152,8 @@ class Linear(Kern): def gradients_Z_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - gamma = variational_posterior.binary_prob - mu = variational_posterior.mean - S = variational_posterior.variance - _, _, _, _, _dpsi2_dZ = linear_psi_comp._psi2computations(self.variances, Z, mu, S, gamma) - - grad = np.einsum('nm,nq,q,nq->mq',dL_dpsi1,gamma, self.variances,mu) +\ - np.einsum('nmo,noq->mq',dL_dpsi2,_dpsi2_dZ) - - return grad + _,dL_dZ,_,_,_ = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variances, Z, variational_posterior) + return dL_dZ else: #psi1 grad = self.gradients_X(dL_dpsi1.T, Z, variational_posterior.mean) @@ -188,19 +163,8 @@ class Linear(Kern): def gradients_qX_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - gamma = variational_posterior.binary_prob - mu = variational_posterior.mean - S = variational_posterior.variance - mu2S = np.square(mu)+S - _, _dpsi2_dgamma, _dpsi2_dmu, _dpsi2_dS, _ = linear_psi_comp._psi2computations(self.variances, Z, mu, S, gamma) - - grad_gamma = np.einsum('n,q,nq->nq',dL_dpsi0,self.variances,mu2S) + np.einsum('nm,q,mq,nq->nq',dL_dpsi1,self.variances,Z,mu) +\ - np.einsum('nmo,nmoq->nq',dL_dpsi2,_dpsi2_dgamma) - grad_mu = np.einsum('n,nq,q,nq->nq',dL_dpsi0,gamma,2.*self.variances,mu) + np.einsum('nm,nq,q,mq->nq',dL_dpsi1,gamma,self.variances,Z) +\ - np.einsum('nmo,nmoq->nq',dL_dpsi2,_dpsi2_dmu) - grad_S = np.einsum('n,nq,q->nq',dL_dpsi0,gamma,self.variances) + np.einsum('nmo,nmoq->nq',dL_dpsi2,_dpsi2_dS) - - return grad_mu, grad_S, grad_gamma + _,_,dL_dmu, dL_dS, dL_dgamma = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variances, Z, variational_posterior) + return dL_dmu, dL_dS, dL_dgamma else: grad_mu, grad_S = np.zeros(variational_posterior.mean.shape), np.zeros(variational_posterior.mean.shape) # psi0 diff --git a/GPy/kern/_src/psi_comp/linear_psi_comp.py b/GPy/kern/_src/psi_comp/linear_psi_comp.py index 22147366..03483b6b 100644 --- a/GPy/kern/_src/psi_comp/linear_psi_comp.py +++ b/GPy/kern/_src/psi_comp/linear_psi_comp.py @@ -8,44 +8,100 @@ The package for the Psi statistics computation of the linear kernel for SSGPLVM import numpy as np from GPy.util.caching import Cache_this -#@Cache_this(limit=1) -def _psi2computations(variance, Z, mu, S, gamma): - """ - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - """ - # here are the "statistics" for psi1 and psi2 - # Produced intermediate results: - # _psi2 NxMxM - # _psi2_dvariance NxMxMxQ - # _psi2_dZ NxMxQ - # _psi2_dgamma NxMxMxQ - # _psi2_dmu NxMxMxQ - # _psi2_dS NxMxMxQ - - mu2 = np.square(mu) - gamma2 = np.square(gamma) - variance2 = np.square(variance) - mu2S = mu2+S # NxQ - common_sum = np.einsum('nq,q,mq,nq->nm',gamma,variance,Z,mu) # NxM - - _dpsi2_dvariance = np.einsum('nq,q,mq,oq->nmoq',2.*(gamma*mu2S-gamma2*mu2),variance,Z,Z)+\ - np.einsum('nq,mq,nq,no->nmoq',gamma,Z,mu,common_sum)+\ - np.einsum('nq,oq,nq,nm->nmoq',gamma,Z,mu,common_sum) - - _dpsi2_dgamma = np.einsum('q,mq,oq,nq->nmoq',variance2,Z,Z,(mu2S-2.*gamma*mu2))+\ - np.einsum('q,mq,nq,no->nmoq',variance,Z,mu,common_sum)+\ - np.einsum('q,oq,nq,nm->nmoq',variance,Z,mu,common_sum) - - _dpsi2_dmu = np.einsum('q,mq,oq,nq,nq->nmoq',variance2,Z,Z,mu,2.*(gamma-gamma2))+\ - np.einsum('nq,q,mq,no->nmoq',gamma,variance,Z,common_sum)+\ - np.einsum('nq,q,oq,nm->nmoq',gamma,variance,Z,common_sum) - - _dpsi2_dS = np.einsum('nq,q,mq,oq->nmoq',gamma,variance2,Z,Z) - - _dpsi2_dZ = 2.*(np.einsum('nq,q,mq,nq->nmq',gamma,variance2,Z,mu2S)+np.einsum('nq,q,nq,nm->nmq',gamma,variance,mu,common_sum) - -np.einsum('nq,q,mq,nq->nmq',gamma2,variance2,Z,mu2)) +class PSICOMP_SSLinear(object): + #@Cache_this(limit=1, ignore_args=(0,)) + def psicomputations(self, variance, Z, mu, S, gamma): + """ + Compute psi-statistics for ss-linear kernel + """ + # here are the "statistics" for psi0, psi1 and psi2 + # Produced intermediate results: + # psi0 N + # psi1 NxM + # psi2 MxM - return _dpsi2_dvariance, _dpsi2_dgamma, _dpsi2_dmu, _dpsi2_dS, _dpsi2_dZ \ No newline at end of file + psi0 = np.einsum('q,nq,nq->n',variance,gamma,np.square(mu)+S) + psi1 = np.einsum('nq,q,mq,nq->nm',gamma,variance,Z,mu) + mu2 = np.square(mu) + variances2 = np.square(variance) + tmp = np.einsum('nq,q,mq,nq->nm',gamma,variance,Z,mu) + psi2 = np.einsum('nq,q,mq,oq,nq->mo',gamma,variances2,Z,Z,mu2+S)+\ + np.einsum('nm,no->mo',tmp,tmp) - np.einsum('nq,q,mq,oq,nq->mo',np.square(gamma),variances2,Z,Z,mu2) + + return psi0, psi1, psi2 + + #@Cache_this(limit=1, ignore_args=(0,1,2,3)) + def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior): + mu = variational_posterior.mean + S = variational_posterior.variance + gamma = variational_posterior.binary_prob + + dL_dvar, dL_dgamma, dL_dmu, dL_dS, dL_dZ = self._psi2computations(dL_dpsi2, variance, Z, mu, S, gamma) + + # Compute for psi0 and psi1 + mu2S = np.square(mu)+S + dL_dvar += np.einsum('n,nq,nq->q',dL_dpsi0,gamma,mu2S) + np.einsum('nm,nq,mq,nq->q',dL_dpsi1,gamma,Z,mu) + dL_dgamma += np.einsum('n,q,nq->nq',dL_dpsi0,variance,mu2S) + np.einsum('nm,q,mq,nq->nq',dL_dpsi1,variance,Z,mu) + dL_dmu += np.einsum('n,nq,q,nq->nq',dL_dpsi0,gamma,2.*variance,mu) + np.einsum('nm,nq,q,mq->nq',dL_dpsi1,gamma,variance,Z) + dL_dS += np.einsum('n,nq,q->nq',dL_dpsi0,gamma,variance) + dL_dZ += np.einsum('nm,nq,q,nq->mq',dL_dpsi1,gamma, variance,mu) + + return dL_dvar, dL_dZ, dL_dmu, dL_dS, dL_dgamma + + def _psi2computations(self, dL_dpsi2, variance, Z, mu, S, gamma): + """ + Z - MxQ + mu - NxQ + S - NxQ + gamma - NxQ + """ + # here are the "statistics" for psi1 and psi2 + # Produced intermediate results: + # _psi2_dvariance Q + # _psi2_dZ MxQ + # _psi2_dgamma NxQ + # _psi2_dmu NxQ + # _psi2_dS NxQ + + mu2 = np.square(mu) + gamma2 = np.square(gamma) + variance2 = np.square(variance) + mu2S = mu2+S # NxQ + common_sum = np.einsum('nq,q,mq,nq->nm',gamma,variance,Z,mu) # NxM + +# _dpsi2_dvariance = np.einsum('nq,q,mq,oq->nmoq',2.*(gamma*mu2S-gamma2*mu2),variance,Z,Z)+\ +# np.einsum('nq,mq,nq,no->nmoq',gamma,Z,mu,common_sum)+\ +# np.einsum('nq,oq,nq,nm->nmoq',gamma,Z,mu,common_sum) +# +# _dpsi2_dgamma = np.einsum('q,mq,oq,nq->nmoq',variance2,Z,Z,(mu2S-2.*gamma*mu2))+\ +# np.einsum('q,mq,nq,no->nmoq',variance,Z,mu,common_sum)+\ +# np.einsum('q,oq,nq,nm->nmoq',variance,Z,mu,common_sum) +# + _dpsi2_dmu = np.einsum('q,mq,oq,nq,nq->nmoq',variance2,Z,Z,mu,2.*(gamma-gamma2))+\ + np.einsum('nq,q,mq,no->nmoq',gamma,variance,Z,common_sum)+\ + np.einsum('nq,q,oq,nm->nmoq',gamma,variance,Z,common_sum) +# +# _dpsi2_dS = np.einsum('nq,q,mq,oq->nmoq',gamma,variance2,Z,Z) +# +# _dpsi2_dZ = 2.*(np.einsum('nq,q,mq,nq->nmq',gamma,variance2,Z,mu2S)+np.einsum('nq,q,nq,nm->nmq',gamma,variance,mu,common_sum) +# -np.einsum('nq,q,mq,nq->nmq',gamma2,variance2,Z,mu2)) + dL_dmu = np.einsum('mo,nmoq->nq', dL_dpsi2, _dpsi2_dmu) + + dL_dvar = np.einsum('mo,nq,q,mq,oq->q',dL_dpsi2,2.*(gamma*mu2S-gamma2*mu2),variance,Z,Z)+\ + np.einsum('mo,nq,mq,nq,no->q',dL_dpsi2,gamma,Z,mu,common_sum)+\ + np.einsum('mo,nq,oq,nq,nm->q',dL_dpsi2,gamma,Z,mu,common_sum) + + dL_dgamma = np.einsum('mo,q,mq,oq,nq->nq',dL_dpsi2,variance2,Z,Z,(mu2S-2.*gamma*mu2))+\ + np.einsum('mo,q,mq,nq,no->nq',dL_dpsi2,variance,Z,mu,common_sum)+\ + np.einsum('mo,q,oq,nq,nm->nq',dL_dpsi2,variance,Z,mu,common_sum) + +# dL_dmu = np.einsum('mo,q,mq,oq,nq,nq->nq',dL_dpsi2,variance2,Z,Z,mu,2.*(gamma-gamma2))+\ +# np.einsum('mo,nq,q,mq,no->nq',dL_dpsi2,gamma,variance,Z,common_sum)+\ +# np.einsum('mo,nq,q,oq,nm->nq',dL_dpsi2,gamma,variance,Z,common_sum) + + dL_dS = np.einsum('mo,nq,q,mq,oq->nq',dL_dpsi2,gamma,variance2,Z,Z) + + dL_dZ = 2.*(np.einsum('om,nq,q,mq,nq->oq',dL_dpsi2,gamma,variance2,Z,mu2S)+np.einsum('om,nq,q,nq,nm->oq',dL_dpsi2,gamma,variance,mu,common_sum) + -np.einsum('om,nq,q,mq,nq->oq',dL_dpsi2,gamma2,variance2,Z,mu2)) + + return dL_dvar, dL_dgamma, dL_dmu, dL_dS, dL_dZ diff --git a/GPy/kern/_src/rbf.py b/GPy/kern/_src/rbf.py index 5944e765..f3590d40 100644 --- a/GPy/kern/_src/rbf.py +++ b/GPy/kern/_src/rbf.py @@ -9,7 +9,7 @@ from stationary import Stationary from GPy.util.caching import Cache_this from ...core.parameterization import variational from psi_comp import ssrbf_psi_comp -from psi_comp.ssrbf_psi_gpucomp import PSICOMP_SSRBF +from psi_comp import ssrbf_psi_gpucomp class RBF(Stationary): """ @@ -26,8 +26,11 @@ class RBF(Stationary): self.weave_options = {} self.group_spike_prob = False + def set_for_SpikeAndSlab(self): if self.useGPU: - self.psicomp = PSICOMP_SSRBF() + self.psicomp = ssrbf_psi_gpucomp.PSICOMP_SSRBF() + else: + self.psicomp = ssrbf_psi_comp def K_of_r(self, r): diff --git a/GPy/models/ss_gplvm.py b/GPy/models/ss_gplvm.py index cc57b191..76e2f0ef 100644 --- a/GPy/models/ss_gplvm.py +++ b/GPy/models/ss_gplvm.py @@ -44,8 +44,10 @@ class SSGPLVM(SparseGP): X_variance = np.random.uniform(0,.1,X.shape) gamma = np.empty_like(X, order='F') # The posterior probabilities of the binary variable in the variational approximation - #gamma[:] = 0.5 + 0.01 * np.random.randn(X.shape[0], input_dim) - gamma[:] = 0.5 + gamma[:] = 0.5 + 0.1 * np.random.randn(X.shape[0], input_dim) + gamma[gamma>=1. - 1e-9] = 1e-9 + gamma[gamma<1e-9] = 1e-9 + #gamma[:] = 0.5 if group_spike: gamma[:] = gamma.mean(axis=0) @@ -57,19 +59,20 @@ class SSGPLVM(SparseGP): pi = np.empty((input_dim)) pi[:] = 0.5 - if mpi_comm != None: - mpi_comm.Bcast(X, root=0) - mpi_comm.Bcast(fracs, root=0) - mpi_comm.Bcast(X_variance, root=0) - mpi_comm.Bcast(gamma, root=0) - mpi_comm.Bcast(Z, root=0) - mpi_comm.Bcast(pi, root=0) +# if mpi_comm != None: +# mpi_comm.Bcast(X, root=0) +# mpi_comm.Bcast(fracs, root=0) +# mpi_comm.Bcast(X_variance, root=0) +# mpi_comm.Bcast(gamma, root=0) +# mpi_comm.Bcast(Z, root=0) +# mpi_comm.Bcast(pi, root=0) if likelihood is None: likelihood = Gaussian() if kernel is None: kernel = kern.RBF(input_dim, lengthscale=fracs, ARD=True) # + kern.white(input_dim) + kernel.set_for_SpikeAndSlab() self.variational_prior = SpikeAndSlabPrior(pi=pi) # the prior probability of the latent binary variable b @@ -90,6 +93,7 @@ class SSGPLVM(SparseGP): self.X_local = self.X[Y_start:Y_end] self.Y_range = (Y_start, Y_end) self.Y_list = np.array(Y_list) + [mpi_comm.Bcast(p, root=0) for p in self.flattened_parameters] def set_X_gradients(self, X, X_grad): """Set the gradients of the posterior distribution of X in its specific form.""" @@ -125,3 +129,16 @@ class SSGPLVM(SparseGP): return dim_reduction_plots.plot_latent(self, plot_inducing=plot_inducing, *args, **kwargs) + def __getstate__(self): + dc = super(SSGPLVM, self).__getstate__() + del dc['mpi_comm'] + del dc['Y_local'] + del dc['X_local'] + return dc + + def __setstate__(self, state): + state['mpi_comm'] = None + Y_range = state['Y_range'] + state['Y_local'] = state['Y'][Y_range[0]:Y_range[1]] + state['X_local'] = state['X'][Y_range[0]:Y_range[1]] + return super(SSGPLVM, self).__setstate__(state) From ab501ddcba6737f7f13dea95a6d8fb8b24054768 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Thu, 15 May 2014 12:11:55 +0100 Subject: [PATCH 06/55] [SSGPLVM] linear kernel cpu ready --- GPy/kern/_src/psi_comp/linear_psi_comp.py | 26 ++++------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/GPy/kern/_src/psi_comp/linear_psi_comp.py b/GPy/kern/_src/psi_comp/linear_psi_comp.py index 03483b6b..11c49900 100644 --- a/GPy/kern/_src/psi_comp/linear_psi_comp.py +++ b/GPy/kern/_src/psi_comp/linear_psi_comp.py @@ -68,24 +68,6 @@ class PSICOMP_SSLinear(object): variance2 = np.square(variance) mu2S = mu2+S # NxQ common_sum = np.einsum('nq,q,mq,nq->nm',gamma,variance,Z,mu) # NxM - -# _dpsi2_dvariance = np.einsum('nq,q,mq,oq->nmoq',2.*(gamma*mu2S-gamma2*mu2),variance,Z,Z)+\ -# np.einsum('nq,mq,nq,no->nmoq',gamma,Z,mu,common_sum)+\ -# np.einsum('nq,oq,nq,nm->nmoq',gamma,Z,mu,common_sum) -# -# _dpsi2_dgamma = np.einsum('q,mq,oq,nq->nmoq',variance2,Z,Z,(mu2S-2.*gamma*mu2))+\ -# np.einsum('q,mq,nq,no->nmoq',variance,Z,mu,common_sum)+\ -# np.einsum('q,oq,nq,nm->nmoq',variance,Z,mu,common_sum) -# - _dpsi2_dmu = np.einsum('q,mq,oq,nq,nq->nmoq',variance2,Z,Z,mu,2.*(gamma-gamma2))+\ - np.einsum('nq,q,mq,no->nmoq',gamma,variance,Z,common_sum)+\ - np.einsum('nq,q,oq,nm->nmoq',gamma,variance,Z,common_sum) -# -# _dpsi2_dS = np.einsum('nq,q,mq,oq->nmoq',gamma,variance2,Z,Z) -# -# _dpsi2_dZ = 2.*(np.einsum('nq,q,mq,nq->nmq',gamma,variance2,Z,mu2S)+np.einsum('nq,q,nq,nm->nmq',gamma,variance,mu,common_sum) -# -np.einsum('nq,q,mq,nq->nmq',gamma2,variance2,Z,mu2)) - dL_dmu = np.einsum('mo,nmoq->nq', dL_dpsi2, _dpsi2_dmu) dL_dvar = np.einsum('mo,nq,q,mq,oq->q',dL_dpsi2,2.*(gamma*mu2S-gamma2*mu2),variance,Z,Z)+\ np.einsum('mo,nq,mq,nq,no->q',dL_dpsi2,gamma,Z,mu,common_sum)+\ @@ -95,10 +77,10 @@ class PSICOMP_SSLinear(object): np.einsum('mo,q,mq,nq,no->nq',dL_dpsi2,variance,Z,mu,common_sum)+\ np.einsum('mo,q,oq,nq,nm->nq',dL_dpsi2,variance,Z,mu,common_sum) -# dL_dmu = np.einsum('mo,q,mq,oq,nq,nq->nq',dL_dpsi2,variance2,Z,Z,mu,2.*(gamma-gamma2))+\ -# np.einsum('mo,nq,q,mq,no->nq',dL_dpsi2,gamma,variance,Z,common_sum)+\ -# np.einsum('mo,nq,q,oq,nm->nq',dL_dpsi2,gamma,variance,Z,common_sum) - + dL_dmu = np.einsum('mo,q,mq,oq,nq,nq->nq',dL_dpsi2,variance2,Z,Z,mu,2.*(gamma-gamma2))+\ + np.einsum('mo,nq,q,mq,no->nq',dL_dpsi2,gamma,variance,Z,common_sum)+\ + np.einsum('mo,nq,q,oq,nm->nq',dL_dpsi2,gamma,variance,Z,common_sum) + dL_dS = np.einsum('mo,nq,q,mq,oq->nq',dL_dpsi2,gamma,variance2,Z,Z) dL_dZ = 2.*(np.einsum('om,nq,q,mq,nq->oq',dL_dpsi2,gamma,variance2,Z,mu2S)+np.einsum('om,nq,q,nq,nm->oq',dL_dpsi2,gamma,variance,mu,common_sum) From c568bad4fbef7a0e91079af0c14bb808770a8be8 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Fri, 16 May 2014 10:40:45 +0100 Subject: [PATCH 07/55] bug fix for mpi SSGPLVM --- .../var_dtc_parallel.py | 4 ++++ GPy/models/ss_gplvm.py | 22 +++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/GPy/inference/latent_function_inference/var_dtc_parallel.py b/GPy/inference/latent_function_inference/var_dtc_parallel.py index 4123e5b2..44babf25 100644 --- a/GPy/inference/latent_function_inference/var_dtc_parallel.py +++ b/GPy/inference/latent_function_inference/var_dtc_parallel.py @@ -400,6 +400,10 @@ def update_gradients(model, mpi_comm=None): mpi_comm.Allreduce([np.float64(KL_div), MPI.DOUBLE], [KL_div_all, MPI.DOUBLE]) KL_div = KL_div_all [mpi_comm.Allgatherv([pp.copy(), MPI.DOUBLE], [pa, (model.Y_list*pa.shape[-1], None), MPI.DOUBLE]) for pp,pa in zip(model.get_X_gradients(X),model.get_X_gradients(model.X))] + from ...models import SSGPLVM + if isinstance(model, SSGPLVM): + grad_pi = np.array(model.variational_prior.pi.gradient) + mpi_comm.Allreduce([grad_pi.copy(), MPI.DOUBLE], [model.variational_prior.pi.gradient, MPI.DOUBLE]) model._log_marginal_likelihood -= KL_div # dL_dthetaL diff --git a/GPy/models/ss_gplvm.py b/GPy/models/ss_gplvm.py index 76e2f0ef..27d5158f 100644 --- a/GPy/models/ss_gplvm.py +++ b/GPy/models/ss_gplvm.py @@ -45,9 +45,9 @@ class SSGPLVM(SparseGP): gamma = np.empty_like(X, order='F') # The posterior probabilities of the binary variable in the variational approximation gamma[:] = 0.5 + 0.1 * np.random.randn(X.shape[0], input_dim) - gamma[gamma>=1. - 1e-9] = 1e-9 + gamma[gamma>1.-1e-9] = 1.-1e-9 gamma[gamma<1e-9] = 1e-9 - #gamma[:] = 0.5 + gamma[:] = 0.5 if group_spike: gamma[:] = gamma.mean(axis=0) @@ -142,3 +142,21 @@ class SSGPLVM(SparseGP): state['Y_local'] = state['Y'][Y_range[0]:Y_range[1]] state['X_local'] = state['X'][Y_range[0]:Y_range[1]] return super(SSGPLVM, self).__setstate__(state) + + def _grads(self, x): + if self.mpi_comm != None: + self.mpi_comm.Bcast(x, root=0) + obj_grads = super(SSGPLVM, self)._grads(x) + return obj_grads + + def _objective(self, x): + if self.mpi_comm != None: + self.mpi_comm.Bcast(x, root=0) + obj = super(SSGPLVM, self)._objective(x) + return obj + + def _objective_grads(self, x): + if self.mpi_comm != None: + self.mpi_comm.Bcast(x, root=0) + obj_f, obj_grads = super(SSGPLVM, self)._objective_grads(x) + return obj_f, obj_grads From 17b6b94db3d80687b07a326667664a733666bf2e Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Fri, 16 May 2014 10:48:18 +0100 Subject: [PATCH 08/55] Logistic transformation numerical robustness --- GPy/core/parameterization/transformations.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/GPy/core/parameterization/transformations.py b/GPy/core/parameterization/transformations.py index 506d80cd..df31f09f 100644 --- a/GPy/core/parameterization/transformations.py +++ b/GPy/core/parameterization/transformations.py @@ -195,6 +195,9 @@ class Logistic(Transformation): self.lower, self.upper = float(lower), float(upper) self.difference = self.upper - self.lower def f(self, x): + if (x<-300.).any(): + x = x.copy() + x[x<-300.] = -300. return self.lower + self.difference / (1. + np.exp(-x)) def finv(self, f): return np.log(np.clip(f - self.lower, 1e-10, np.inf) / np.clip(self.upper - f, 1e-10, np.inf)) From e6d07ad5acb4b047e5dc28014fbe7822b8d9e6ee Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Fri, 16 May 2014 11:23:44 +0100 Subject: [PATCH 09/55] fix pickle --- .../var_dtc_parallel.py | 7 +- GPy/kern/_src/linear.py | 6 +- GPy/kern/_src/psi_comp/__init__.py | 17 + GPy/kern/_src/psi_comp/linear_psi_comp.py | 12 +- GPy/kern/_src/psi_comp/ssrbf_psi_comp.py | 374 +++++++++--------- GPy/kern/_src/rbf.py | 66 +--- GPy/models/ss_gplvm.py | 23 +- 7 files changed, 230 insertions(+), 275 deletions(-) diff --git a/GPy/inference/latent_function_inference/var_dtc_parallel.py b/GPy/inference/latent_function_inference/var_dtc_parallel.py index 44babf25..7b11085e 100644 --- a/GPy/inference/latent_function_inference/var_dtc_parallel.py +++ b/GPy/inference/latent_function_inference/var_dtc_parallel.py @@ -26,7 +26,7 @@ class VarDTC_minibatch(LatentFunctionInference): """ const_jitter = 1e-6 - def __init__(self, batchsize, limit=1, mpi_comm=None): + def __init__(self, batchsize=None, limit=1, mpi_comm=None): self.batchsize = batchsize self.mpi_comm = mpi_comm @@ -74,9 +74,12 @@ class VarDTC_minibatch(LatentFunctionInference): return jitchol(tdot(Y)) def gatherPsiStat(self, kern, X, Z, Y, beta, uncertain_inputs, het_noise): - + num_inducing = Z.shape[0] num_data, output_dim = Y.shape + + if self.batchsize == None or self.batchsize>num_data: + self.batchsize = num_data trYYT = self.get_trYYT(Y) diff --git a/GPy/kern/_src/linear.py b/GPy/kern/_src/linear.py index 7a2f899f..649c1222 100644 --- a/GPy/kern/_src/linear.py +++ b/GPy/kern/_src/linear.py @@ -111,20 +111,20 @@ class Linear(Kern): def psi0(self, Z, variational_posterior): if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - return self.psicomp.psicomputations(self.variances, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[0] + return self.psicomp.psicomputations(self.variances, Z, variational_posterior)[0] else: return np.sum(self.variances * self._mu2S(variational_posterior), 1) def psi1(self, Z, variational_posterior): if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - return self.psicomp.psicomputations(self.variances, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[1] + return self.psicomp.psicomputations(self.variances, Z, variational_posterior)[1] else: return self.K(variational_posterior.mean, Z) #the variance, it does nothing @Cache_this(limit=1) def psi2(self, Z, variational_posterior): if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - return self.psicomp.psicomputations(self.variances, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[2] + return self.psicomp.psicomputations(self.variances, Z, variational_posterior)[2] else: ZA = Z * self.variances ZAinner = self._ZAinner(variational_posterior, Z) diff --git a/GPy/kern/_src/psi_comp/__init__.py b/GPy/kern/_src/psi_comp/__init__.py index 4c0d373d..17ea2e83 100644 --- a/GPy/kern/_src/psi_comp/__init__.py +++ b/GPy/kern/_src/psi_comp/__init__.py @@ -1,2 +1,19 @@ # Copyright (c) 2012, GPy authors (see AUTHORS.txt). # Licensed under the BSD 3-clause license (see LICENSE.txt) + +from ....core.parameterization.parameter_core import Pickleable + +class PSICOMP(Pickleable): + + def psicomputations(self, variance, Z, variational_posterior): + """ + Compute psi-statistics + """ + pass + + def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior): + """ + Compute the derivatives of parameters by combing dL_dpsi and dpsi_dparam + """ + pass + diff --git a/GPy/kern/_src/psi_comp/linear_psi_comp.py b/GPy/kern/_src/psi_comp/linear_psi_comp.py index 11c49900..3404c987 100644 --- a/GPy/kern/_src/psi_comp/linear_psi_comp.py +++ b/GPy/kern/_src/psi_comp/linear_psi_comp.py @@ -6,11 +6,12 @@ The package for the Psi statistics computation of the linear kernel for SSGPLVM """ import numpy as np +from . import PSICOMP from GPy.util.caching import Cache_this -class PSICOMP_SSLinear(object): - #@Cache_this(limit=1, ignore_args=(0,)) - def psicomputations(self, variance, Z, mu, S, gamma): +class PSICOMP_SSLinear(PSICOMP): + @Cache_this(limit=1, ignore_args=(0,)) + def psicomputations(self, variance, Z, variational_posterior): """ Compute psi-statistics for ss-linear kernel """ @@ -19,6 +20,9 @@ class PSICOMP_SSLinear(object): # psi0 N # psi1 NxM # psi2 MxM + mu = variational_posterior.mean + S = variational_posterior.variance + gamma = variational_posterior.binary_prob psi0 = np.einsum('q,nq,nq->n',variance,gamma,np.square(mu)+S) psi1 = np.einsum('nq,q,mq,nq->nm',gamma,variance,Z,mu) @@ -30,7 +34,7 @@ class PSICOMP_SSLinear(object): return psi0, psi1, psi2 - #@Cache_this(limit=1, ignore_args=(0,1,2,3)) + @Cache_this(limit=1, ignore_args=(0,1,2,3)) def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior): mu = variational_posterior.mean S = variational_posterior.variance diff --git a/GPy/kern/_src/psi_comp/ssrbf_psi_comp.py b/GPy/kern/_src/psi_comp/ssrbf_psi_comp.py index 24e04ae4..8a24c60a 100644 --- a/GPy/kern/_src/psi_comp/ssrbf_psi_comp.py +++ b/GPy/kern/_src/psi_comp/ssrbf_psi_comp.py @@ -6,210 +6,200 @@ The package for the psi statistics computation """ import numpy as np +from . import PSICOMP from GPy.util.caching import Cache_this,Cacher -@Cache_this(limit=1) -def psicomputations(variance, lengthscale, Z, mu, S, gamma): - """ - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - """ - # here are the "statistics" for psi0, psi1 and psi2 - # Produced intermediate results: - # _psi1 NxM +class PSICOMP_SSRBF(PSICOMP): - psi0 = np.empty(mu.shape[0]) - psi0[:] = variance - psi1 = _psi1computations(variance, lengthscale, Z, mu, S, gamma) - psi2 = _psi2computations(variance, lengthscale, Z, mu, S, gamma) - return psi0, psi1, psi2 - -def _psi1computations(variance, lengthscale, Z, mu, S, gamma): - """ - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - """ - # here are the "statistics" for psi1 - # Produced intermediate results: - # _psi1 NxM - + @Cache_this(limit=1, ignore_args=(0,)) + def psicomputations(self, variance, lengthscale, Z, variational_posterior): + """ + Z - MxQ + mu - NxQ + S - NxQ + gamma - NxQ + """ + # here are the "statistics" for psi0, psi1 and psi2 + # Produced intermediate results: + # _psi1 NxM + mu = variational_posterior.mean + S = variational_posterior.variance + gamma = variational_posterior.binary_prob + + psi0 = np.empty(mu.shape[0]) + psi0[:] = variance + psi1 = self._psi1computations(variance, lengthscale, Z, mu, S, gamma) + psi2 = self._psi2computations(variance, lengthscale, Z, mu, S, gamma) + return psi0, psi1, psi2 - lengthscale2 = np.square(lengthscale) - - # psi1 - _psi1_denom = S[:, None, :] / lengthscale2 + 1. # Nx1xQ - _psi1_denom_sqrt = np.sqrt(_psi1_denom) #Nx1xQ - _psi1_dist = Z[None, :, :] - mu[:, None, :] # NxMxQ - _psi1_dist_sq = np.square(_psi1_dist) / (lengthscale2 * _psi1_denom) # NxMxQ - _psi1_common = gamma[:,None,:] / (lengthscale2*_psi1_denom*_psi1_denom_sqrt) #Nx1xQ - _psi1_exponent1 = np.log(gamma[:,None,:]) - (_psi1_dist_sq + np.log(_psi1_denom))/2. # NxMxQ - _psi1_exponent2 = np.log(1.-gamma[:,None,:]) - (np.square(Z[None,:,:])/lengthscale2)/2. # NxMxQ - _psi1_exponent_max = np.maximum(_psi1_exponent1,_psi1_exponent2) - _psi1_exponent = _psi1_exponent_max+np.log(np.exp(_psi1_exponent1-_psi1_exponent_max) + np.exp(_psi1_exponent2-_psi1_exponent_max)) #NxMxQ - _psi1_exp_sum = _psi1_exponent.sum(axis=-1) #NxM - _psi1 = variance * np.exp(_psi1_exp_sum) # NxM - - return _psi1 - -def _psi2computations(variance, lengthscale, Z, mu, S, gamma): - """ - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - """ - # here are the "statistics" for psi2 - # Produced intermediate results: - # _psi2 MxM + def _psi1computations(self, variance, lengthscale, Z, mu, S, gamma): + """ + Z - MxQ + mu - NxQ + S - NxQ + gamma - NxQ + """ + # here are the "statistics" for psi1 + # Produced intermediate results: + # _psi1 NxM - lengthscale2 = np.square(lengthscale) + lengthscale2 = np.square(lengthscale) - _psi2_Zhat = 0.5 * (Z[:, None, :] + Z[None, :, :]) # M,M,Q - _psi2_Zdist = 0.5 * (Z[:, None, :] - Z[None, :, :]) # M,M,Q - _psi2_Zdist_sq = np.square(_psi2_Zdist / lengthscale) # M,M,Q - _psi2_Z_sq_sum = (np.square(Z[:,None,:])+np.square(Z[None,:,:]))/lengthscale2 # MxMxQ - - # psi2 - _psi2_denom = 2.*S[:, None, None, :] / lengthscale2 + 1. # Nx1x1xQ - _psi2_denom_sqrt = np.sqrt(_psi2_denom) - _psi2_mudist = mu[:,None,None,:]-_psi2_Zhat #N,M,M,Q - _psi2_mudist_sq = np.square(_psi2_mudist)/(lengthscale2*_psi2_denom) - _psi2_common = gamma[:,None,None,:]/(lengthscale2 * _psi2_denom * _psi2_denom_sqrt) # Nx1x1xQ - _psi2_exponent1 = -_psi2_Zdist_sq -_psi2_mudist_sq -0.5*np.log(_psi2_denom)+np.log(gamma[:,None,None,:]) #N,M,M,Q - _psi2_exponent2 = np.log(1.-gamma[:,None,None,:]) - 0.5*(_psi2_Z_sq_sum) # NxMxMxQ - _psi2_exponent_max = np.maximum(_psi2_exponent1, _psi2_exponent2) - _psi2_exponent = _psi2_exponent_max+np.log(np.exp(_psi2_exponent1-_psi2_exponent_max) + np.exp(_psi2_exponent2-_psi2_exponent_max)) - _psi2_exp_sum = _psi2_exponent.sum(axis=-1) #NxM - _psi2 = variance*variance * (np.exp(_psi2_exp_sum).sum(axis=0)) # MxM - - return _psi2 - -def _psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior): - ARD = (len(lengthscale)!=1) + # psi1 + _psi1_denom = S[:, None, :] / lengthscale2 + 1. # Nx1xQ + _psi1_denom_sqrt = np.sqrt(_psi1_denom) #Nx1xQ + _psi1_dist = Z[None, :, :] - mu[:, None, :] # NxMxQ + _psi1_dist_sq = np.square(_psi1_dist) / (lengthscale2 * _psi1_denom) # NxMxQ + _psi1_common = gamma[:,None,:] / (lengthscale2*_psi1_denom*_psi1_denom_sqrt) #Nx1xQ + _psi1_exponent1 = np.log(gamma[:,None,:]) - (_psi1_dist_sq + np.log(_psi1_denom))/2. # NxMxQ + _psi1_exponent2 = np.log(1.-gamma[:,None,:]) - (np.square(Z[None,:,:])/lengthscale2)/2. # NxMxQ + _psi1_exponent_max = np.maximum(_psi1_exponent1,_psi1_exponent2) + _psi1_exponent = _psi1_exponent_max+np.log(np.exp(_psi1_exponent1-_psi1_exponent_max) + np.exp(_psi1_exponent2-_psi1_exponent_max)) #NxMxQ + _psi1_exp_sum = _psi1_exponent.sum(axis=-1) #NxM + _psi1 = variance * np.exp(_psi1_exp_sum) # NxM - dvar_psi1, dl_psi1, dZ_psi1, dmu_psi1, dS_psi1, dgamma_psi1 = _psi1compDer(dL_dpsi1, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) - dvar_psi2, dl_psi2, dZ_psi2, dmu_psi2, dS_psi2, dgamma_psi2 = _psi2compDer(dL_dpsi2, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) - - dL_dvar = np.sum(dL_dpsi0) + dvar_psi1 + dvar_psi2 + return _psi1 - dL_dlengscale = dl_psi1 + dl_psi2 - if not ARD: - dL_dlengscale = dL_dlengscale.sum() - - dL_dgamma = dgamma_psi1 + dgamma_psi2 - dL_dmu = dmu_psi1 + dmu_psi2 - dL_dS = dS_psi1 + dS_psi2 - dL_dZ = dZ_psi1 + dZ_psi2 + def _psi2computations(self, variance, lengthscale, Z, mu, S, gamma): + """ + Z - MxQ + mu - NxQ + S - NxQ + gamma - NxQ + """ + # here are the "statistics" for psi2 + # Produced intermediate results: + # _psi2 MxM + + lengthscale2 = np.square(lengthscale) + + _psi2_Zhat = 0.5 * (Z[:, None, :] + Z[None, :, :]) # M,M,Q + _psi2_Zdist = 0.5 * (Z[:, None, :] - Z[None, :, :]) # M,M,Q + _psi2_Zdist_sq = np.square(_psi2_Zdist / lengthscale) # M,M,Q + _psi2_Z_sq_sum = (np.square(Z[:,None,:])+np.square(Z[None,:,:]))/lengthscale2 # MxMxQ - return dL_dvar, dL_dlengscale, dL_dZ, dL_dmu, dL_dS, dL_dgamma - -def _psi1compDer(dL_dpsi1, variance, lengthscale, Z, mu, S, gamma): - """ - dL_dpsi1 - NxM - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - """ - # here are the "statistics" for psi1 - # Produced intermediate results: dL_dparams w.r.t. psi1 - # _dL_dvariance 1 - # _dL_dlengthscale Q - # _dL_dZ MxQ - # _dL_dgamma NxQ - # _dL_dmu NxQ - # _dL_dS NxQ + # psi2 + _psi2_denom = 2.*S[:, None, None, :] / lengthscale2 + 1. # Nx1x1xQ + _psi2_denom_sqrt = np.sqrt(_psi2_denom) + _psi2_mudist = mu[:,None,None,:]-_psi2_Zhat #N,M,M,Q + _psi2_mudist_sq = np.square(_psi2_mudist)/(lengthscale2*_psi2_denom) + _psi2_common = gamma[:,None,None,:]/(lengthscale2 * _psi2_denom * _psi2_denom_sqrt) # Nx1x1xQ + _psi2_exponent1 = -_psi2_Zdist_sq -_psi2_mudist_sq -0.5*np.log(_psi2_denom)+np.log(gamma[:,None,None,:]) #N,M,M,Q + _psi2_exponent2 = np.log(1.-gamma[:,None,None,:]) - 0.5*(_psi2_Z_sq_sum) # NxMxMxQ + _psi2_exponent_max = np.maximum(_psi2_exponent1, _psi2_exponent2) + _psi2_exponent = _psi2_exponent_max+np.log(np.exp(_psi2_exponent1-_psi2_exponent_max) + np.exp(_psi2_exponent2-_psi2_exponent_max)) + _psi2_exp_sum = _psi2_exponent.sum(axis=-1) #NxM + _psi2 = variance*variance * (np.exp(_psi2_exp_sum).sum(axis=0)) # MxM - lengthscale2 = np.square(lengthscale) - - # psi1 - _psi1_denom = S / lengthscale2 + 1. # NxQ - _psi1_denom_sqrt = np.sqrt(_psi1_denom) #NxQ - _psi1_dist = Z[None, :, :] - mu[:, None, :] # NxMxQ - _psi1_dist_sq = np.square(_psi1_dist) / (lengthscale2 * _psi1_denom[:,None,:]) # NxMxQ - _psi1_common = gamma / (lengthscale2*_psi1_denom*_psi1_denom_sqrt) #NxQ - _psi1_exponent1 = np.log(gamma[:,None,:]) -0.5 * (_psi1_dist_sq + np.log(_psi1_denom[:, None,:])) # NxMxQ - _psi1_exponent2 = np.log(1.-gamma[:,None,:]) -0.5 * (np.square(Z[None,:,:])/lengthscale2) # NxMxQ - _psi1_exponent_max = np.maximum(_psi1_exponent1,_psi1_exponent2) - _psi1_exponent = _psi1_exponent_max+np.log(np.exp(_psi1_exponent1-_psi1_exponent_max) + np.exp(_psi1_exponent2-_psi1_exponent_max)) #NxMxQ - _psi1_exp_sum = _psi1_exponent.sum(axis=-1) #NxM - _psi1_exp_dist_sq = np.exp(-0.5*_psi1_dist_sq) # NxMxQ - _psi1_exp_Z = np.exp(-0.5*np.square(Z[None,:,:])/lengthscale2) # 1xMxQ - _psi1_q = variance * np.exp(_psi1_exp_sum[:,:,None] - _psi1_exponent) # NxMxQ - _psi1 = variance * np.exp(_psi1_exp_sum) # NxM - _dL_dvariance = np.einsum('nm,nm->',dL_dpsi1, _psi1)/variance # 1 - _dL_dgamma = np.einsum('nm,nmq,nmq->nq',dL_dpsi1, _psi1_q, (_psi1_exp_dist_sq/_psi1_denom_sqrt[:,None,:]-_psi1_exp_Z)) # NxQ - _dL_dmu = np.einsum('nm, nmq, nmq, nmq, nq->nq',dL_dpsi1,_psi1_q,_psi1_exp_dist_sq,_psi1_dist,_psi1_common) # NxQ - _dL_dS = np.einsum('nm,nmq,nmq,nq,nmq->nq',dL_dpsi1,_psi1_q,_psi1_exp_dist_sq,_psi1_common,(_psi1_dist_sq-1.))/2. # NxQ - _dL_dZ = np.einsum('nm,nmq,nmq->mq',dL_dpsi1,_psi1_q, (- _psi1_common[:,None,:] * _psi1_dist * _psi1_exp_dist_sq - (1-gamma[:,None,:])/lengthscale2*Z[None,:,:]*_psi1_exp_Z)) - _dL_dlengthscale = lengthscale* np.einsum('nm,nmq,nmq->q',dL_dpsi1,_psi1_q,(_psi1_common[:,None,:]*(S[:,None,:]/lengthscale2+_psi1_dist_sq)*_psi1_exp_dist_sq + (1-gamma[:,None,:])*np.square(Z[None,:,:]/lengthscale2)*_psi1_exp_Z)) + return _psi2 -# _dpsi1_dmu = _psi1_q * (_psi1_exp_dist_sq * _psi1_dist * _psi1_common) # NxMxQ -# _dpsi1_dS = _psi1_q * (_psi1_exp_dist_sq * _psi1_common * 0.5 * (_psi1_dist_sq - 1.)) # NxMxQ -# _dpsi1_dZ = _psi1_q * (- _psi1_common * _psi1_dist * _psi1_exp_dist_sq - (1-gamma[:,None,:])/lengthscale2*Z[None,:,:]*_psi1_exp_Z) # NxMxQ -# _dpsi1_dlengthscale = 2.*lengthscale*_psi1_q * (0.5*_psi1_common*(S[:,None,:]/lengthscale2+_psi1_dist_sq)*_psi1_exp_dist_sq + 0.5*(1-gamma[:,None,:])*np.square(Z[None,:,:]/lengthscale2)*_psi1_exp_Z) # NxMxQ - - return _dL_dvariance, _dL_dlengthscale, _dL_dZ, _dL_dmu, _dL_dS, _dL_dgamma - -def _psi2compDer(dL_dpsi2, variance, lengthscale, Z, mu, S, gamma): - """ - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - dL_dpsi2 - MxM - """ - # here are the "statistics" for psi2 - # Produced the derivatives w.r.t. psi2: - # _dL_dvariance 1 - # _dL_dlengthscale Q - # _dL_dZ MxQ - # _dL_dgamma NxQ - # _dL_dmu NxQ - # _dL_dS NxQ + @Cache_this(limit=1, ignore_args=(0,1,2,3)) + def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior): + ARD = (len(lengthscale)!=1) + + dvar_psi1, dl_psi1, dZ_psi1, dmu_psi1, dS_psi1, dgamma_psi1 = self._psi1compDer(dL_dpsi1, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) + dvar_psi2, dl_psi2, dZ_psi2, dmu_psi2, dS_psi2, dgamma_psi2 = self._psi2compDer(dL_dpsi2, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) - lengthscale2 = np.square(lengthscale) + dL_dvar = np.sum(dL_dpsi0) + dvar_psi1 + dvar_psi2 + + dL_dlengscale = dl_psi1 + dl_psi2 + if not ARD: + dL_dlengscale = dL_dlengscale.sum() - _psi2_Zhat = 0.5 * (Z[:, None, :] + Z[None, :, :]) # M,M,Q - _psi2_Zdist = 0.5 * (Z[:, None, :] - Z[None, :, :]) # M,M,Q - _psi2_Zdist_sq = np.square(_psi2_Zdist / lengthscale) # M,M,Q - _psi2_Z_sq_sum = (np.square(Z[:,None,:])+np.square(Z[None,:,:]))/lengthscale2 # MxMxQ - - # psi2 - _psi2_denom = 2.*S / lengthscale2 + 1. # NxQ - _psi2_denom_sqrt = np.sqrt(_psi2_denom) - _psi2_mudist = mu[:,None,None,:]-_psi2_Zhat #N,M,M,Q - _psi2_mudist_sq = np.square(_psi2_mudist)/(lengthscale2*_psi2_denom[:,None,None,:]) - _psi2_common = gamma/(lengthscale2 * _psi2_denom * _psi2_denom_sqrt) # NxQ - _psi2_exponent1 = -_psi2_Zdist_sq -_psi2_mudist_sq -0.5*np.log(_psi2_denom[:,None,None,:])+np.log(gamma[:,None,None,:]) #N,M,M,Q - _psi2_exponent2 = np.log(1.-gamma[:,None,None,:]) - 0.5*(_psi2_Z_sq_sum) # NxMxMxQ - _psi2_exponent_max = np.maximum(_psi2_exponent1, _psi2_exponent2) - _psi2_exponent = _psi2_exponent_max+np.log(np.exp(_psi2_exponent1-_psi2_exponent_max) + np.exp(_psi2_exponent2-_psi2_exponent_max)) - _psi2_exp_sum = _psi2_exponent.sum(axis=-1) #NxM - _psi2_q = variance*variance * np.exp(_psi2_exp_sum[:,:,:,None]-_psi2_exponent) # NxMxMxQ - _psi2_exp_dist_sq = np.exp(-_psi2_Zdist_sq -_psi2_mudist_sq) # NxMxMxQ - _psi2_exp_Z = np.exp(-0.5*_psi2_Z_sq_sum) # MxMxQ - _psi2 = variance*variance * (np.exp(_psi2_exp_sum).sum(axis=0)) # MxM - _dL_dvariance = np.einsum('mo,mo->',dL_dpsi2,_psi2)*2./variance - _dL_dgamma = np.einsum('mo,nmoq,nmoq->nq',dL_dpsi2,_psi2_q,(_psi2_exp_dist_sq/_psi2_denom_sqrt[:,None,None,:] - _psi2_exp_Z)) - _dL_dmu = -2.*np.einsum('mo,nmoq,nq,nmoq,nmoq->nq',dL_dpsi2,_psi2_q,_psi2_common,_psi2_mudist,_psi2_exp_dist_sq) - _dL_dS = np.einsum('mo,nmoq,nq,nmoq,nmoq->nq',dL_dpsi2,_psi2_q, _psi2_common, (2.*_psi2_mudist_sq-1.), _psi2_exp_dist_sq) - _dL_dZ = 2.*np.einsum('mo,nmoq,nmoq->mq',dL_dpsi2,_psi2_q,(_psi2_common[:,None,None,:]*(-_psi2_Zdist*_psi2_denom[:,None,None,:]+_psi2_mudist)*_psi2_exp_dist_sq - (1-gamma[:,None,None,:])*Z[:,None,:]/lengthscale2*_psi2_exp_Z)) -# print _psi2_common[:,None,None,:]*(S[:,None,None,:]/lengthscale2+_psi2_Zdist_sq*_psi2_denom[:,None,None,:]+_psi2_mudist_sq)*_psi2_exp_dist_sq #+(1-gamma[:,None,None,:])*_psi2_Z_sq_sum*0.5/lengthscale2*_psi2_exp_Z) - _dL_dlengthscale = 2.*lengthscale* np.einsum('mo,nmoq,nmoq->q',dL_dpsi2,_psi2_q,(_psi2_common[:,None,None,:]*(S[:,None,None,:]/lengthscale2+_psi2_Zdist_sq*_psi2_denom[:,None,None,:]+_psi2_mudist_sq)*_psi2_exp_dist_sq+(1-gamma[:,None,None,:])*_psi2_Z_sq_sum*0.5/lengthscale2*_psi2_exp_Z)) + dL_dgamma = dgamma_psi1 + dgamma_psi2 + dL_dmu = dmu_psi1 + dmu_psi2 + dL_dS = dS_psi1 + dS_psi2 + dL_dZ = dZ_psi1 + dZ_psi2 + + return dL_dvar, dL_dlengscale, dL_dZ, dL_dmu, dL_dS, dL_dgamma + def _psi1compDer(self, dL_dpsi1, variance, lengthscale, Z, mu, S, gamma): + """ + dL_dpsi1 - NxM + Z - MxQ + mu - NxQ + S - NxQ + gamma - NxQ + """ + # here are the "statistics" for psi1 + # Produced intermediate results: dL_dparams w.r.t. psi1 + # _dL_dvariance 1 + # _dL_dlengthscale Q + # _dL_dZ MxQ + # _dL_dgamma NxQ + # _dL_dmu NxQ + # _dL_dS NxQ + + lengthscale2 = np.square(lengthscale) -# _dpsi2_dvariance = 2. * _psi2/variance # NxMxM -# _dpsi2_dgamma = _psi2_q * (_psi2_exp_dist_sq/_psi2_denom_sqrt - _psi2_exp_Z) # NxMxMxQ -# _dpsi2_dmu = _psi2_q * (-2.*_psi2_common*_psi2_mudist * _psi2_exp_dist_sq) # NxMxMxQ -# _dpsi2_dS = _psi2_q * (_psi2_common * (2.*_psi2_mudist_sq - 1.) * _psi2_exp_dist_sq) # NxMxMxQ -# _dpsi2_dZ = 2.*_psi2_q * (_psi2_common*(-_psi2_Zdist*_psi2_denom+_psi2_mudist)*_psi2_exp_dist_sq - (1-gamma[:,None,None,:])*Z[:,None,:]/lengthscale2*_psi2_exp_Z) # NxMxMxQ -# _dpsi2_dlengthscale = 2.*lengthscale* _psi2_q * (_psi2_common*(S[:,None,None,:]/lengthscale2+_psi2_Zdist_sq*_psi2_denom+_psi2_mudist_sq)*_psi2_exp_dist_sq+(1-gamma[:,None,None,:])*_psi2_Z_sq_sum*0.5/lengthscale2*_psi2_exp_Z) # NxMxMxQ - - return _dL_dvariance, _dL_dlengthscale, _dL_dZ, _dL_dmu, _dL_dS, _dL_dgamma - -psiDerivativecomputations = Cacher(_psiDerivativecomputations, limit=1, ignore_args=(0,1,2,)) + # psi1 + _psi1_denom = S / lengthscale2 + 1. # NxQ + _psi1_denom_sqrt = np.sqrt(_psi1_denom) #NxQ + _psi1_dist = Z[None, :, :] - mu[:, None, :] # NxMxQ + _psi1_dist_sq = np.square(_psi1_dist) / (lengthscale2 * _psi1_denom[:,None,:]) # NxMxQ + _psi1_common = gamma / (lengthscale2*_psi1_denom*_psi1_denom_sqrt) #NxQ + _psi1_exponent1 = np.log(gamma[:,None,:]) -0.5 * (_psi1_dist_sq + np.log(_psi1_denom[:, None,:])) # NxMxQ + _psi1_exponent2 = np.log(1.-gamma[:,None,:]) -0.5 * (np.square(Z[None,:,:])/lengthscale2) # NxMxQ + _psi1_exponent_max = np.maximum(_psi1_exponent1,_psi1_exponent2) + _psi1_exponent = _psi1_exponent_max+np.log(np.exp(_psi1_exponent1-_psi1_exponent_max) + np.exp(_psi1_exponent2-_psi1_exponent_max)) #NxMxQ + _psi1_exp_sum = _psi1_exponent.sum(axis=-1) #NxM + _psi1_exp_dist_sq = np.exp(-0.5*_psi1_dist_sq) # NxMxQ + _psi1_exp_Z = np.exp(-0.5*np.square(Z[None,:,:])/lengthscale2) # 1xMxQ + _psi1_q = variance * np.exp(_psi1_exp_sum[:,:,None] - _psi1_exponent) # NxMxQ + _psi1 = variance * np.exp(_psi1_exp_sum) # NxM + _dL_dvariance = np.einsum('nm,nm->',dL_dpsi1, _psi1)/variance # 1 + _dL_dgamma = np.einsum('nm,nmq,nmq->nq',dL_dpsi1, _psi1_q, (_psi1_exp_dist_sq/_psi1_denom_sqrt[:,None,:]-_psi1_exp_Z)) # NxQ + _dL_dmu = np.einsum('nm, nmq, nmq, nmq, nq->nq',dL_dpsi1,_psi1_q,_psi1_exp_dist_sq,_psi1_dist,_psi1_common) # NxQ + _dL_dS = np.einsum('nm,nmq,nmq,nq,nmq->nq',dL_dpsi1,_psi1_q,_psi1_exp_dist_sq,_psi1_common,(_psi1_dist_sq-1.))/2. # NxQ + _dL_dZ = np.einsum('nm,nmq,nmq->mq',dL_dpsi1,_psi1_q, (- _psi1_common[:,None,:] * _psi1_dist * _psi1_exp_dist_sq - (1-gamma[:,None,:])/lengthscale2*Z[None,:,:]*_psi1_exp_Z)) + _dL_dlengthscale = lengthscale* np.einsum('nm,nmq,nmq->q',dL_dpsi1,_psi1_q,(_psi1_common[:,None,:]*(S[:,None,:]/lengthscale2+_psi1_dist_sq)*_psi1_exp_dist_sq + (1-gamma[:,None,:])*np.square(Z[None,:,:]/lengthscale2)*_psi1_exp_Z)) + + return _dL_dvariance, _dL_dlengthscale, _dL_dZ, _dL_dmu, _dL_dS, _dL_dgamma + + def _psi2compDer(self, dL_dpsi2, variance, lengthscale, Z, mu, S, gamma): + """ + Z - MxQ + mu - NxQ + S - NxQ + gamma - NxQ + dL_dpsi2 - MxM + """ + # here are the "statistics" for psi2 + # Produced the derivatives w.r.t. psi2: + # _dL_dvariance 1 + # _dL_dlengthscale Q + # _dL_dZ MxQ + # _dL_dgamma NxQ + # _dL_dmu NxQ + # _dL_dS NxQ + + lengthscale2 = np.square(lengthscale) + + _psi2_Zhat = 0.5 * (Z[:, None, :] + Z[None, :, :]) # M,M,Q + _psi2_Zdist = 0.5 * (Z[:, None, :] - Z[None, :, :]) # M,M,Q + _psi2_Zdist_sq = np.square(_psi2_Zdist / lengthscale) # M,M,Q + _psi2_Z_sq_sum = (np.square(Z[:,None,:])+np.square(Z[None,:,:]))/lengthscale2 # MxMxQ + + # psi2 + _psi2_denom = 2.*S / lengthscale2 + 1. # NxQ + _psi2_denom_sqrt = np.sqrt(_psi2_denom) + _psi2_mudist = mu[:,None,None,:]-_psi2_Zhat #N,M,M,Q + _psi2_mudist_sq = np.square(_psi2_mudist)/(lengthscale2*_psi2_denom[:,None,None,:]) + _psi2_common = gamma/(lengthscale2 * _psi2_denom * _psi2_denom_sqrt) # NxQ + _psi2_exponent1 = -_psi2_Zdist_sq -_psi2_mudist_sq -0.5*np.log(_psi2_denom[:,None,None,:])+np.log(gamma[:,None,None,:]) #N,M,M,Q + _psi2_exponent2 = np.log(1.-gamma[:,None,None,:]) - 0.5*(_psi2_Z_sq_sum) # NxMxMxQ + _psi2_exponent_max = np.maximum(_psi2_exponent1, _psi2_exponent2) + _psi2_exponent = _psi2_exponent_max+np.log(np.exp(_psi2_exponent1-_psi2_exponent_max) + np.exp(_psi2_exponent2-_psi2_exponent_max)) + _psi2_exp_sum = _psi2_exponent.sum(axis=-1) #NxM + _psi2_q = variance*variance * np.exp(_psi2_exp_sum[:,:,:,None]-_psi2_exponent) # NxMxMxQ + _psi2_exp_dist_sq = np.exp(-_psi2_Zdist_sq -_psi2_mudist_sq) # NxMxMxQ + _psi2_exp_Z = np.exp(-0.5*_psi2_Z_sq_sum) # MxMxQ + _psi2 = variance*variance * (np.exp(_psi2_exp_sum).sum(axis=0)) # MxM + _dL_dvariance = np.einsum('mo,mo->',dL_dpsi2,_psi2)*2./variance + _dL_dgamma = np.einsum('mo,nmoq,nmoq->nq',dL_dpsi2,_psi2_q,(_psi2_exp_dist_sq/_psi2_denom_sqrt[:,None,None,:] - _psi2_exp_Z)) + _dL_dmu = -2.*np.einsum('mo,nmoq,nq,nmoq,nmoq->nq',dL_dpsi2,_psi2_q,_psi2_common,_psi2_mudist,_psi2_exp_dist_sq) + _dL_dS = np.einsum('mo,nmoq,nq,nmoq,nmoq->nq',dL_dpsi2,_psi2_q, _psi2_common, (2.*_psi2_mudist_sq-1.), _psi2_exp_dist_sq) + _dL_dZ = 2.*np.einsum('mo,nmoq,nmoq->mq',dL_dpsi2,_psi2_q,(_psi2_common[:,None,None,:]*(-_psi2_Zdist*_psi2_denom[:,None,None,:]+_psi2_mudist)*_psi2_exp_dist_sq - (1-gamma[:,None,None,:])*Z[:,None,:]/lengthscale2*_psi2_exp_Z)) + _dL_dlengthscale = 2.*lengthscale* np.einsum('mo,nmoq,nmoq->q',dL_dpsi2,_psi2_q,(_psi2_common[:,None,None,:]*(S[:,None,None,:]/lengthscale2+_psi2_Zdist_sq*_psi2_denom[:,None,None,:]+_psi2_mudist_sq)*_psi2_exp_dist_sq+(1-gamma[:,None,None,:])*_psi2_Z_sq_sum*0.5/lengthscale2*_psi2_exp_Z)) + + return _dL_dvariance, _dL_dlengthscale, _dL_dZ, _dL_dmu, _dL_dS, _dL_dgamma diff --git a/GPy/kern/_src/rbf.py b/GPy/kern/_src/rbf.py index 51db1879..c75e4341 100644 --- a/GPy/kern/_src/rbf.py +++ b/GPy/kern/_src/rbf.py @@ -31,7 +31,7 @@ class RBF(Stationary): if self.useGPU: self.psicomp = ssrbf_psi_gpucomp.PSICOMP_SSRBF() else: - self.psicomp = ssrbf_psi_comp + self.psicomp = ssrbf_psi_comp.PSICOMP_SSRBF() def K_of_r(self, r): return self.variance * np.exp(-0.5 * r**2) @@ -48,7 +48,7 @@ class RBF(Stationary): if self.useGPU: return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[0] else: - return ssrbf_psi_comp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[0] + return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[0] else: return self.Kdiag(variational_posterior.mean) @@ -57,7 +57,7 @@ class RBF(Stationary): if self.useGPU: return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[1] else: - return ssrbf_psi_comp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[1] + return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[1] else: _, _, _, psi1 = self._psi1computations(Z, variational_posterior) return psi1 @@ -67,7 +67,7 @@ class RBF(Stationary): if self.useGPU: return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[2] else: - return ssrbf_psi_comp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[2] + return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[2] else: _, _, _, _, psi2 = self._psi2computations(Z, variational_posterior) return psi2 @@ -78,30 +78,9 @@ class RBF(Stationary): if self.useGPU: self.psicomp.update_gradients_expectations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) else: -# dL_dvar, dL_dlengscale, dL_dZ, dL_dgamma, dL_dmu, dL_dS = ssrbf_psi_comp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) - dL_dvar, dL_dlengscale, _, _, _, _ = ssrbf_psi_comp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) + dL_dvar, dL_dlengscale, _, _, _, _ = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) self.variance.gradient = dL_dvar self.lengthscale.gradient = dL_dlengscale - -# _, _dpsi1_dvariance, _, _, _, _, _dpsi1_dlengthscale = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) -# _, _dpsi2_dvariance, _, _, _, _, _dpsi2_dlengthscale = ssrbf_psi_comp._psi2computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) -# -# #contributions from psi0: -# self.variance.gradient = np.sum(dL_dpsi0) -# -# #from psi1 -# self.variance.gradient += np.sum(dL_dpsi1 * _dpsi1_dvariance) -# if self.ARD: -# self.lengthscale.gradient = (dL_dpsi1[:,:,None]*_dpsi1_dlengthscale).reshape(-1,self.input_dim).sum(axis=0) -# else: -# self.lengthscale.gradient = (dL_dpsi1[:,:,None]*_dpsi1_dlengthscale).sum() -# -# #from psi2 -# self.variance.gradient += (dL_dpsi2 * _dpsi2_dvariance).sum() -# if self.ARD: -# self.lengthscale.gradient += (dL_dpsi2[:,:,:,None] * _dpsi2_dlengthscale).reshape(-1,self.input_dim).sum(axis=0) -# else: -# self.lengthscale.gradient += (dL_dpsi2[:,:,:,None] * _dpsi2_dlengthscale).sum() elif isinstance(variational_posterior, variational.NormalPosterior): l2 = self.lengthscale**2 @@ -140,20 +119,9 @@ class RBF(Stationary): if self.useGPU: return self.psicomp.gradients_Z_expectations(dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) else: - _, _, dL_dZ, _, _, _ = ssrbf_psi_comp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) + _, _, dL_dZ, _, _, _ = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) return dL_dZ -# _, _, _, _, _, _dpsi1_dZ, _ = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) -# _, _, _, _, _, _dpsi2_dZ, _ = ssrbf_psi_comp._psi2computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) -# -# #psi1 -# grad = (dL_dpsi1[:, :, None] * _dpsi1_dZ).sum(axis=0) -# -# #psi2 -# grad += (dL_dpsi2[:, :, :, None] * _dpsi2_dZ).sum(axis=0).sum(axis=1) -# -# return grad - elif isinstance(variational_posterior, variational.NormalPosterior): l2 = self.lengthscale **2 @@ -179,29 +147,9 @@ class RBF(Stationary): if self.useGPU: return self.psicomp.gradients_qX_expectations(dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) else: - _, _, _, dL_dmu, dL_dS, dL_dgamma = ssrbf_psi_comp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) + _, _, _, dL_dmu, dL_dS, dL_dgamma = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) return dL_dmu, dL_dS, dL_dgamma -# ndata = variational_posterior.mean.shape[0] -# -# _, _, _dpsi1_dgamma, _dpsi1_dmu, _dpsi1_dS, _, _ = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) -# _, _, _dpsi2_dgamma, _dpsi2_dmu, _dpsi2_dS, _, _ = ssrbf_psi_comp._psi2computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) -# -# #psi1 -# grad_mu = (dL_dpsi1[:, :, None] * _dpsi1_dmu).sum(axis=1) -# grad_S = (dL_dpsi1[:, :, None] * _dpsi1_dS).sum(axis=1) -# grad_gamma = (dL_dpsi1[:,:,None] * _dpsi1_dgamma).sum(axis=1) -# -# #psi2 -# grad_mu += (dL_dpsi2[:, :, :, None] * _dpsi2_dmu).reshape(ndata,-1,self.input_dim).sum(axis=1) -# grad_S += (dL_dpsi2[:, :, :, None] * _dpsi2_dS).reshape(ndata,-1,self.input_dim).sum(axis=1) -# grad_gamma += (dL_dpsi2[:,:,:, None] * _dpsi2_dgamma).reshape(ndata,-1,self.input_dim).sum(axis=1) -# -# if self.group_spike_prob: -# grad_gamma[:] = grad_gamma.mean(axis=0) -# -# return grad_mu, grad_S, grad_gamma - elif isinstance(variational_posterior, variational.NormalPosterior): l2 = self.lengthscale **2 diff --git a/GPy/models/ss_gplvm.py b/GPy/models/ss_gplvm.py index 27d5158f..8581bf7c 100644 --- a/GPy/models/ss_gplvm.py +++ b/GPy/models/ss_gplvm.py @@ -59,20 +59,15 @@ class SSGPLVM(SparseGP): pi = np.empty((input_dim)) pi[:] = 0.5 -# if mpi_comm != None: -# mpi_comm.Bcast(X, root=0) -# mpi_comm.Bcast(fracs, root=0) -# mpi_comm.Bcast(X_variance, root=0) -# mpi_comm.Bcast(gamma, root=0) -# mpi_comm.Bcast(Z, root=0) -# mpi_comm.Bcast(pi, root=0) - if likelihood is None: likelihood = Gaussian() if kernel is None: kernel = kern.RBF(input_dim, lengthscale=fracs, ARD=True) # + kern.white(input_dim) kernel.set_for_SpikeAndSlab() + + if inference_method is None: + inference_method = VarDTC_minibatch(mpi_comm=mpi_comm) self.variational_prior = SpikeAndSlabPrior(pi=pi) # the prior probability of the latent binary variable b @@ -131,16 +126,14 @@ class SSGPLVM(SparseGP): def __getstate__(self): dc = super(SSGPLVM, self).__getstate__() - del dc['mpi_comm'] - del dc['Y_local'] - del dc['X_local'] + dc['mpi_comm'] = None + if self.mpi_comm != None: + del dc['Y_local'] + del dc['X_local'] + del dc['Y_range'] return dc def __setstate__(self, state): - state['mpi_comm'] = None - Y_range = state['Y_range'] - state['Y_local'] = state['Y'][Y_range[0]:Y_range[1]] - state['X_local'] = state['X'][Y_range[0]:Y_range[1]] return super(SSGPLVM, self).__setstate__(state) def _grads(self, x): From e7177b6d37adc42154c9860cf17e6b1073cacccf Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Sat, 17 May 2014 22:18:00 +0100 Subject: [PATCH 10/55] [mpi] fix the bug of mpi --- .../var_dtc_parallel.py | 2 +- GPy/models/ss_gplvm.py | 48 ++++++++++++------- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/GPy/inference/latent_function_inference/var_dtc_parallel.py b/GPy/inference/latent_function_inference/var_dtc_parallel.py index 7b11085e..dc4f45d5 100644 --- a/GPy/inference/latent_function_inference/var_dtc_parallel.py +++ b/GPy/inference/latent_function_inference/var_dtc_parallel.py @@ -78,7 +78,7 @@ class VarDTC_minibatch(LatentFunctionInference): num_inducing = Z.shape[0] num_data, output_dim = Y.shape - if self.batchsize == None or self.batchsize>num_data: + if self.batchsize == None: self.batchsize = num_data trYYT = self.get_trYYT(Y) diff --git a/GPy/models/ss_gplvm.py b/GPy/models/ss_gplvm.py index 8581bf7c..08dc6d95 100644 --- a/GPy/models/ss_gplvm.py +++ b/GPy/models/ss_gplvm.py @@ -14,7 +14,6 @@ from ..core.parameterization.variational import SpikeAndSlabPrior, SpikeAndSlabP from ..inference.latent_function_inference.var_dtc_parallel import update_gradients, VarDTC_minibatch from ..inference.latent_function_inference.var_dtc_gpu import VarDTC_GPU - class SSGPLVM(SparseGP): """ Spike-and-Slab Gaussian Process Latent Variable Model @@ -88,7 +87,8 @@ class SSGPLVM(SparseGP): self.X_local = self.X[Y_start:Y_end] self.Y_range = (Y_start, Y_end) self.Y_list = np.array(Y_list) - [mpi_comm.Bcast(p, root=0) for p in self.flattened_parameters] + print self.mpi_comm.rank, self.Y_range + mpi_comm.Bcast(self.param_array, root=0) def set_X_gradients(self, X, X_grad): """Set the gradients of the posterior distribution of X in its specific form.""" @@ -136,20 +136,32 @@ class SSGPLVM(SparseGP): def __setstate__(self, state): return super(SSGPLVM, self).__setstate__(state) - def _grads(self, x): + #===================================================== + # The MPI parallelization + # - can move to model at some point + #===================================================== + + def _set_params_transformed(self, p): if self.mpi_comm != None: - self.mpi_comm.Bcast(x, root=0) - obj_grads = super(SSGPLVM, self)._grads(x) - return obj_grads - - def _objective(self, x): - if self.mpi_comm != None: - self.mpi_comm.Bcast(x, root=0) - obj = super(SSGPLVM, self)._objective(x) - return obj - - def _objective_grads(self, x): - if self.mpi_comm != None: - self.mpi_comm.Bcast(x, root=0) - obj_f, obj_grads = super(SSGPLVM, self)._objective_grads(x) - return obj_f, obj_grads + if self.mpi_comm.rank==0: + self.mpi_comm.Bcast(np.int32(1),root=0) + self.mpi_comm.Bcast(p, root=0) + super(SSGPLVM, self)._set_params_transformed(p) + + def optimize(self, optimizer=None, start=None, **kwargs): + if self.mpi_comm==None: + super(SSGPLVM, self).optimize(optimizer,start,**kwargs) + elif self.mpi_comm.rank==0: + super(SSGPLVM, self).optimize(optimizer,start,**kwargs) + self.mpi_comm.Bcast(np.int32(-1),root=0) + elif self.mpi_comm.rank>0: + x = self._get_params_transformed().copy() + flag = np.empty(1,dtype=np.int32) + while True: + self.mpi_comm.Bcast(flag,root=0) + if flag==1: + self._set_params_transformed(x) + elif flag==-1: + break + else: + raise Exception("Unrecognizable flag for synchronization!") From 001db6b089c3a332fc9fed1078ee362461df0625 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Sun, 18 May 2014 10:05:35 +0100 Subject: [PATCH 11/55] [mpi] enable checkgrad --- GPy/models/ss_gplvm.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/GPy/models/ss_gplvm.py b/GPy/models/ss_gplvm.py index 08dc6d95..b0fe48b1 100644 --- a/GPy/models/ss_gplvm.py +++ b/GPy/models/ss_gplvm.py @@ -30,6 +30,7 @@ class SSGPLVM(SparseGP): Z=None, kernel=None, inference_method=None, likelihood=None, name='Spike-and-Slab GPLVM', group_spike=False, mpi_comm=None, **kwargs): self.mpi_comm = mpi_comm + self.__IN_OPTIMIZATION__ = False if X == None: from ..util.initialization import initialize_latent @@ -143,12 +144,13 @@ class SSGPLVM(SparseGP): def _set_params_transformed(self, p): if self.mpi_comm != None: - if self.mpi_comm.rank==0: + if self.__IN_OPTIMIZATION__ and self.mpi_comm.rank==0: self.mpi_comm.Bcast(np.int32(1),root=0) self.mpi_comm.Bcast(p, root=0) super(SSGPLVM, self)._set_params_transformed(p) - def optimize(self, optimizer=None, start=None, **kwargs): + def optimize(self, optimizer=None, start=None, **kwargs): + self.__IN_OPTIMIZATION__ = True if self.mpi_comm==None: super(SSGPLVM, self).optimize(optimizer,start,**kwargs) elif self.mpi_comm.rank==0: @@ -164,4 +166,6 @@ class SSGPLVM(SparseGP): elif flag==-1: break else: + self.__IN_OPTIMIZATION__ = False raise Exception("Unrecognizable flag for synchronization!") + self.__IN_OPTIMIZATION__ = False From a2203179f671e68993382e01dfbf51005b991816 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Wed, 21 May 2014 10:34:51 +0100 Subject: [PATCH 12/55] [psi2] implement RBF cpu --- GPy/core/sparse_gp.py | 2 - GPy/kern/_src/psi_comp/ssrbf_psi_comp.py | 2 +- GPy/kern/_src/rbf.py | 132 ++++++++++++----------- GPy/models/bayesian_gplvm.py | 4 +- GPy/plotting/matplot_dep/visualize.py | 27 +++-- 5 files changed, 92 insertions(+), 75 deletions(-) diff --git a/GPy/core/sparse_gp.py b/GPy/core/sparse_gp.py index 388b682a..7c963913 100644 --- a/GPy/core/sparse_gp.py +++ b/GPy/core/sparse_gp.py @@ -107,5 +107,3 @@ class SparseGP(GP): psi2 = kern.psi2(self.Z, Xnew) var = Kxx - np.sum(np.sum(psi2 * Kmmi_LmiBLmi[None, :, :], 1), 1) return mu, var - - diff --git a/GPy/kern/_src/psi_comp/ssrbf_psi_comp.py b/GPy/kern/_src/psi_comp/ssrbf_psi_comp.py index 8a24c60a..0ded9e14 100644 --- a/GPy/kern/_src/psi_comp/ssrbf_psi_comp.py +++ b/GPy/kern/_src/psi_comp/ssrbf_psi_comp.py @@ -7,7 +7,7 @@ The package for the psi statistics computation import numpy as np from . import PSICOMP -from GPy.util.caching import Cache_this,Cacher +from GPy.util.caching import Cache_this class PSICOMP_SSRBF(PSICOMP): diff --git a/GPy/kern/_src/rbf.py b/GPy/kern/_src/rbf.py index c75e4341..99692b0a 100644 --- a/GPy/kern/_src/rbf.py +++ b/GPy/kern/_src/rbf.py @@ -8,8 +8,7 @@ from ...util.misc import param_to_array from stationary import Stationary from GPy.util.caching import Cache_this from ...core.parameterization import variational -from psi_comp import ssrbf_psi_comp -from psi_comp import ssrbf_psi_gpucomp +from psi_comp import ssrbf_psi_comp,ssrbf_psi_gpucomp,rbf_psi_comp from ...util.config import * class RBF(Stationary): @@ -26,6 +25,7 @@ class RBF(Stationary): super(RBF, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name, useGPU=useGPU) self.weave_options = {} self.group_spike_prob = False + self.psicomp = rbf_psi_comp.PSICOMP_RBF() def set_for_SpikeAndSlab(self): if self.useGPU: @@ -50,7 +50,8 @@ class RBF(Stationary): else: return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[0] else: - return self.Kdiag(variational_posterior.mean) +# return self.Kdiag(variational_posterior.mean) + return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[0] def psi1(self, Z, variational_posterior): if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): @@ -59,8 +60,7 @@ class RBF(Stationary): else: return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[1] else: - _, _, _, psi1 = self._psi1computations(Z, variational_posterior) - return psi1 + return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[1] def psi2(self, Z, variational_posterior): if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): @@ -69,8 +69,7 @@ class RBF(Stationary): else: return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[2] else: - _, _, _, _, psi2 = self._psi2computations(Z, variational_posterior) - return psi2 + return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[2] def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): # Spike-and-Slab GPLVM @@ -83,32 +82,37 @@ class RBF(Stationary): self.lengthscale.gradient = dL_dlengscale elif isinstance(variational_posterior, variational.NormalPosterior): - l2 = self.lengthscale**2 - if l2.size != self.input_dim: - l2 = l2*np.ones(self.input_dim) + dL_dvar, dL_dlengscale, _, _, _ = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) + self.variance.gradient = dL_dvar + self.lengthscale.gradient = dL_dlengscale - #contributions from psi0: - self.variance.gradient = np.sum(dL_dpsi0) - self.lengthscale.gradient = 0. - - #from psi1 - denom, _, dist_sq, psi1 = self._psi1computations(Z, variational_posterior) - d_length = psi1[:,:,None] * ((dist_sq - 1.)/(self.lengthscale*denom) +1./self.lengthscale) - dpsi1_dlength = d_length * dL_dpsi1[:, :, None] - if self.ARD: - self.lengthscale.gradient += dpsi1_dlength.sum(0).sum(0) - else: - self.lengthscale.gradient += dpsi1_dlength.sum() - self.variance.gradient += np.sum(dL_dpsi1 * psi1) / self.variance - #from psi2 - S = variational_posterior.variance - _, Zdist_sq, _, mudist_sq, psi2 = self._psi2computations(Z, variational_posterior) - if not self.ARD: - self.lengthscale.gradient += self._weave_psi2_lengthscale_grads(dL_dpsi2, psi2, Zdist_sq, S, mudist_sq, l2).sum() - else: - self.lengthscale.gradient += self._weave_psi2_lengthscale_grads(dL_dpsi2, psi2, Zdist_sq, S, mudist_sq, l2) - - self.variance.gradient += 2.*np.sum(dL_dpsi2 * psi2)/self.variance +# l2 = self.lengthscale**2 +# if l2.size != self.input_dim: +# l2 = l2*np.ones(self.input_dim) +# #contributions from psi0: +# self.variance.gradient = np.sum(dL_dpsi0) +# self.lengthscale.gradient = 0. +# +# # #from psi1 +# denom, _, dist_sq, psi1 = self._psi1computations(Z, variational_posterior) +# d_length = psi1[:,:,None] * ((dist_sq - 1.)/(self.lengthscale*denom) +1./self.lengthscale) +# dpsi1_dlength = d_length * dL_dpsi1[:, :, None] +# print dpsi1_dlength.sum(0).sum(0) +# if self.ARD: +# self.lengthscale.gradient += dpsi1_dlength.sum(0).sum(0) +# else: +# self.lengthscale.gradient += dpsi1_dlength.sum() +# self.variance.gradient += np.sum(dL_dpsi1 * psi1) / self.variance +# #from psi2 +# S = variational_posterior.variance +# _, Zdist_sq, _, mudist_sq, psi2 = self._psi2computations(Z, variational_posterior) +# if not self.ARD: +# self.lengthscale.gradient += self._weave_psi2_lengthscale_grads(dL_dpsi2, psi2, Zdist_sq, S, mudist_sq, l2).sum() +# else: +# self.lengthscale.gradient += self._weave_psi2_lengthscale_grads(dL_dpsi2, psi2, Zdist_sq, S, mudist_sq, l2) +# # print self._weave_psi2_lengthscale_grads(dL_dpsi2, psi2, Zdist_sq, S, mudist_sq, l2) +# +# self.variance.gradient += 2.*np.sum(dL_dpsi2 * psi2)/self.variance else: raise ValueError, "unknown distriubtion received for psi-statistics" @@ -123,21 +127,24 @@ class RBF(Stationary): return dL_dZ elif isinstance(variational_posterior, variational.NormalPosterior): - l2 = self.lengthscale **2 - - #psi1 - denom, dist, dist_sq, psi1 = self._psi1computations(Z, variational_posterior) - grad = np.einsum('ij,ij,ijk,ijk->jk', dL_dpsi1, psi1, dist, -1./(denom*l2)) - - #psi2 - Zdist, Zdist_sq, mudist, mudist_sq, psi2 = self._psi2computations(Z, variational_posterior) - term1 = Zdist / l2 # M, M, Q - S = variational_posterior.variance - term2 = mudist / (2.*S[:,None,None,:] + l2) # N, M, M, Q - - grad += 2.*np.einsum('ijk,ijk,ijkl->kl', dL_dpsi2, psi2, term1[None,:,:,:] + term2) - - return grad + _, _, dL_dZ, _, _ = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) + return dL_dZ +# +# l2 = self.lengthscale **2 +# +# #psi1 +# denom, dist, dist_sq, psi1 = self._psi1computations(Z, variational_posterior) +# grad = np.einsum('ij,ij,ijk,ijk->jk', dL_dpsi1, psi1, dist, -1./(denom*l2)) +# +# #psi2 +# Zdist, Zdist_sq, mudist, mudist_sq, psi2 = self._psi2computations(Z, variational_posterior) +# term1 = Zdist / l2 # M, M, Q +# S = variational_posterior.variance +# term2 = mudist / (2.*S[:,None,None,:] + l2) # N, M, M, Q +# +# grad += 2.*np.einsum('ijk,ijk,ijkl->kl', dL_dpsi2, psi2, term1[None,:,:,:] + term2) +# +# return grad else: raise ValueError, "unknown distriubtion received for psi-statistics" @@ -151,24 +158,27 @@ class RBF(Stationary): return dL_dmu, dL_dS, dL_dgamma elif isinstance(variational_posterior, variational.NormalPosterior): - - l2 = self.lengthscale **2 - #psi1 - denom, dist, dist_sq, psi1 = self._psi1computations(Z, variational_posterior) - tmp = psi1[:, :, None] / l2 / denom - grad_mu = np.sum(dL_dpsi1[:, :, None] * tmp * dist, 1) - grad_S = np.sum(dL_dpsi1[:, :, None] * 0.5 * tmp * (dist_sq - 1), 1) - #psi2 - _, _, mudist, mudist_sq, psi2 = self._psi2computations(Z, variational_posterior) - S = variational_posterior.variance - tmp = psi2[:, :, :, None] / (2.*S[:,None,None,:] + l2) - grad_mu += -2.*np.einsum('ijk,ijkl,ijkl->il', dL_dpsi2, tmp , mudist) - grad_S += np.einsum('ijk,ijkl,ijkl->il', dL_dpsi2 , tmp , (2.*mudist_sq - 1)) + _, _, _, dL_dmu, dL_dS = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) + +# l2 = self.lengthscale **2 +# #psi1 +# denom, dist, dist_sq, psi1 = self._psi1computations(Z, variational_posterior) +# tmp = psi1[:, :, None] / l2 / denom +# grad_mu = np.sum(dL_dpsi1[:, :, None] * tmp * dist, 1) +# grad_S = np.sum(dL_dpsi1[:, :, None] * 0.5 * tmp * (dist_sq - 1), 1) +# #psi2 +# _, _, mudist, mudist_sq, psi2 = self._psi2computations(Z, variational_posterior) +# S = variational_posterior.variance +# tmp = psi2[:, :, :, None] / (2.*S[:,None,None,:] + l2) +# grad_mu += -2.*np.einsum('jk,ijkl,ijkl->il', dL_dpsi2, tmp , mudist) +# grad_S += np.einsum('jk,ijkl,ijkl->il', dL_dpsi2 , tmp , (2.*mudist_sq - 1)) + + return dL_dmu, dL_dS else: raise ValueError, "unknown distriubtion received for psi-statistics" - return grad_mu, grad_S + #return grad_mu, grad_S #---------------------------------------# # Precomputations # diff --git a/GPy/models/bayesian_gplvm.py b/GPy/models/bayesian_gplvm.py index 01cb07dc..2e7589a0 100644 --- a/GPy/models/bayesian_gplvm.py +++ b/GPy/models/bayesian_gplvm.py @@ -8,7 +8,7 @@ from ..likelihoods import Gaussian from ..inference.optimization import SCG from ..util import linalg from ..core.parameterization.variational import NormalPosterior, NormalPrior, VariationalPosterior -from ..inference.latent_function_inference.var_dtc_parallel import update_gradients +from ..inference.latent_function_inference.var_dtc_parallel import update_gradients, VarDTC_minibatch from ..inference.latent_function_inference.var_dtc_gpu import VarDTC_GPU class BayesianGPLVM(SparseGP): @@ -67,7 +67,7 @@ class BayesianGPLVM(SparseGP): X.mean.gradient, X.variance.gradient = X_grad def parameters_changed(self): - if isinstance(self.inference_method, VarDTC_GPU): + if isinstance(self.inference_method, VarDTC_GPU) or isinstance(self.inference_method, VarDTC_minibatch): update_gradients(self) return diff --git a/GPy/plotting/matplot_dep/visualize.py b/GPy/plotting/matplot_dep/visualize.py index d7884730..ee505af9 100644 --- a/GPy/plotting/matplot_dep/visualize.py +++ b/GPy/plotting/matplot_dep/visualize.py @@ -89,7 +89,7 @@ class vector_show(matplotlib_show): class lvm(matplotlib_show): - def __init__(self, vals, model, data_visualize, latent_axes=None, sense_axes=None, latent_index=[0,1]): + def __init__(self, vals, model, data_visualize, latent_axes=None, sense_axes=None, latent_index=[0,1], disable_drag=False): """Visualize a latent variable model :param model: the latent variable model to visualize. @@ -108,12 +108,14 @@ class lvm(matplotlib_show): if isinstance(latent_axes,mpl.axes.Axes): self.cid = latent_axes.figure.canvas.mpl_connect('button_press_event', self.on_click) - self.cid = latent_axes.figure.canvas.mpl_connect('motion_notify_event', self.on_move) + if not disable_drag: + self.cid = latent_axes.figure.canvas.mpl_connect('motion_notify_event', self.on_move) self.cid = latent_axes.figure.canvas.mpl_connect('axes_leave_event', self.on_leave) self.cid = latent_axes.figure.canvas.mpl_connect('axes_enter_event', self.on_enter) else: self.cid = latent_axes[0].figure.canvas.mpl_connect('button_press_event', self.on_click) - self.cid = latent_axes[0].figure.canvas.mpl_connect('motion_notify_event', self.on_move) + if not disable_drag: + self.cid = latent_axes[0].figure.canvas.mpl_connect('motion_notify_event', self.on_move) self.cid = latent_axes[0].figure.canvas.mpl_connect('axes_leave_event', self.on_leave) self.cid = latent_axes[0].figure.canvas.mpl_connect('axes_enter_event', self.on_enter) @@ -125,6 +127,7 @@ class lvm(matplotlib_show): self.move_on = False self.latent_index = latent_index self.latent_dim = model.input_dim + self.disable_drag = disable_drag # The red cross which shows current latent point. self.latent_values = vals @@ -149,8 +152,13 @@ class lvm(matplotlib_show): def on_click(self, event): print 'click!' if event.inaxes!=self.latent_axes: return - self.move_on = not self.move_on - self.called = True + if self.disable_drag: + self.move_on = True + self.called = True + self.on_move(event) + else: + self.move_on = not self.move_on + self.called = True def on_move(self, event): if event.inaxes!=self.latent_axes: return @@ -400,7 +408,7 @@ class mocap_data_show(matplotlib_show): def __init__(self, vals, axes=None, connect=None): if axes==None: fig = plt.figure() - axes = fig.add_subplot(111, projection='3d',aspect='equal') + axes = fig.add_subplot(111, projection='3d', aspect='equal') matplotlib_show.__init__(self, vals, axes) self.connect = connect @@ -438,6 +446,7 @@ class mocap_data_show(matplotlib_show): self.process_values() self.initialize_axes_modify() self.draw_vertices() + self.initialize_axes() self.finalize_axes_modify() self.draw_edges() self.axes.figure.canvas.draw() @@ -460,10 +469,10 @@ class mocap_data_show(matplotlib_show): self.axes.set_xlim(self.x_lim) self.axes.set_ylim(self.y_lim) self.axes.set_zlim(self.z_lim) - self.axes.auto_scale_xyz([-1., 1.], [-1., 1.], [-1.5, 1.5]) + self.axes.auto_scale_xyz([-1., 1.], [-1., 1.], [-1., 1.]) - #self.axes.set_aspect('equal') - self.axes.autoscale(enable=False) +# self.axes.set_aspect('equal') +# self.axes.autoscale(enable=False) def finalize_axes_modify(self): self.axes.set_xlim(self.x_lim) From 04ab93a9618cfd5dd1a8c09a9fbec8e69ec28b5f Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Wed, 21 May 2014 12:44:24 +0100 Subject: [PATCH 13/55] BayersianGPLVM mpi support --- GPy/models/bayesian_gplvm.py | 65 ++++++++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/GPy/models/bayesian_gplvm.py b/GPy/models/bayesian_gplvm.py index bc30bcfc..b77768ae 100644 --- a/GPy/models/bayesian_gplvm.py +++ b/GPy/models/bayesian_gplvm.py @@ -24,7 +24,9 @@ class BayesianGPLVM(SparseGP): """ def __init__(self, Y, input_dim, X=None, X_variance=None, init='PCA', num_inducing=10, - Z=None, kernel=None, inference_method=None, likelihood=None, name='bayesian gplvm', **kwargs): + Z=None, kernel=None, inference_method=None, likelihood=None, name='bayesian gplvm', mpi_comm=None, **kwargs): + self.mpi_comm = mpi_comm + self.__IN_OPTIMIZATION__ = False if X == None: from ..util.initialization import initialize_latent X, fracs = initialize_latent(init, input_dim, Y) @@ -55,6 +57,8 @@ class BayesianGPLVM(SparseGP): if np.any(np.isnan(Y)): from ..inference.latent_function_inference.var_dtc import VarDTCMissingData inference_method = VarDTCMissingData() + elif mpi_comm != None: + inference_method = VarDTC_minibatch(mpi_comm=mpi_comm) else: from ..inference.latent_function_inference.var_dtc import VarDTC inference_method = VarDTC() @@ -62,13 +66,26 @@ class BayesianGPLVM(SparseGP): SparseGP.__init__(self, X, Y, Z, kernel, likelihood, inference_method, name, **kwargs) self.add_parameter(self.X, index=0) + if mpi_comm != None: + from ..util.mpi import divide_data + Y_start, Y_end, Y_list = divide_data(Y.shape[0], mpi_comm) + self.Y_local = self.Y[Y_start:Y_end] + self.X_local = self.X[Y_start:Y_end] + self.Y_range = (Y_start, Y_end) + self.Y_list = np.array(Y_list) + mpi_comm.Bcast(self.param_array, root=0) + def set_X_gradients(self, X, X_grad): """Set the gradients of the posterior distribution of X in its specific form.""" X.mean.gradient, X.variance.gradient = X_grad + + def get_X_gradients(self, X): + """Get the gradients of the posterior distribution of X in its specific form.""" + return X.mean.gradient, X.variance.gradient def parameters_changed(self): if isinstance(self.inference_method, VarDTC_GPU) or isinstance(self.inference_method, VarDTC_minibatch): - update_gradients(self) + update_gradients(self, mpi_comm=self.mpi_comm) return super(BayesianGPLVM, self).parameters_changed() @@ -160,6 +177,50 @@ class BayesianGPLVM(SparseGP): return dim_reduction_plots.plot_steepest_gradient_map(self,*args,**kwargs) + def __getstate__(self): + dc = super(BayesianGPLVM, self).__getstate__() + dc['mpi_comm'] = None + if self.mpi_comm != None: + del dc['Y_local'] + del dc['X_local'] + del dc['Y_range'] + return dc + + def __setstate__(self, state): + return super(BayesianGPLVM, self).__setstate__(state) + + #===================================================== + # The MPI parallelization + # - can move to model at some point + #===================================================== + + def _set_params_transformed(self, p): + if self.mpi_comm != None: + if self.__IN_OPTIMIZATION__ and self.mpi_comm.rank==0: + self.mpi_comm.Bcast(np.int32(1),root=0) + self.mpi_comm.Bcast(p, root=0) + super(BayesianGPLVM, self)._set_params_transformed(p) + + def optimize(self, optimizer=None, start=None, **kwargs): + self.__IN_OPTIMIZATION__ = True + if self.mpi_comm==None: + super(BayesianGPLVM, self).optimize(optimizer,start,**kwargs) + elif self.mpi_comm.rank==0: + super(BayesianGPLVM, self).optimize(optimizer,start,**kwargs) + self.mpi_comm.Bcast(np.int32(-1),root=0) + elif self.mpi_comm.rank>0: + x = self._get_params_transformed().copy() + flag = np.empty(1,dtype=np.int32) + while True: + self.mpi_comm.Bcast(flag,root=0) + if flag==1: + self._set_params_transformed(x) + elif flag==-1: + break + else: + self.__IN_OPTIMIZATION__ = False + raise Exception("Unrecognizable flag for synchronization!") + self.__IN_OPTIMIZATION__ = False def latent_cost_and_grad(mu_S, kern, Z, dL_dpsi0, dL_dpsi1, dL_dpsi2): """ From b945c2000467d8217b83170864b22a386d560859 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Wed, 21 May 2014 16:42:35 +0100 Subject: [PATCH 14/55] restructure rbf kernel --- GPy/kern/_src/psi_comp/__init__.py | 23 ++ GPy/kern/_src/psi_comp/rbf_psi_comp.py | 308 +++++++++---------- GPy/kern/_src/psi_comp/ssrbf_psi_comp.py | 368 +++++++++++------------ GPy/kern/_src/rbf.py | 292 ++---------------- GPy/models/ss_gplvm.py | 1 - 5 files changed, 372 insertions(+), 620 deletions(-) diff --git a/GPy/kern/_src/psi_comp/__init__.py b/GPy/kern/_src/psi_comp/__init__.py index 17ea2e83..77668e7f 100644 --- a/GPy/kern/_src/psi_comp/__init__.py +++ b/GPy/kern/_src/psi_comp/__init__.py @@ -2,6 +2,10 @@ # Licensed under the BSD 3-clause license (see LICENSE.txt) from ....core.parameterization.parameter_core import Pickleable +from GPy.util.caching import Cache_this +from ....core.parameterization import variational +import rbf_psi_comp +import ssrbf_psi_comp class PSICOMP(Pickleable): @@ -17,3 +21,22 @@ class PSICOMP(Pickleable): """ pass +class PSICOMP_RBF(Pickleable): + + @Cache_this(limit=1, ignore_args=(0,)) + def psicomputations(self, variance, lengthscale, Z, variational_posterior): + if isinstance(variational_posterior, variational.NormalPosterior): + return rbf_psi_comp.psicomputations(variance, lengthscale, Z, variational_posterior) + elif isinstance(variational_posterior, variational.SpikeAndSlabPosterior): + return ssrbf_psi_comp.psicomputations(variance, lengthscale, Z, variational_posterior) + else: + raise ValueError, "unknown distriubtion received for psi-statistics" + + @Cache_this(limit=1, ignore_args=(0,1,2,3)) + def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior): + if isinstance(variational_posterior, variational.NormalPosterior): + return rbf_psi_comp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior) + elif isinstance(variational_posterior, variational.SpikeAndSlabPosterior): + return ssrbf_psi_comp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior) + else: + raise ValueError, "unknown distriubtion received for psi-statistics" \ No newline at end of file diff --git a/GPy/kern/_src/psi_comp/rbf_psi_comp.py b/GPy/kern/_src/psi_comp/rbf_psi_comp.py index d5cd8951..93399ea7 100644 --- a/GPy/kern/_src/psi_comp/rbf_psi_comp.py +++ b/GPy/kern/_src/psi_comp/rbf_psi_comp.py @@ -3,168 +3,160 @@ The module for psi-statistics for RBF kernel """ import numpy as np -from . import PSICOMP -from GPy.util.caching import Cache_this -from ....util.misc import param_to_array -from scipy import weave -from ....util.config import * +from GPy.util.caching import Cacher -class PSICOMP_RBF(PSICOMP): +def psicomputations(variance, lengthscale, Z, variational_posterior): + """ + Z - MxQ + mu - NxQ + S - NxQ + gamma - NxQ + """ + # here are the "statistics" for psi0, psi1 and psi2 + # Produced intermediate results: + # _psi1 NxM + mu = variational_posterior.mean + S = variational_posterior.variance - @Cache_this(limit=1, ignore_args=(0,)) - def psicomputations(self, variance, lengthscale, Z, variational_posterior): - """ - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - """ - # here are the "statistics" for psi0, psi1 and psi2 - # Produced intermediate results: - # _psi1 NxM - mu = variational_posterior.mean - S = variational_posterior.variance - - psi0 = np.empty(mu.shape[0]) - psi0[:] = variance - psi1 = self._psi1computations(variance, lengthscale, Z, mu, S) - psi2 = self._psi2computations(variance, lengthscale, Z, mu, S).sum(axis=0) - return psi0, psi1, psi2 - - @Cache_this(limit=1, ignore_args=(0,)) - def _psi1computations(self, variance, lengthscale, Z, mu, S): - """ - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - """ - # here are the "statistics" for psi1 - # Produced intermediate results: - # _psi1 NxM - - lengthscale2 = np.square(lengthscale) - - # psi1 - _psi1_logdenom = np.log(S/lengthscale2+1.).sum(axis=-1) # N - _psi1_log = (_psi1_logdenom[:,None]+np.einsum('nmq,nq->nm',np.square(mu[:,None,:]-Z[None,:,:]),1./(S+lengthscale2)))/(-2.) - _psi1 = variance*np.exp(_psi1_log) - - return _psi1 - - @Cache_this(limit=1, ignore_args=(0,)) - def _psi2computations(self, variance, lengthscale, Z, mu, S): - """ - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - """ - # here are the "statistics" for psi2 - # Produced intermediate results: - # _psi2 MxM - - lengthscale2 = np.square(lengthscale) - - _psi2_logdenom = np.log(2.*S/lengthscale2+1.).sum(axis=-1)/(-2.) # N - _psi2_exp1 = (np.square(Z[:,None,:]-Z[None,:,:])/lengthscale2).sum(axis=-1)/(-4.) #MxM - Z_hat = (Z[:,None,:]+Z[None,:,:])/2. #MxMxQ - denom = 1./(2.*S+lengthscale2) - _psi2_exp2 = -(np.square(mu)*denom).sum(axis=-1)[:,None,None]+2.*np.einsum('nq,moq,nq->nmo',mu,Z_hat,denom)-np.einsum('moq,nq->nmo',np.square(Z_hat),denom) - _psi2 = variance*variance*np.exp(_psi2_logdenom[:,None,None]+_psi2_exp1[None,:,:]+_psi2_exp2) - - - return _psi2 - - @Cache_this(limit=1, ignore_args=(0,1,2,3)) - def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior): - ARD = (len(lengthscale)!=1) - - dvar_psi1, dl_psi1, dZ_psi1, dmu_psi1, dS_psi1 = self._psi1compDer(dL_dpsi1, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance) - dvar_psi2, dl_psi2, dZ_psi2, dmu_psi2, dS_psi2 = self._psi2compDer(dL_dpsi2, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance) - - dL_dvar = np.sum(dL_dpsi0) + dvar_psi1 + dvar_psi2 - - dL_dlengscale = dl_psi1 + dl_psi2 - if not ARD: - dL_dlengscale = dL_dlengscale.sum() + psi0 = np.empty(mu.shape[0]) + psi0[:] = variance + psi1 = _psi1computations(variance, lengthscale, Z, mu, S) + psi2 = _psi2computations(variance, lengthscale, Z, mu, S).sum(axis=0) + return psi0, psi1, psi2 - dL_dmu = dmu_psi1 + dmu_psi2 - dL_dS = dS_psi1 + dS_psi2 - dL_dZ = dZ_psi1 + dZ_psi2 - - return dL_dvar, dL_dlengscale, dL_dZ, dL_dmu, dL_dS - - def _psi1compDer(self, dL_dpsi1, variance, lengthscale, Z, mu, S): - """ - dL_dpsi1 - NxM - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - """ - # here are the "statistics" for psi1 - # Produced intermediate results: dL_dparams w.r.t. psi1 - # _dL_dvariance 1 - # _dL_dlengthscale Q - # _dL_dZ MxQ - # _dL_dgamma NxQ - # _dL_dmu NxQ - # _dL_dS NxQ - - lengthscale2 = np.square(lengthscale) - - _psi1 = self._psi1computations(variance, lengthscale, Z, mu, S) - Lpsi1 = dL_dpsi1*_psi1 - Zmu = Z[None,:,:]-mu[:,None,:] # NxMxQ - denom = 1./(S+lengthscale2) - Zmu2_denom = np.square(Zmu)*denom[:,None,:] #NxMxQ - _dL_dvar = Lpsi1.sum()/variance - _dL_dmu = np.einsum('nm,nmq,nq->nq',Lpsi1,Zmu,denom) - _dL_dS = np.einsum('nm,nmq,nq->nq',Lpsi1,(Zmu2_denom-1.),denom)/2. - _dL_dZ = -np.einsum('nm,nmq,nq->mq',Lpsi1,Zmu,denom) - _dL_dl = np.einsum('nm,nmq,nq->q',Lpsi1,(Zmu2_denom+(S/lengthscale2)[:,None,:]),denom*lengthscale) - - return _dL_dvar, _dL_dl, _dL_dZ, _dL_dmu, _dL_dS - - def _psi2compDer(self, dL_dpsi2, variance, lengthscale, Z, mu, S): - """ - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - dL_dpsi2 - MxM - """ - # here are the "statistics" for psi2 - # Produced the derivatives w.r.t. psi2: - # _dL_dvariance 1 - # _dL_dlengthscale Q - # _dL_dZ MxQ - # _dL_dgamma NxQ - # _dL_dmu NxQ - # _dL_dS NxQ - - lengthscale2 = np.square(lengthscale) - denom = 1./(2*S+lengthscale2) - denom2 = np.square(denom) +def __psi1computations(variance, lengthscale, Z, mu, S): + """ + Z - MxQ + mu - NxQ + S - NxQ + gamma - NxQ + """ + # here are the "statistics" for psi1 + # Produced intermediate results: + # _psi1 NxM - _psi2 = self._psi2computations(variance, lengthscale, Z, mu, S) # NxMxM + lengthscale2 = np.square(lengthscale) + + # psi1 + _psi1_logdenom = np.log(S/lengthscale2+1.).sum(axis=-1) # N + _psi1_log = (_psi1_logdenom[:,None]+np.einsum('nmq,nq->nm',np.square(mu[:,None,:]-Z[None,:,:]),1./(S+lengthscale2)))/(-2.) + _psi1 = variance*np.exp(_psi1_log) - Lpsi2 = dL_dpsi2[None,:,:]*_psi2 - Lpsi2sum = np.einsum('nmo->n',Lpsi2) #N - Lpsi2Z = np.einsum('nmo,oq->nq',Lpsi2,Z) #NxQ - Lpsi2Z2 = np.einsum('nmo,oq,oq->nq',Lpsi2,Z,Z) #NxQ - Lpsi2Z2p = np.einsum('nmo,mq,oq->nq',Lpsi2,Z,Z) #NxQ - Lpsi2Zhat = Lpsi2Z - Lpsi2Zhat2 = (Lpsi2Z2+Lpsi2Z2p)/2 - - _dL_dvar = Lpsi2sum.sum()*2/variance - _dL_dmu = (-2*denom) * (mu*Lpsi2sum[:,None]-Lpsi2Zhat) - _dL_dS = (2*np.square(denom))*(np.square(mu)*Lpsi2sum[:,None]-2*mu*Lpsi2Zhat+Lpsi2Zhat2) - denom*Lpsi2sum[:,None] - _dL_dZ = -np.einsum('nmo,oq->oq',Lpsi2,Z)/lengthscale2+np.einsum('nmo,oq->mq',Lpsi2,Z)/lengthscale2+ \ - 2*np.einsum('nmo,nq,nq->mq',Lpsi2,mu,denom) - np.einsum('nmo,nq,mq->mq',Lpsi2,denom,Z) - np.einsum('nmo,oq,nq->mq',Lpsi2,Z,denom) - _dL_dl = 2*lengthscale* ((S/lengthscale2*denom+np.square(mu*denom))*Lpsi2sum[:,None]+(Lpsi2Z2-Lpsi2Z2p)/(2*np.square(lengthscale2))- - (2*mu*denom2)*Lpsi2Zhat+denom2*Lpsi2Zhat2).sum(axis=0) + return _psi1 + +def __psi2computations(variance, lengthscale, Z, mu, S): + """ + Z - MxQ + mu - NxQ + S - NxQ + gamma - NxQ + """ + # here are the "statistics" for psi2 + # Produced intermediate results: + # _psi2 MxM - return _dL_dvar, _dL_dl, _dL_dZ, _dL_dmu, _dL_dS + lengthscale2 = np.square(lengthscale) + _psi2_logdenom = np.log(2.*S/lengthscale2+1.).sum(axis=-1)/(-2.) # N + _psi2_exp1 = (np.square(Z[:,None,:]-Z[None,:,:])/lengthscale2).sum(axis=-1)/(-4.) #MxM + Z_hat = (Z[:,None,:]+Z[None,:,:])/2. #MxMxQ + denom = 1./(2.*S+lengthscale2) + _psi2_exp2 = -(np.square(mu)*denom).sum(axis=-1)[:,None,None]+2.*np.einsum('nq,moq,nq->nmo',mu,Z_hat,denom)-np.einsum('moq,nq->nmo',np.square(Z_hat),denom) + _psi2 = variance*variance*np.exp(_psi2_logdenom[:,None,None]+_psi2_exp1[None,:,:]+_psi2_exp2) + + + return _psi2 + +def psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior): + ARD = (len(lengthscale)!=1) + + dvar_psi1, dl_psi1, dZ_psi1, dmu_psi1, dS_psi1 = _psi1compDer(dL_dpsi1, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance) + dvar_psi2, dl_psi2, dZ_psi2, dmu_psi2, dS_psi2 = _psi2compDer(dL_dpsi2, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance) + + dL_dvar = np.sum(dL_dpsi0) + dvar_psi1 + dvar_psi2 + + dL_dlengscale = dl_psi1 + dl_psi2 + if not ARD: + dL_dlengscale = dL_dlengscale.sum() + + dL_dmu = dmu_psi1 + dmu_psi2 + dL_dS = dS_psi1 + dS_psi2 + dL_dZ = dZ_psi1 + dZ_psi2 + + return dL_dvar, dL_dlengscale, dL_dZ, dL_dmu, dL_dS + +def _psi1compDer(dL_dpsi1, variance, lengthscale, Z, mu, S): + """ + dL_dpsi1 - NxM + Z - MxQ + mu - NxQ + S - NxQ + gamma - NxQ + """ + # here are the "statistics" for psi1 + # Produced intermediate results: dL_dparams w.r.t. psi1 + # _dL_dvariance 1 + # _dL_dlengthscale Q + # _dL_dZ MxQ + # _dL_dgamma NxQ + # _dL_dmu NxQ + # _dL_dS NxQ + + lengthscale2 = np.square(lengthscale) + + _psi1 = _psi1computations(variance, lengthscale, Z, mu, S) + Lpsi1 = dL_dpsi1*_psi1 + Zmu = Z[None,:,:]-mu[:,None,:] # NxMxQ + denom = 1./(S+lengthscale2) + Zmu2_denom = np.square(Zmu)*denom[:,None,:] #NxMxQ + _dL_dvar = Lpsi1.sum()/variance + _dL_dmu = np.einsum('nm,nmq,nq->nq',Lpsi1,Zmu,denom) + _dL_dS = np.einsum('nm,nmq,nq->nq',Lpsi1,(Zmu2_denom-1.),denom)/2. + _dL_dZ = -np.einsum('nm,nmq,nq->mq',Lpsi1,Zmu,denom) + _dL_dl = np.einsum('nm,nmq,nq->q',Lpsi1,(Zmu2_denom+(S/lengthscale2)[:,None,:]),denom*lengthscale) + + return _dL_dvar, _dL_dl, _dL_dZ, _dL_dmu, _dL_dS + +def _psi2compDer(dL_dpsi2, variance, lengthscale, Z, mu, S): + """ + Z - MxQ + mu - NxQ + S - NxQ + gamma - NxQ + dL_dpsi2 - MxM + """ + # here are the "statistics" for psi2 + # Produced the derivatives w.r.t. psi2: + # _dL_dvariance 1 + # _dL_dlengthscale Q + # _dL_dZ MxQ + # _dL_dgamma NxQ + # _dL_dmu NxQ + # _dL_dS NxQ + + lengthscale2 = np.square(lengthscale) + denom = 1./(2*S+lengthscale2) + denom2 = np.square(denom) + + _psi2 = _psi2computations(variance, lengthscale, Z, mu, S) # NxMxM + + Lpsi2 = dL_dpsi2[None,:,:]*_psi2 + Lpsi2sum = np.einsum('nmo->n',Lpsi2) #N + Lpsi2Z = np.einsum('nmo,oq->nq',Lpsi2,Z) #NxQ + Lpsi2Z2 = np.einsum('nmo,oq,oq->nq',Lpsi2,Z,Z) #NxQ + Lpsi2Z2p = np.einsum('nmo,mq,oq->nq',Lpsi2,Z,Z) #NxQ + Lpsi2Zhat = Lpsi2Z + Lpsi2Zhat2 = (Lpsi2Z2+Lpsi2Z2p)/2 + + _dL_dvar = Lpsi2sum.sum()*2/variance + _dL_dmu = (-2*denom) * (mu*Lpsi2sum[:,None]-Lpsi2Zhat) + _dL_dS = (2*np.square(denom))*(np.square(mu)*Lpsi2sum[:,None]-2*mu*Lpsi2Zhat+Lpsi2Zhat2) - denom*Lpsi2sum[:,None] + _dL_dZ = -np.einsum('nmo,oq->oq',Lpsi2,Z)/lengthscale2+np.einsum('nmo,oq->mq',Lpsi2,Z)/lengthscale2+ \ + 2*np.einsum('nmo,nq,nq->mq',Lpsi2,mu,denom) - np.einsum('nmo,nq,mq->mq',Lpsi2,denom,Z) - np.einsum('nmo,oq,nq->mq',Lpsi2,Z,denom) + _dL_dl = 2*lengthscale* ((S/lengthscale2*denom+np.square(mu*denom))*Lpsi2sum[:,None]+(Lpsi2Z2-Lpsi2Z2p)/(2*np.square(lengthscale2))- + (2*mu*denom2)*Lpsi2Zhat+denom2*Lpsi2Zhat2).sum(axis=0) + + return _dL_dvar, _dL_dl, _dL_dZ, _dL_dmu, _dL_dS + +_psi1computations = Cacher(__psi1computations, limit=1) +_psi2computations = Cacher(__psi2computations, limit=1) diff --git a/GPy/kern/_src/psi_comp/ssrbf_psi_comp.py b/GPy/kern/_src/psi_comp/ssrbf_psi_comp.py index 0ded9e14..6302a590 100644 --- a/GPy/kern/_src/psi_comp/ssrbf_psi_comp.py +++ b/GPy/kern/_src/psi_comp/ssrbf_psi_comp.py @@ -6,200 +6,194 @@ The package for the psi statistics computation """ import numpy as np -from . import PSICOMP -from GPy.util.caching import Cache_this -class PSICOMP_SSRBF(PSICOMP): +def psicomputations(variance, lengthscale, Z, variational_posterior): + """ + Z - MxQ + mu - NxQ + S - NxQ + gamma - NxQ + """ + # here are the "statistics" for psi0, psi1 and psi2 + # Produced intermediate results: + # _psi1 NxM + mu = variational_posterior.mean + S = variational_posterior.variance + gamma = variational_posterior.binary_prob - @Cache_this(limit=1, ignore_args=(0,)) - def psicomputations(self, variance, lengthscale, Z, variational_posterior): - """ - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - """ - # here are the "statistics" for psi0, psi1 and psi2 - # Produced intermediate results: - # _psi1 NxM - mu = variational_posterior.mean - S = variational_posterior.variance - gamma = variational_posterior.binary_prob - - psi0 = np.empty(mu.shape[0]) - psi0[:] = variance - psi1 = self._psi1computations(variance, lengthscale, Z, mu, S, gamma) - psi2 = self._psi2computations(variance, lengthscale, Z, mu, S, gamma) - return psi0, psi1, psi2 + psi0 = np.empty(mu.shape[0]) + psi0[:] = variance + psi1 = _psi1computations(variance, lengthscale, Z, mu, S, gamma) + psi2 = _psi2computations(variance, lengthscale, Z, mu, S, gamma) + return psi0, psi1, psi2 + +def _psi1computations(variance, lengthscale, Z, mu, S, gamma): + """ + Z - MxQ + mu - NxQ + S - NxQ + gamma - NxQ + """ + # here are the "statistics" for psi1 + # Produced intermediate results: + # _psi1 NxM + + lengthscale2 = np.square(lengthscale) + + # psi1 + _psi1_denom = S[:, None, :] / lengthscale2 + 1. # Nx1xQ + _psi1_denom_sqrt = np.sqrt(_psi1_denom) #Nx1xQ + _psi1_dist = Z[None, :, :] - mu[:, None, :] # NxMxQ + _psi1_dist_sq = np.square(_psi1_dist) / (lengthscale2 * _psi1_denom) # NxMxQ + _psi1_common = gamma[:,None,:] / (lengthscale2*_psi1_denom*_psi1_denom_sqrt) #Nx1xQ + _psi1_exponent1 = np.log(gamma[:,None,:]) - (_psi1_dist_sq + np.log(_psi1_denom))/2. # NxMxQ + _psi1_exponent2 = np.log(1.-gamma[:,None,:]) - (np.square(Z[None,:,:])/lengthscale2)/2. # NxMxQ + _psi1_exponent_max = np.maximum(_psi1_exponent1,_psi1_exponent2) + _psi1_exponent = _psi1_exponent_max+np.log(np.exp(_psi1_exponent1-_psi1_exponent_max) + np.exp(_psi1_exponent2-_psi1_exponent_max)) #NxMxQ + _psi1_exp_sum = _psi1_exponent.sum(axis=-1) #NxM + _psi1 = variance * np.exp(_psi1_exp_sum) # NxM + + return _psi1 + +def _psi2computations(variance, lengthscale, Z, mu, S, gamma): + """ + Z - MxQ + mu - NxQ + S - NxQ + gamma - NxQ + """ + # here are the "statistics" for psi2 + # Produced intermediate results: + # _psi2 MxM - def _psi1computations(self, variance, lengthscale, Z, mu, S, gamma): - """ - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - """ - # here are the "statistics" for psi1 - # Produced intermediate results: - # _psi1 NxM + lengthscale2 = np.square(lengthscale) - lengthscale2 = np.square(lengthscale) + _psi2_Zhat = 0.5 * (Z[:, None, :] + Z[None, :, :]) # M,M,Q + _psi2_Zdist = 0.5 * (Z[:, None, :] - Z[None, :, :]) # M,M,Q + _psi2_Zdist_sq = np.square(_psi2_Zdist / lengthscale) # M,M,Q + _psi2_Z_sq_sum = (np.square(Z[:,None,:])+np.square(Z[None,:,:]))/lengthscale2 # MxMxQ + + # psi2 + _psi2_denom = 2.*S[:, None, None, :] / lengthscale2 + 1. # Nx1x1xQ + _psi2_denom_sqrt = np.sqrt(_psi2_denom) + _psi2_mudist = mu[:,None,None,:]-_psi2_Zhat #N,M,M,Q + _psi2_mudist_sq = np.square(_psi2_mudist)/(lengthscale2*_psi2_denom) + _psi2_common = gamma[:,None,None,:]/(lengthscale2 * _psi2_denom * _psi2_denom_sqrt) # Nx1x1xQ + _psi2_exponent1 = -_psi2_Zdist_sq -_psi2_mudist_sq -0.5*np.log(_psi2_denom)+np.log(gamma[:,None,None,:]) #N,M,M,Q + _psi2_exponent2 = np.log(1.-gamma[:,None,None,:]) - 0.5*(_psi2_Z_sq_sum) # NxMxMxQ + _psi2_exponent_max = np.maximum(_psi2_exponent1, _psi2_exponent2) + _psi2_exponent = _psi2_exponent_max+np.log(np.exp(_psi2_exponent1-_psi2_exponent_max) + np.exp(_psi2_exponent2-_psi2_exponent_max)) + _psi2_exp_sum = _psi2_exponent.sum(axis=-1) #NxM + _psi2 = variance*variance * (np.exp(_psi2_exp_sum).sum(axis=0)) # MxM + + return _psi2 + +def psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior): + ARD = (len(lengthscale)!=1) - # psi1 - _psi1_denom = S[:, None, :] / lengthscale2 + 1. # Nx1xQ - _psi1_denom_sqrt = np.sqrt(_psi1_denom) #Nx1xQ - _psi1_dist = Z[None, :, :] - mu[:, None, :] # NxMxQ - _psi1_dist_sq = np.square(_psi1_dist) / (lengthscale2 * _psi1_denom) # NxMxQ - _psi1_common = gamma[:,None,:] / (lengthscale2*_psi1_denom*_psi1_denom_sqrt) #Nx1xQ - _psi1_exponent1 = np.log(gamma[:,None,:]) - (_psi1_dist_sq + np.log(_psi1_denom))/2. # NxMxQ - _psi1_exponent2 = np.log(1.-gamma[:,None,:]) - (np.square(Z[None,:,:])/lengthscale2)/2. # NxMxQ - _psi1_exponent_max = np.maximum(_psi1_exponent1,_psi1_exponent2) - _psi1_exponent = _psi1_exponent_max+np.log(np.exp(_psi1_exponent1-_psi1_exponent_max) + np.exp(_psi1_exponent2-_psi1_exponent_max)) #NxMxQ - _psi1_exp_sum = _psi1_exponent.sum(axis=-1) #NxM - _psi1 = variance * np.exp(_psi1_exp_sum) # NxM + dvar_psi1, dl_psi1, dZ_psi1, dmu_psi1, dS_psi1, dgamma_psi1 = _psi1compDer(dL_dpsi1, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) + dvar_psi2, dl_psi2, dZ_psi2, dmu_psi2, dS_psi2, dgamma_psi2 = _psi2compDer(dL_dpsi2, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) + + dL_dvar = np.sum(dL_dpsi0) + dvar_psi1 + dvar_psi2 - return _psi1 + dL_dlengscale = dl_psi1 + dl_psi2 + if not ARD: + dL_dlengscale = dL_dlengscale.sum() + + dL_dgamma = dgamma_psi1 + dgamma_psi2 + dL_dmu = dmu_psi1 + dmu_psi2 + dL_dS = dS_psi1 + dS_psi2 + dL_dZ = dZ_psi1 + dZ_psi2 - def _psi2computations(self, variance, lengthscale, Z, mu, S, gamma): - """ - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - """ - # here are the "statistics" for psi2 - # Produced intermediate results: - # _psi2 MxM - - lengthscale2 = np.square(lengthscale) - - _psi2_Zhat = 0.5 * (Z[:, None, :] + Z[None, :, :]) # M,M,Q - _psi2_Zdist = 0.5 * (Z[:, None, :] - Z[None, :, :]) # M,M,Q - _psi2_Zdist_sq = np.square(_psi2_Zdist / lengthscale) # M,M,Q - _psi2_Z_sq_sum = (np.square(Z[:,None,:])+np.square(Z[None,:,:]))/lengthscale2 # MxMxQ + return dL_dvar, dL_dlengscale, dL_dZ, dL_dmu, dL_dS, dL_dgamma + +def _psi1compDer(dL_dpsi1, variance, lengthscale, Z, mu, S, gamma): + """ + dL_dpsi1 - NxM + Z - MxQ + mu - NxQ + S - NxQ + gamma - NxQ + """ + # here are the "statistics" for psi1 + # Produced intermediate results: dL_dparams w.r.t. psi1 + # _dL_dvariance 1 + # _dL_dlengthscale Q + # _dL_dZ MxQ + # _dL_dgamma NxQ + # _dL_dmu NxQ + # _dL_dS NxQ - # psi2 - _psi2_denom = 2.*S[:, None, None, :] / lengthscale2 + 1. # Nx1x1xQ - _psi2_denom_sqrt = np.sqrt(_psi2_denom) - _psi2_mudist = mu[:,None,None,:]-_psi2_Zhat #N,M,M,Q - _psi2_mudist_sq = np.square(_psi2_mudist)/(lengthscale2*_psi2_denom) - _psi2_common = gamma[:,None,None,:]/(lengthscale2 * _psi2_denom * _psi2_denom_sqrt) # Nx1x1xQ - _psi2_exponent1 = -_psi2_Zdist_sq -_psi2_mudist_sq -0.5*np.log(_psi2_denom)+np.log(gamma[:,None,None,:]) #N,M,M,Q - _psi2_exponent2 = np.log(1.-gamma[:,None,None,:]) - 0.5*(_psi2_Z_sq_sum) # NxMxMxQ - _psi2_exponent_max = np.maximum(_psi2_exponent1, _psi2_exponent2) - _psi2_exponent = _psi2_exponent_max+np.log(np.exp(_psi2_exponent1-_psi2_exponent_max) + np.exp(_psi2_exponent2-_psi2_exponent_max)) - _psi2_exp_sum = _psi2_exponent.sum(axis=-1) #NxM - _psi2 = variance*variance * (np.exp(_psi2_exp_sum).sum(axis=0)) # MxM + lengthscale2 = np.square(lengthscale) + + # psi1 + _psi1_denom = S / lengthscale2 + 1. # NxQ + _psi1_denom_sqrt = np.sqrt(_psi1_denom) #NxQ + _psi1_dist = Z[None, :, :] - mu[:, None, :] # NxMxQ + _psi1_dist_sq = np.square(_psi1_dist) / (lengthscale2 * _psi1_denom[:,None,:]) # NxMxQ + _psi1_common = gamma / (lengthscale2*_psi1_denom*_psi1_denom_sqrt) #NxQ + _psi1_exponent1 = np.log(gamma[:,None,:]) -0.5 * (_psi1_dist_sq + np.log(_psi1_denom[:, None,:])) # NxMxQ + _psi1_exponent2 = np.log(1.-gamma[:,None,:]) -0.5 * (np.square(Z[None,:,:])/lengthscale2) # NxMxQ + _psi1_exponent_max = np.maximum(_psi1_exponent1,_psi1_exponent2) + _psi1_exponent = _psi1_exponent_max+np.log(np.exp(_psi1_exponent1-_psi1_exponent_max) + np.exp(_psi1_exponent2-_psi1_exponent_max)) #NxMxQ + _psi1_exp_sum = _psi1_exponent.sum(axis=-1) #NxM + _psi1_exp_dist_sq = np.exp(-0.5*_psi1_dist_sq) # NxMxQ + _psi1_exp_Z = np.exp(-0.5*np.square(Z[None,:,:])/lengthscale2) # 1xMxQ + _psi1_q = variance * np.exp(_psi1_exp_sum[:,:,None] - _psi1_exponent) # NxMxQ + _psi1 = variance * np.exp(_psi1_exp_sum) # NxM + _dL_dvariance = np.einsum('nm,nm->',dL_dpsi1, _psi1)/variance # 1 + _dL_dgamma = np.einsum('nm,nmq,nmq->nq',dL_dpsi1, _psi1_q, (_psi1_exp_dist_sq/_psi1_denom_sqrt[:,None,:]-_psi1_exp_Z)) # NxQ + _dL_dmu = np.einsum('nm, nmq, nmq, nmq, nq->nq',dL_dpsi1,_psi1_q,_psi1_exp_dist_sq,_psi1_dist,_psi1_common) # NxQ + _dL_dS = np.einsum('nm,nmq,nmq,nq,nmq->nq',dL_dpsi1,_psi1_q,_psi1_exp_dist_sq,_psi1_common,(_psi1_dist_sq-1.))/2. # NxQ + _dL_dZ = np.einsum('nm,nmq,nmq->mq',dL_dpsi1,_psi1_q, (- _psi1_common[:,None,:] * _psi1_dist * _psi1_exp_dist_sq - (1-gamma[:,None,:])/lengthscale2*Z[None,:,:]*_psi1_exp_Z)) + _dL_dlengthscale = lengthscale* np.einsum('nm,nmq,nmq->q',dL_dpsi1,_psi1_q,(_psi1_common[:,None,:]*(S[:,None,:]/lengthscale2+_psi1_dist_sq)*_psi1_exp_dist_sq + (1-gamma[:,None,:])*np.square(Z[None,:,:]/lengthscale2)*_psi1_exp_Z)) + + return _dL_dvariance, _dL_dlengthscale, _dL_dZ, _dL_dmu, _dL_dS, _dL_dgamma + +def _psi2compDer(dL_dpsi2, variance, lengthscale, Z, mu, S, gamma): + """ + Z - MxQ + mu - NxQ + S - NxQ + gamma - NxQ + dL_dpsi2 - MxM + """ + # here are the "statistics" for psi2 + # Produced the derivatives w.r.t. psi2: + # _dL_dvariance 1 + # _dL_dlengthscale Q + # _dL_dZ MxQ + # _dL_dgamma NxQ + # _dL_dmu NxQ + # _dL_dS NxQ - return _psi2 + lengthscale2 = np.square(lengthscale) - @Cache_this(limit=1, ignore_args=(0,1,2,3)) - def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior): - ARD = (len(lengthscale)!=1) - - dvar_psi1, dl_psi1, dZ_psi1, dmu_psi1, dS_psi1, dgamma_psi1 = self._psi1compDer(dL_dpsi1, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) - dvar_psi2, dl_psi2, dZ_psi2, dmu_psi2, dS_psi2, dgamma_psi2 = self._psi2compDer(dL_dpsi2, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) - - dL_dvar = np.sum(dL_dpsi0) + dvar_psi1 + dvar_psi2 - - dL_dlengscale = dl_psi1 + dl_psi2 - if not ARD: - dL_dlengscale = dL_dlengscale.sum() - - dL_dgamma = dgamma_psi1 + dgamma_psi2 - dL_dmu = dmu_psi1 + dmu_psi2 - dL_dS = dS_psi1 + dS_psi2 - dL_dZ = dZ_psi1 + dZ_psi2 - - return dL_dvar, dL_dlengscale, dL_dZ, dL_dmu, dL_dS, dL_dgamma - - def _psi1compDer(self, dL_dpsi1, variance, lengthscale, Z, mu, S, gamma): - """ - dL_dpsi1 - NxM - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - """ - # here are the "statistics" for psi1 - # Produced intermediate results: dL_dparams w.r.t. psi1 - # _dL_dvariance 1 - # _dL_dlengthscale Q - # _dL_dZ MxQ - # _dL_dgamma NxQ - # _dL_dmu NxQ - # _dL_dS NxQ - - lengthscale2 = np.square(lengthscale) - - # psi1 - _psi1_denom = S / lengthscale2 + 1. # NxQ - _psi1_denom_sqrt = np.sqrt(_psi1_denom) #NxQ - _psi1_dist = Z[None, :, :] - mu[:, None, :] # NxMxQ - _psi1_dist_sq = np.square(_psi1_dist) / (lengthscale2 * _psi1_denom[:,None,:]) # NxMxQ - _psi1_common = gamma / (lengthscale2*_psi1_denom*_psi1_denom_sqrt) #NxQ - _psi1_exponent1 = np.log(gamma[:,None,:]) -0.5 * (_psi1_dist_sq + np.log(_psi1_denom[:, None,:])) # NxMxQ - _psi1_exponent2 = np.log(1.-gamma[:,None,:]) -0.5 * (np.square(Z[None,:,:])/lengthscale2) # NxMxQ - _psi1_exponent_max = np.maximum(_psi1_exponent1,_psi1_exponent2) - _psi1_exponent = _psi1_exponent_max+np.log(np.exp(_psi1_exponent1-_psi1_exponent_max) + np.exp(_psi1_exponent2-_psi1_exponent_max)) #NxMxQ - _psi1_exp_sum = _psi1_exponent.sum(axis=-1) #NxM - _psi1_exp_dist_sq = np.exp(-0.5*_psi1_dist_sq) # NxMxQ - _psi1_exp_Z = np.exp(-0.5*np.square(Z[None,:,:])/lengthscale2) # 1xMxQ - _psi1_q = variance * np.exp(_psi1_exp_sum[:,:,None] - _psi1_exponent) # NxMxQ - _psi1 = variance * np.exp(_psi1_exp_sum) # NxM - _dL_dvariance = np.einsum('nm,nm->',dL_dpsi1, _psi1)/variance # 1 - _dL_dgamma = np.einsum('nm,nmq,nmq->nq',dL_dpsi1, _psi1_q, (_psi1_exp_dist_sq/_psi1_denom_sqrt[:,None,:]-_psi1_exp_Z)) # NxQ - _dL_dmu = np.einsum('nm, nmq, nmq, nmq, nq->nq',dL_dpsi1,_psi1_q,_psi1_exp_dist_sq,_psi1_dist,_psi1_common) # NxQ - _dL_dS = np.einsum('nm,nmq,nmq,nq,nmq->nq',dL_dpsi1,_psi1_q,_psi1_exp_dist_sq,_psi1_common,(_psi1_dist_sq-1.))/2. # NxQ - _dL_dZ = np.einsum('nm,nmq,nmq->mq',dL_dpsi1,_psi1_q, (- _psi1_common[:,None,:] * _psi1_dist * _psi1_exp_dist_sq - (1-gamma[:,None,:])/lengthscale2*Z[None,:,:]*_psi1_exp_Z)) - _dL_dlengthscale = lengthscale* np.einsum('nm,nmq,nmq->q',dL_dpsi1,_psi1_q,(_psi1_common[:,None,:]*(S[:,None,:]/lengthscale2+_psi1_dist_sq)*_psi1_exp_dist_sq + (1-gamma[:,None,:])*np.square(Z[None,:,:]/lengthscale2)*_psi1_exp_Z)) - - return _dL_dvariance, _dL_dlengthscale, _dL_dZ, _dL_dmu, _dL_dS, _dL_dgamma - - def _psi2compDer(self, dL_dpsi2, variance, lengthscale, Z, mu, S, gamma): - """ - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - dL_dpsi2 - MxM - """ - # here are the "statistics" for psi2 - # Produced the derivatives w.r.t. psi2: - # _dL_dvariance 1 - # _dL_dlengthscale Q - # _dL_dZ MxQ - # _dL_dgamma NxQ - # _dL_dmu NxQ - # _dL_dS NxQ - - lengthscale2 = np.square(lengthscale) - - _psi2_Zhat = 0.5 * (Z[:, None, :] + Z[None, :, :]) # M,M,Q - _psi2_Zdist = 0.5 * (Z[:, None, :] - Z[None, :, :]) # M,M,Q - _psi2_Zdist_sq = np.square(_psi2_Zdist / lengthscale) # M,M,Q - _psi2_Z_sq_sum = (np.square(Z[:,None,:])+np.square(Z[None,:,:]))/lengthscale2 # MxMxQ - - # psi2 - _psi2_denom = 2.*S / lengthscale2 + 1. # NxQ - _psi2_denom_sqrt = np.sqrt(_psi2_denom) - _psi2_mudist = mu[:,None,None,:]-_psi2_Zhat #N,M,M,Q - _psi2_mudist_sq = np.square(_psi2_mudist)/(lengthscale2*_psi2_denom[:,None,None,:]) - _psi2_common = gamma/(lengthscale2 * _psi2_denom * _psi2_denom_sqrt) # NxQ - _psi2_exponent1 = -_psi2_Zdist_sq -_psi2_mudist_sq -0.5*np.log(_psi2_denom[:,None,None,:])+np.log(gamma[:,None,None,:]) #N,M,M,Q - _psi2_exponent2 = np.log(1.-gamma[:,None,None,:]) - 0.5*(_psi2_Z_sq_sum) # NxMxMxQ - _psi2_exponent_max = np.maximum(_psi2_exponent1, _psi2_exponent2) - _psi2_exponent = _psi2_exponent_max+np.log(np.exp(_psi2_exponent1-_psi2_exponent_max) + np.exp(_psi2_exponent2-_psi2_exponent_max)) - _psi2_exp_sum = _psi2_exponent.sum(axis=-1) #NxM - _psi2_q = variance*variance * np.exp(_psi2_exp_sum[:,:,:,None]-_psi2_exponent) # NxMxMxQ - _psi2_exp_dist_sq = np.exp(-_psi2_Zdist_sq -_psi2_mudist_sq) # NxMxMxQ - _psi2_exp_Z = np.exp(-0.5*_psi2_Z_sq_sum) # MxMxQ - _psi2 = variance*variance * (np.exp(_psi2_exp_sum).sum(axis=0)) # MxM - _dL_dvariance = np.einsum('mo,mo->',dL_dpsi2,_psi2)*2./variance - _dL_dgamma = np.einsum('mo,nmoq,nmoq->nq',dL_dpsi2,_psi2_q,(_psi2_exp_dist_sq/_psi2_denom_sqrt[:,None,None,:] - _psi2_exp_Z)) - _dL_dmu = -2.*np.einsum('mo,nmoq,nq,nmoq,nmoq->nq',dL_dpsi2,_psi2_q,_psi2_common,_psi2_mudist,_psi2_exp_dist_sq) - _dL_dS = np.einsum('mo,nmoq,nq,nmoq,nmoq->nq',dL_dpsi2,_psi2_q, _psi2_common, (2.*_psi2_mudist_sq-1.), _psi2_exp_dist_sq) - _dL_dZ = 2.*np.einsum('mo,nmoq,nmoq->mq',dL_dpsi2,_psi2_q,(_psi2_common[:,None,None,:]*(-_psi2_Zdist*_psi2_denom[:,None,None,:]+_psi2_mudist)*_psi2_exp_dist_sq - (1-gamma[:,None,None,:])*Z[:,None,:]/lengthscale2*_psi2_exp_Z)) - _dL_dlengthscale = 2.*lengthscale* np.einsum('mo,nmoq,nmoq->q',dL_dpsi2,_psi2_q,(_psi2_common[:,None,None,:]*(S[:,None,None,:]/lengthscale2+_psi2_Zdist_sq*_psi2_denom[:,None,None,:]+_psi2_mudist_sq)*_psi2_exp_dist_sq+(1-gamma[:,None,None,:])*_psi2_Z_sq_sum*0.5/lengthscale2*_psi2_exp_Z)) - - return _dL_dvariance, _dL_dlengthscale, _dL_dZ, _dL_dmu, _dL_dS, _dL_dgamma + _psi2_Zhat = 0.5 * (Z[:, None, :] + Z[None, :, :]) # M,M,Q + _psi2_Zdist = 0.5 * (Z[:, None, :] - Z[None, :, :]) # M,M,Q + _psi2_Zdist_sq = np.square(_psi2_Zdist / lengthscale) # M,M,Q + _psi2_Z_sq_sum = (np.square(Z[:,None,:])+np.square(Z[None,:,:]))/lengthscale2 # MxMxQ + + # psi2 + _psi2_denom = 2.*S / lengthscale2 + 1. # NxQ + _psi2_denom_sqrt = np.sqrt(_psi2_denom) + _psi2_mudist = mu[:,None,None,:]-_psi2_Zhat #N,M,M,Q + _psi2_mudist_sq = np.square(_psi2_mudist)/(lengthscale2*_psi2_denom[:,None,None,:]) + _psi2_common = gamma/(lengthscale2 * _psi2_denom * _psi2_denom_sqrt) # NxQ + _psi2_exponent1 = -_psi2_Zdist_sq -_psi2_mudist_sq -0.5*np.log(_psi2_denom[:,None,None,:])+np.log(gamma[:,None,None,:]) #N,M,M,Q + _psi2_exponent2 = np.log(1.-gamma[:,None,None,:]) - 0.5*(_psi2_Z_sq_sum) # NxMxMxQ + _psi2_exponent_max = np.maximum(_psi2_exponent1, _psi2_exponent2) + _psi2_exponent = _psi2_exponent_max+np.log(np.exp(_psi2_exponent1-_psi2_exponent_max) + np.exp(_psi2_exponent2-_psi2_exponent_max)) + _psi2_exp_sum = _psi2_exponent.sum(axis=-1) #NxM + _psi2_q = variance*variance * np.exp(_psi2_exp_sum[:,:,:,None]-_psi2_exponent) # NxMxMxQ + _psi2_exp_dist_sq = np.exp(-_psi2_Zdist_sq -_psi2_mudist_sq) # NxMxMxQ + _psi2_exp_Z = np.exp(-0.5*_psi2_Z_sq_sum) # MxMxQ + _psi2 = variance*variance * (np.exp(_psi2_exp_sum).sum(axis=0)) # MxM + _dL_dvariance = np.einsum('mo,mo->',dL_dpsi2,_psi2)*2./variance + _dL_dgamma = np.einsum('mo,nmoq,nmoq->nq',dL_dpsi2,_psi2_q,(_psi2_exp_dist_sq/_psi2_denom_sqrt[:,None,None,:] - _psi2_exp_Z)) + _dL_dmu = -2.*np.einsum('mo,nmoq,nq,nmoq,nmoq->nq',dL_dpsi2,_psi2_q,_psi2_common,_psi2_mudist,_psi2_exp_dist_sq) + _dL_dS = np.einsum('mo,nmoq,nq,nmoq,nmoq->nq',dL_dpsi2,_psi2_q, _psi2_common, (2.*_psi2_mudist_sq-1.), _psi2_exp_dist_sq) + _dL_dZ = 2.*np.einsum('mo,nmoq,nmoq->mq',dL_dpsi2,_psi2_q,(_psi2_common[:,None,None,:]*(-_psi2_Zdist*_psi2_denom[:,None,None,:]+_psi2_mudist)*_psi2_exp_dist_sq - (1-gamma[:,None,None,:])*Z[:,None,:]/lengthscale2*_psi2_exp_Z)) + _dL_dlengthscale = 2.*lengthscale* np.einsum('mo,nmoq,nmoq->q',dL_dpsi2,_psi2_q,(_psi2_common[:,None,None,:]*(S[:,None,None,:]/lengthscale2+_psi2_Zdist_sq*_psi2_denom[:,None,None,:]+_psi2_mudist_sq)*_psi2_exp_dist_sq+(1-gamma[:,None,None,:])*_psi2_Z_sq_sum*0.5/lengthscale2*_psi2_exp_Z)) + + return _dL_dvariance, _dL_dlengthscale, _dL_dZ, _dL_dmu, _dL_dS, _dL_dgamma diff --git a/GPy/kern/_src/rbf.py b/GPy/kern/_src/rbf.py index 99692b0a..221ca593 100644 --- a/GPy/kern/_src/rbf.py +++ b/GPy/kern/_src/rbf.py @@ -8,7 +8,7 @@ from ...util.misc import param_to_array from stationary import Stationary from GPy.util.caching import Cache_this from ...core.parameterization import variational -from psi_comp import ssrbf_psi_comp,ssrbf_psi_gpucomp,rbf_psi_comp +from psi_comp import PSICOMP_RBF from ...util.config import * class RBF(Stationary): @@ -25,13 +25,7 @@ class RBF(Stationary): super(RBF, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name, useGPU=useGPU) self.weave_options = {} self.group_spike_prob = False - self.psicomp = rbf_psi_comp.PSICOMP_RBF() - - def set_for_SpikeAndSlab(self): - if self.useGPU: - self.psicomp = ssrbf_psi_gpucomp.PSICOMP_SSRBF() - else: - self.psicomp = ssrbf_psi_comp.PSICOMP_SSRBF() + self.psicomp = PSICOMP_RBF() def K_of_r(self, r): return self.variance * np.exp(-0.5 * r**2) @@ -44,289 +38,39 @@ class RBF(Stationary): #---------------------------------------# def psi0(self, Z, variational_posterior): - if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - if self.useGPU: - return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[0] - else: - return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[0] + if self.useGPU: + return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[0] else: -# return self.Kdiag(variational_posterior.mean) return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[0] def psi1(self, Z, variational_posterior): - if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - if self.useGPU: - return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[1] - else: - return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[1] + if self.useGPU: + return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[1] else: return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[1] def psi2(self, Z, variational_posterior): - if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - if self.useGPU: - return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[2] - else: - return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[2] + if self.useGPU: + return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[2] else: return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[2] def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): - # Spike-and-Slab GPLVM - if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - if self.useGPU: - self.psicomp.update_gradients_expectations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) - else: - dL_dvar, dL_dlengscale, _, _, _, _ = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) - self.variance.gradient = dL_dvar - self.lengthscale.gradient = dL_dlengscale - - elif isinstance(variational_posterior, variational.NormalPosterior): - dL_dvar, dL_dlengscale, _, _, _ = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) + if self.useGPU: + self.psicomp.update_gradients_expectations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) + else: + dL_dvar, dL_dlengscale = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[:2] self.variance.gradient = dL_dvar self.lengthscale.gradient = dL_dlengscale -# l2 = self.lengthscale**2 -# if l2.size != self.input_dim: -# l2 = l2*np.ones(self.input_dim) -# #contributions from psi0: -# self.variance.gradient = np.sum(dL_dpsi0) -# self.lengthscale.gradient = 0. -# -# # #from psi1 -# denom, _, dist_sq, psi1 = self._psi1computations(Z, variational_posterior) -# d_length = psi1[:,:,None] * ((dist_sq - 1.)/(self.lengthscale*denom) +1./self.lengthscale) -# dpsi1_dlength = d_length * dL_dpsi1[:, :, None] -# print dpsi1_dlength.sum(0).sum(0) -# if self.ARD: -# self.lengthscale.gradient += dpsi1_dlength.sum(0).sum(0) -# else: -# self.lengthscale.gradient += dpsi1_dlength.sum() -# self.variance.gradient += np.sum(dL_dpsi1 * psi1) / self.variance -# #from psi2 -# S = variational_posterior.variance -# _, Zdist_sq, _, mudist_sq, psi2 = self._psi2computations(Z, variational_posterior) -# if not self.ARD: -# self.lengthscale.gradient += self._weave_psi2_lengthscale_grads(dL_dpsi2, psi2, Zdist_sq, S, mudist_sq, l2).sum() -# else: -# self.lengthscale.gradient += self._weave_psi2_lengthscale_grads(dL_dpsi2, psi2, Zdist_sq, S, mudist_sq, l2) -# # print self._weave_psi2_lengthscale_grads(dL_dpsi2, psi2, Zdist_sq, S, mudist_sq, l2) -# -# self.variance.gradient += 2.*np.sum(dL_dpsi2 * psi2)/self.variance - - else: - raise ValueError, "unknown distriubtion received for psi-statistics" - def gradients_Z_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): - # Spike-and-Slab GPLVM - if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - if self.useGPU: - return self.psicomp.gradients_Z_expectations(dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) - else: - _, _, dL_dZ, _, _, _ = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) - return dL_dZ - - elif isinstance(variational_posterior, variational.NormalPosterior): - _, _, dL_dZ, _, _ = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) - return dL_dZ -# -# l2 = self.lengthscale **2 -# -# #psi1 -# denom, dist, dist_sq, psi1 = self._psi1computations(Z, variational_posterior) -# grad = np.einsum('ij,ij,ijk,ijk->jk', dL_dpsi1, psi1, dist, -1./(denom*l2)) -# -# #psi2 -# Zdist, Zdist_sq, mudist, mudist_sq, psi2 = self._psi2computations(Z, variational_posterior) -# term1 = Zdist / l2 # M, M, Q -# S = variational_posterior.variance -# term2 = mudist / (2.*S[:,None,None,:] + l2) # N, M, M, Q -# -# grad += 2.*np.einsum('ijk,ijk,ijkl->kl', dL_dpsi2, psi2, term1[None,:,:,:] + term2) -# -# return grad + if self.useGPU: + return self.psicomp.gradients_Z_expectations(dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) else: - raise ValueError, "unknown distriubtion received for psi-statistics" + return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[2] def gradients_qX_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): - # Spike-and-Slab GPLVM - if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - if self.useGPU: - return self.psicomp.gradients_qX_expectations(dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) - else: - _, _, _, dL_dmu, dL_dS, dL_dgamma = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) - return dL_dmu, dL_dS, dL_dgamma - - elif isinstance(variational_posterior, variational.NormalPosterior): - _, _, _, dL_dmu, dL_dS = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) - -# l2 = self.lengthscale **2 -# #psi1 -# denom, dist, dist_sq, psi1 = self._psi1computations(Z, variational_posterior) -# tmp = psi1[:, :, None] / l2 / denom -# grad_mu = np.sum(dL_dpsi1[:, :, None] * tmp * dist, 1) -# grad_S = np.sum(dL_dpsi1[:, :, None] * 0.5 * tmp * (dist_sq - 1), 1) -# #psi2 -# _, _, mudist, mudist_sq, psi2 = self._psi2computations(Z, variational_posterior) -# S = variational_posterior.variance -# tmp = psi2[:, :, :, None] / (2.*S[:,None,None,:] + l2) -# grad_mu += -2.*np.einsum('jk,ijkl,ijkl->il', dL_dpsi2, tmp , mudist) -# grad_S += np.einsum('jk,ijkl,ijkl->il', dL_dpsi2 , tmp , (2.*mudist_sq - 1)) - - return dL_dmu, dL_dS - + if self.useGPU: + return self.psicomp.gradients_qX_expectations(dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) else: - raise ValueError, "unknown distriubtion received for psi-statistics" - - #return grad_mu, grad_S - - #---------------------------------------# - # Precomputations # - #---------------------------------------# - - @Cache_this(limit=1) - def _psi1computations(self, Z, vp): - mu, S = vp.mean, vp.variance - l2 = self.lengthscale **2 - denom = S[:, None, :] / l2 + 1. # N,1,Q - dist = Z[None, :, :] - mu[:, None, :] # N,M,Q - dist_sq = np.square(dist) / l2 / denom # N,M,Q - exponent = -0.5 * np.sum(dist_sq + np.log(denom), -1)#N,M - psi1 = self.variance * np.exp(exponent) # N,M - return denom, dist, dist_sq, psi1 - - - @Cache_this(limit=1, ignore_args=(0,)) - def _Z_distances(self, Z): - Zhat = 0.5 * (Z[:, None, :] + Z[None, :, :]) # M,M,Q - Zdist = 0.5 * (Z[:, None, :] - Z[None, :, :]) # M,M,Q - return Zhat, Zdist - - @Cache_this(limit=1) - def _psi2computations(self, Z, vp): - - if config.getboolean('parallel', 'openmp'): - pragma_string = '#pragma omp parallel for private(tmp, exponent_tmp)' - header_string = '#include ' - libraries = ['gomp'] - else: - pragma_string = '' - header_string = '' - libraries = [] - - mu, S = vp.mean, vp.variance - - N, Q = mu.shape - M = Z.shape[0] - - #compute required distances - Zhat, Zdist = self._Z_distances(Z) - Zdist_sq = np.square(Zdist / self.lengthscale) # M,M,Q - - #allocate memory for the things we want to compute - mudist = np.empty((N, M, M, Q)) - mudist_sq = np.empty((N, M, M, Q)) - psi2 = np.empty((N, M, M)) - - l2 = self.lengthscale **2 - denom = (2.*S[:,None,None,:] / l2) + 1. # N,Q - half_log_denom = 0.5 * np.log(denom[:,0,0,:]) - denom_l2 = denom[:,0,0,:]*l2 - - variance_sq = float(np.square(self.variance)) - code = """ - double tmp, exponent_tmp; - %s - for (int n=0; n - """ % header_string - mu = param_to_array(mu) - weave.inline(code, support_code=support_code, libraries=libraries, - arg_names=['N', 'M', 'Q', 'mu', 'Zhat', 'mudist_sq', 'mudist', 'denom_l2', 'Zdist_sq', 'half_log_denom', 'psi2', 'variance_sq'], - type_converters=weave.converters.blitz, **self.weave_options) - - return Zdist, Zdist_sq, mudist, mudist_sq, psi2 - - def _weave_psi2_lengthscale_grads(self, dL_dpsi2, psi2, Zdist_sq, S, mudist_sq, l2): - - #here's the einsum equivalent, it's ~3 times slower - #return 2.*np.einsum( 'ijk,ijk,ijkl,il->l', dL_dpsi2, psi2, Zdist_sq * (2.*S[:,None,None,:]/l2 + 1.) + mudist_sq + S[:, None, None, :] / l2, 1./(2.*S + l2))*self.lengthscale - - result = np.zeros(self.input_dim) - if config.getboolean('parallel', 'openmp'): - pragma_string = '#pragma omp parallel for reduction(+:tmp)' - header_string = '#include ' - libraries = ['gomp'] - else: - pragma_string = '' - header_string = '' - libraries = [] - code = """ - double tmp; - for(int q=0; q - """ % header_string - N,Q = S.shape - M = psi2.shape[-1] - - S = param_to_array(S) - weave.inline(code, support_code=support_code, libraries=libraries, - arg_names=['psi2', 'dL_dpsi2', 'N', 'M', 'Q', 'mudist_sq', 'l2', 'Zdist_sq', 'S', 'result'], - type_converters=weave.converters.blitz, **self.weave_options) - - return 2.*result*self.lengthscale + return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[3:] diff --git a/GPy/models/ss_gplvm.py b/GPy/models/ss_gplvm.py index b0fe48b1..b10991ab 100644 --- a/GPy/models/ss_gplvm.py +++ b/GPy/models/ss_gplvm.py @@ -64,7 +64,6 @@ class SSGPLVM(SparseGP): if kernel is None: kernel = kern.RBF(input_dim, lengthscale=fracs, ARD=True) # + kern.white(input_dim) - kernel.set_for_SpikeAndSlab() if inference_method is None: inference_method = VarDTC_minibatch(mpi_comm=mpi_comm) From ac2d28e2fdf3098704d9346f567a62bd5e75cd51 Mon Sep 17 00:00:00 2001 From: James Hensman Date: Thu, 29 May 2014 09:55:49 +0100 Subject: [PATCH 15/55] reverting the fixing behaviour. two reasons: 1) the new behaviour is confusing for new users. Either something is fixed, or it's not. 2) the fixing didn't work! things that should have been fixed were passed to the optimizer for optimization. If we really want to save keystrokes, consider this: m.foo.fix() m.foo.unfix() m.foo.constrain_positive() is the same as m.foo.fix() m.foo.constrain_positive() but the latter throws a warning. --- GPy/core/parameterization/parameter_core.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/GPy/core/parameterization/parameter_core.py b/GPy/core/parameterization/parameter_core.py index 9cedc46d..dab5c2d0 100644 --- a/GPy/core/parameterization/parameter_core.py +++ b/GPy/core/parameterization/parameter_core.py @@ -324,7 +324,6 @@ class Indexable(Nameable, Observable): self._default_constraint_ = default_constraint from index_operations import ParameterIndexOperations self.constraints = ParameterIndexOperations() - self._old_constraints = ParameterIndexOperations() self.priors = ParameterIndexOperations() if self._default_constraint_ is not None: self.constrain(self._default_constraint_) @@ -388,8 +387,8 @@ class Indexable(Nameable, Observable): if value is not None: self[:] = value - index = self._raveled_index() - # reconstrained = self.unconstrain() + #index = self._raveled_index() + index = self.unconstrain() index = self._add_to_index_operations(self.constraints, index, __fixed__, warning) self._highest_parent_._set_fixed(self, index) self.notify_observers(self, None if trigger_parent else -np.inf) From 8eaa0bbf8a64c8bfd130d6bff135a36c21e09a7a Mon Sep 17 00:00:00 2001 From: James Hensman Date: Fri, 30 May 2014 16:45:51 +0100 Subject: [PATCH 16/55] basic vardtc working --- .../latent_function_inference/var_dtc.py | 49 +++++++------------ 1 file changed, 17 insertions(+), 32 deletions(-) diff --git a/GPy/inference/latent_function_inference/var_dtc.py b/GPy/inference/latent_function_inference/var_dtc.py index a9a137dc..9ade188f 100644 --- a/GPy/inference/latent_function_inference/var_dtc.py +++ b/GPy/inference/latent_function_inference/var_dtc.py @@ -36,11 +36,11 @@ class VarDTC(LatentFunctionInference): return param_to_array(np.sum(np.square(Y))) def __getstate__(self): - # has to be overridden, as Cacher objects cannot be pickled. + # has to be overridden, as Cacher objects cannot be pickled. return self.limit def __setstate__(self, state): - # has to be overridden, as Cacher objects cannot be pickled. + # has to be overridden, as Cacher objects cannot be pickled. self.limit = state from ...util.caching import Cacher self.get_trYYT = Cacher(self._get_trYYT, self.limit) @@ -62,20 +62,9 @@ class VarDTC(LatentFunctionInference): return Y * prec # TODO chache this, and make it effective def inference(self, kern, X, Z, likelihood, Y, Y_metadata=None): - if isinstance(X, VariationalPosterior): - uncertain_inputs = True - psi0 = kern.psi0(Z, X) - psi1 = kern.psi1(Z, X) - psi2 = kern.psi2(Z, X) - else: - uncertain_inputs = False - psi0 = kern.Kdiag(X) - psi1 = kern.K(X, Z) - psi2 = None - - #see whether we're using variational uncertain inputs _, output_dim = Y.shape + uncertain_inputs = isinstance(X, VariationalPosterior) #see whether we've got a different noise variance for each datum beta = 1./np.fmax(likelihood.gaussian_variance(Y_metadata), 1e-6) @@ -96,23 +85,21 @@ class VarDTC(LatentFunctionInference): diag.add(Kmm, self.const_jitter) Lm = jitchol(Kmm) - # The rather complex computations of A + + # The rather complex computations of A, and the psi stats if uncertain_inputs: + psi0 = kern.psi0(Z, X) + psi1 = kern.psi1(Z, X) if het_noise: - psi2_beta = psi2 * (beta.flatten().reshape(num_data, 1, 1)).sum(0) + psi2_beta = np.sum([kern.psi2(Z,X[i:i+1,:]) * beta_i for i,beta_i in enumerate(beta)],0) else: - psi2_beta = psi2.sum(0) * beta - #if 0: - # evals, evecs = linalg.eigh(psi2_beta) - # clipped_evals = np.clip(evals, 0., 1e6) # TODO: make clipping configurable - # if not np.array_equal(evals, clipped_evals): - # pass # print evals - # tmp = evecs * np.sqrt(clipped_evals) - # tmp = tmp.T - # no backsubstitution because of bound explosion on tr(A) if not... + psi2_beta = kern.psi2(Z,X) * beta LmInv = dtrtri(Lm) A = LmInv.dot(psi2_beta.dot(LmInv.T)) else: + psi0 = kern.Kdiag(X) + psi1 = kern.K(X, Z) + psi2 = None if het_noise: tmp = psi1 * (np.sqrt(beta.reshape(num_data, 1))) else: @@ -149,7 +136,7 @@ class VarDTC(LatentFunctionInference): log_marginal = _compute_log_marginal_likelihood(likelihood, num_data, output_dim, beta, het_noise, psi0, A, LB, trYYT, data_fit, VVT_factor) - #put the gradients in the right places + #noise derivatives dL_dR = _compute_dL_dR(likelihood, het_noise, uncertain_inputs, LB, _LBi_Lmi_psi1Vf, DBi_plus_BiPBi, Lm, A, @@ -158,6 +145,7 @@ class VarDTC(LatentFunctionInference): dL_dthetaL = likelihood.exact_inference_gradients(dL_dR,Y_metadata) + #put the gradients in the right places if uncertain_inputs: grad_dict = {'dL_dKmm': dL_dKmm, 'dL_dpsi0':dL_dpsi0, @@ -203,11 +191,11 @@ class VarDTCMissingData(LatentFunctionInference): self._Y.limit = limit def __getstate__(self): - # has to be overridden, as Cacher objects cannot be pickled. + # has to be overridden, as Cacher objects cannot be pickled. return self._Y.limit, self._inan def __setstate__(self, state): - # has to be overridden, as Cacher objects cannot be pickled. + # has to be overridden, as Cacher objects cannot be pickled. from ...util.caching import Cacher self.limit = state[0] self._inan = state[1] @@ -409,10 +397,7 @@ def _compute_dL_dpsi(num_inducing, num_data, output_dim, beta, Lm, VVT_factor, C dL_dpsi2 = None else: dL_dpsi2 = beta * dL_dpsi2_beta - if uncertain_inputs: - # repeat for each of the N psi_2 matrices - dL_dpsi2 = np.repeat(dL_dpsi2[None, :, :], num_data, axis=0) - else: + if not uncertain_inputs: # subsume back into psi1 (==Kmn) dL_dpsi1 += 2.*np.dot(psi1, dL_dpsi2) dL_dpsi2 = None From 6119c3307865c88f40cc1db9c831e7ccbd52d1e2 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Tue, 3 Jun 2014 11:34:32 +0100 Subject: [PATCH 17/55] Support non-symmetric dL_dKmm for stationary kernel --- GPy/kern/_src/stationary.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/GPy/kern/_src/stationary.py b/GPy/kern/_src/stationary.py index ef8900f9..055a945e 100644 --- a/GPy/kern/_src/stationary.py +++ b/GPy/kern/_src/stationary.py @@ -168,11 +168,14 @@ class Stationary(Kern): #the lower memory way with a loop tmp = invdist*dL_dr - if X2 is None: - tmp *= 2. - X2 = X ret = np.empty(X.shape, dtype=np.float64) - [np.einsum('ij,ij->i', tmp, X[:,q][:,None]-X2[:,q][None,:], out=ret[:,q]) for q in xrange(self.input_dim)] + if X2 is None: + [np.einsum('ij,ij->i', tmp, X[:,q][:,None]-X[:,q][None,:], out=ret[:,q]) for q in xrange(self.input_dim)] + ret2 = np.empty(X.shape, dtype=np.float64) + [np.einsum('ij,ji->j', tmp, X[:,q][:,None]-X[:,q][None,:], out=ret2[:,q]) for q in xrange(self.input_dim)] + ret += ret2 + else: + [np.einsum('ij,ij->i', tmp, X[:,q][:,None]-X2[:,q][None,:], out=ret[:,q]) for q in xrange(self.input_dim)] ret /= self.lengthscale**2 return ret From 695bad63c502bd2c213339e22cae9deb4ef16b48 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Mon, 9 Jun 2014 14:21:56 +0100 Subject: [PATCH 18/55] fix Linear kernel with SSGPLVM --- GPy/core/gp.py | 2 +- .../latent_function_inference/posterior.py | 11 ++- GPy/kern/_src/linear.py | 6 +- GPy/kern/_src/psi_comp/__init__.py | 35 ++++--- GPy/kern/_src/psi_comp/linear_psi_comp.py | 93 ------------------- GPy/util/linalg.py | 3 +- 6 files changed, 33 insertions(+), 117 deletions(-) delete mode 100644 GPy/kern/_src/psi_comp/linear_psi_comp.py diff --git a/GPy/core/gp.py b/GPy/core/gp.py index ca2583e0..f4b02128 100644 --- a/GPy/core/gp.py +++ b/GPy/core/gp.py @@ -40,7 +40,7 @@ class GP(Model): assert Y.ndim == 2 self.Y = ObsAr(Y) - assert Y.shape[0] == self.num_data +# assert Y.shape[0] == self.num_data _, self.output_dim = self.Y.shape #TODO: check the type of this is okay? diff --git a/GPy/inference/latent_function_inference/posterior.py b/GPy/inference/latent_function_inference/posterior.py index 70b0e0ea..66c68261 100644 --- a/GPy/inference/latent_function_inference/posterior.py +++ b/GPy/inference/latent_function_inference/posterior.py @@ -153,9 +153,14 @@ class Posterior(object): $$ """ if self._woodbury_inv is None: - self._woodbury_inv, _ = dpotri(self.woodbury_chol, lower=1) - #self._woodbury_inv, _ = dpotrs(self.woodbury_chol, np.eye(self.woodbury_chol.shape[0]), lower=1) - symmetrify(self._woodbury_inv) + if self._woodbury_chol is not None: + self._woodbury_inv, _ = dpotri(self._woodbury_chol, lower=1) + #self._woodbury_inv, _ = dpotrs(self.woodbury_chol, np.eye(self.woodbury_chol.shape[0]), lower=1) + symmetrify(self._woodbury_inv) + elif self._covariance is not None: + B = self._K - self._covariance + tmp, _ = dpotrs(self.K_chol, B) + self._woodbury_inv, _ = dpotrs(self.K_chol, tmp.T) return self._woodbury_inv @property diff --git a/GPy/kern/_src/linear.py b/GPy/kern/_src/linear.py index 649c1222..9a1388c5 100644 --- a/GPy/kern/_src/linear.py +++ b/GPy/kern/_src/linear.py @@ -11,8 +11,8 @@ from ...core.parameterization import Param from ...core.parameterization.transformations import Logexp from ...util.caching import Cache_this from ...core.parameterization import variational -from psi_comp import linear_psi_comp from ...util.config import * +from .psi_comp import PSICOMP_Linear class Linear(Kern): """ @@ -53,10 +53,8 @@ class Linear(Kern): self.variances = Param('variances', variances, Logexp()) self.add_parameter(self.variances) + self.psicomp = PSICOMP_Linear() - def set_for_SpikeAndSlab(self): - self.psicomp = linear_psi_comp.PSICOMP_SSLinear() - @Cache_this(limit=2) def K(self, X, X2=None): if self.ARD: diff --git a/GPy/kern/_src/psi_comp/__init__.py b/GPy/kern/_src/psi_comp/__init__.py index 77668e7f..38094394 100644 --- a/GPy/kern/_src/psi_comp/__init__.py +++ b/GPy/kern/_src/psi_comp/__init__.py @@ -6,20 +6,7 @@ from GPy.util.caching import Cache_this from ....core.parameterization import variational import rbf_psi_comp import ssrbf_psi_comp - -class PSICOMP(Pickleable): - - def psicomputations(self, variance, Z, variational_posterior): - """ - Compute psi-statistics - """ - pass - - def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior): - """ - Compute the derivatives of parameters by combing dL_dpsi and dpsi_dparam - """ - pass +import sslinear_psi_comp class PSICOMP_RBF(Pickleable): @@ -38,5 +25,25 @@ class PSICOMP_RBF(Pickleable): return rbf_psi_comp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior) elif isinstance(variational_posterior, variational.SpikeAndSlabPosterior): return ssrbf_psi_comp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior) + else: + raise ValueError, "unknown distriubtion received for psi-statistics" + +class PSICOMP_Linear(Pickleable): + + @Cache_this(limit=1, ignore_args=(0,)) + def psicomputations(self, variance, Z, variational_posterior): + if isinstance(variational_posterior, variational.NormalPosterior): + raise NotImplementedError + elif isinstance(variational_posterior, variational.SpikeAndSlabPosterior): + return sslinear_psi_comp.psicomputations(variance, Z, variational_posterior) + else: + raise ValueError, "unknown distriubtion received for psi-statistics" + + @Cache_this(limit=1, ignore_args=(0,1,2,3)) + def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior): + if isinstance(variational_posterior, variational.NormalPosterior): + raise NotImplementedError + elif isinstance(variational_posterior, variational.SpikeAndSlabPosterior): + return sslinear_psi_comp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior) else: raise ValueError, "unknown distriubtion received for psi-statistics" \ No newline at end of file diff --git a/GPy/kern/_src/psi_comp/linear_psi_comp.py b/GPy/kern/_src/psi_comp/linear_psi_comp.py deleted file mode 100644 index 3404c987..00000000 --- a/GPy/kern/_src/psi_comp/linear_psi_comp.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright (c) 2012, GPy authors (see AUTHORS.txt). -# Licensed under the BSD 3-clause license (see LICENSE.txt) - -""" -The package for the Psi statistics computation of the linear kernel for SSGPLVM -""" - -import numpy as np -from . import PSICOMP -from GPy.util.caching import Cache_this - -class PSICOMP_SSLinear(PSICOMP): - @Cache_this(limit=1, ignore_args=(0,)) - def psicomputations(self, variance, Z, variational_posterior): - """ - Compute psi-statistics for ss-linear kernel - """ - # here are the "statistics" for psi0, psi1 and psi2 - # Produced intermediate results: - # psi0 N - # psi1 NxM - # psi2 MxM - mu = variational_posterior.mean - S = variational_posterior.variance - gamma = variational_posterior.binary_prob - - psi0 = np.einsum('q,nq,nq->n',variance,gamma,np.square(mu)+S) - psi1 = np.einsum('nq,q,mq,nq->nm',gamma,variance,Z,mu) - mu2 = np.square(mu) - variances2 = np.square(variance) - tmp = np.einsum('nq,q,mq,nq->nm',gamma,variance,Z,mu) - psi2 = np.einsum('nq,q,mq,oq,nq->mo',gamma,variances2,Z,Z,mu2+S)+\ - np.einsum('nm,no->mo',tmp,tmp) - np.einsum('nq,q,mq,oq,nq->mo',np.square(gamma),variances2,Z,Z,mu2) - - return psi0, psi1, psi2 - - @Cache_this(limit=1, ignore_args=(0,1,2,3)) - def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior): - mu = variational_posterior.mean - S = variational_posterior.variance - gamma = variational_posterior.binary_prob - - dL_dvar, dL_dgamma, dL_dmu, dL_dS, dL_dZ = self._psi2computations(dL_dpsi2, variance, Z, mu, S, gamma) - - # Compute for psi0 and psi1 - mu2S = np.square(mu)+S - dL_dvar += np.einsum('n,nq,nq->q',dL_dpsi0,gamma,mu2S) + np.einsum('nm,nq,mq,nq->q',dL_dpsi1,gamma,Z,mu) - dL_dgamma += np.einsum('n,q,nq->nq',dL_dpsi0,variance,mu2S) + np.einsum('nm,q,mq,nq->nq',dL_dpsi1,variance,Z,mu) - dL_dmu += np.einsum('n,nq,q,nq->nq',dL_dpsi0,gamma,2.*variance,mu) + np.einsum('nm,nq,q,mq->nq',dL_dpsi1,gamma,variance,Z) - dL_dS += np.einsum('n,nq,q->nq',dL_dpsi0,gamma,variance) - dL_dZ += np.einsum('nm,nq,q,nq->mq',dL_dpsi1,gamma, variance,mu) - - return dL_dvar, dL_dZ, dL_dmu, dL_dS, dL_dgamma - - def _psi2computations(self, dL_dpsi2, variance, Z, mu, S, gamma): - """ - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - """ - # here are the "statistics" for psi1 and psi2 - # Produced intermediate results: - # _psi2_dvariance Q - # _psi2_dZ MxQ - # _psi2_dgamma NxQ - # _psi2_dmu NxQ - # _psi2_dS NxQ - - mu2 = np.square(mu) - gamma2 = np.square(gamma) - variance2 = np.square(variance) - mu2S = mu2+S # NxQ - common_sum = np.einsum('nq,q,mq,nq->nm',gamma,variance,Z,mu) # NxM - - dL_dvar = np.einsum('mo,nq,q,mq,oq->q',dL_dpsi2,2.*(gamma*mu2S-gamma2*mu2),variance,Z,Z)+\ - np.einsum('mo,nq,mq,nq,no->q',dL_dpsi2,gamma,Z,mu,common_sum)+\ - np.einsum('mo,nq,oq,nq,nm->q',dL_dpsi2,gamma,Z,mu,common_sum) - - dL_dgamma = np.einsum('mo,q,mq,oq,nq->nq',dL_dpsi2,variance2,Z,Z,(mu2S-2.*gamma*mu2))+\ - np.einsum('mo,q,mq,nq,no->nq',dL_dpsi2,variance,Z,mu,common_sum)+\ - np.einsum('mo,q,oq,nq,nm->nq',dL_dpsi2,variance,Z,mu,common_sum) - - dL_dmu = np.einsum('mo,q,mq,oq,nq,nq->nq',dL_dpsi2,variance2,Z,Z,mu,2.*(gamma-gamma2))+\ - np.einsum('mo,nq,q,mq,no->nq',dL_dpsi2,gamma,variance,Z,common_sum)+\ - np.einsum('mo,nq,q,oq,nm->nq',dL_dpsi2,gamma,variance,Z,common_sum) - - dL_dS = np.einsum('mo,nq,q,mq,oq->nq',dL_dpsi2,gamma,variance2,Z,Z) - - dL_dZ = 2.*(np.einsum('om,nq,q,mq,nq->oq',dL_dpsi2,gamma,variance2,Z,mu2S)+np.einsum('om,nq,q,nq,nm->oq',dL_dpsi2,gamma,variance,mu,common_sum) - -np.einsum('om,nq,q,mq,nq->oq',dL_dpsi2,gamma2,variance2,Z,mu2)) - - return dL_dvar, dL_dgamma, dL_dmu, dL_dS, dL_dZ diff --git a/GPy/util/linalg.py b/GPy/util/linalg.py index 661a2b47..8f79ae91 100644 --- a/GPy/util/linalg.py +++ b/GPy/util/linalg.py @@ -16,8 +16,7 @@ import warnings import os from config import * -if np.all(np.float64((scipy.__version__).split('.')[:2]) >= np.array([0, 12])): - #import scipy.linalg.lapack.clapack as lapack +if float('.'.join((scipy.__version__).split('.')[:2])) >= 0.12: from scipy.linalg import lapack else: from scipy.linalg.lapack import flapack as lapack From c2a1ea9cbd3718912504ac2c055c0feb958ff5ba Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Mon, 9 Jun 2014 14:24:20 +0100 Subject: [PATCH 19/55] rename sslinear_psi_comp.py --- GPy/kern/_src/psi_comp/sslinear_psi_comp.py | 88 +++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 GPy/kern/_src/psi_comp/sslinear_psi_comp.py diff --git a/GPy/kern/_src/psi_comp/sslinear_psi_comp.py b/GPy/kern/_src/psi_comp/sslinear_psi_comp.py new file mode 100644 index 00000000..ec81c40f --- /dev/null +++ b/GPy/kern/_src/psi_comp/sslinear_psi_comp.py @@ -0,0 +1,88 @@ +# Copyright (c) 2012, GPy authors (see AUTHORS.txt). +# Licensed under the BSD 3-clause license (see LICENSE.txt) + +""" +The package for the Psi statistics computation of the linear kernel for SSGPLVM +""" + +import numpy as np + +def psicomputations(variance, Z, variational_posterior): + """ + Compute psi-statistics for ss-linear kernel + """ + # here are the "statistics" for psi0, psi1 and psi2 + # Produced intermediate results: + # psi0 N + # psi1 NxM + # psi2 MxM + mu = variational_posterior.mean + S = variational_posterior.variance + gamma = variational_posterior.binary_prob + + psi0 = np.einsum('q,nq,nq->n',variance,gamma,np.square(mu)+S) + psi1 = np.einsum('nq,q,mq,nq->nm',gamma,variance,Z,mu) + mu2 = np.square(mu) + variances2 = np.square(variance) + tmp = np.einsum('nq,q,mq,nq->nm',gamma,variance,Z,mu) + psi2 = np.einsum('nq,q,mq,oq,nq->mo',gamma,variances2,Z,Z,mu2+S)+\ + np.einsum('nm,no->mo',tmp,tmp) - np.einsum('nq,q,mq,oq,nq->mo',np.square(gamma),variances2,Z,Z,mu2) + + return psi0, psi1, psi2 + +def psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior): + mu = variational_posterior.mean + S = variational_posterior.variance + gamma = variational_posterior.binary_prob + + dL_dvar, dL_dgamma, dL_dmu, dL_dS, dL_dZ = _psi2computations(dL_dpsi2, variance, Z, mu, S, gamma) + + # Compute for psi0 and psi1 + mu2S = np.square(mu)+S + dL_dvar += np.einsum('n,nq,nq->q',dL_dpsi0,gamma,mu2S) + np.einsum('nm,nq,mq,nq->q',dL_dpsi1,gamma,Z,mu) + dL_dgamma += np.einsum('n,q,nq->nq',dL_dpsi0,variance,mu2S) + np.einsum('nm,q,mq,nq->nq',dL_dpsi1,variance,Z,mu) + dL_dmu += np.einsum('n,nq,q,nq->nq',dL_dpsi0,gamma,2.*variance,mu) + np.einsum('nm,nq,q,mq->nq',dL_dpsi1,gamma,variance,Z) + dL_dS += np.einsum('n,nq,q->nq',dL_dpsi0,gamma,variance) + dL_dZ += np.einsum('nm,nq,q,nq->mq',dL_dpsi1,gamma, variance,mu) + + return dL_dvar, dL_dZ, dL_dmu, dL_dS, dL_dgamma + +def _psi2computations(dL_dpsi2, variance, Z, mu, S, gamma): + """ + Z - MxQ + mu - NxQ + S - NxQ + gamma - NxQ + """ + # here are the "statistics" for psi1 and psi2 + # Produced intermediate results: + # _psi2_dvariance Q + # _psi2_dZ MxQ + # _psi2_dgamma NxQ + # _psi2_dmu NxQ + # _psi2_dS NxQ + + mu2 = np.square(mu) + gamma2 = np.square(gamma) + variance2 = np.square(variance) + mu2S = mu2+S # NxQ + common_sum = np.einsum('nq,q,mq,nq->nm',gamma,variance,Z,mu) # NxM + + dL_dvar = np.einsum('mo,nq,q,mq,oq->q',dL_dpsi2,2.*(gamma*mu2S-gamma2*mu2),variance,Z,Z)+\ + np.einsum('mo,nq,mq,nq,no->q',dL_dpsi2,gamma,Z,mu,common_sum)+\ + np.einsum('mo,nq,oq,nq,nm->q',dL_dpsi2,gamma,Z,mu,common_sum) + + dL_dgamma = np.einsum('mo,q,mq,oq,nq->nq',dL_dpsi2,variance2,Z,Z,(mu2S-2.*gamma*mu2))+\ + np.einsum('mo,q,mq,nq,no->nq',dL_dpsi2,variance,Z,mu,common_sum)+\ + np.einsum('mo,q,oq,nq,nm->nq',dL_dpsi2,variance,Z,mu,common_sum) + + dL_dmu = np.einsum('mo,q,mq,oq,nq,nq->nq',dL_dpsi2,variance2,Z,Z,mu,2.*(gamma-gamma2))+\ + np.einsum('mo,nq,q,mq,no->nq',dL_dpsi2,gamma,variance,Z,common_sum)+\ + np.einsum('mo,nq,q,oq,nm->nq',dL_dpsi2,gamma,variance,Z,common_sum) + + dL_dS = np.einsum('mo,nq,q,mq,oq->nq',dL_dpsi2,gamma,variance2,Z,Z) + + dL_dZ = 2.*(np.einsum('om,nq,q,mq,nq->oq',dL_dpsi2,gamma,variance2,Z,mu2S)+np.einsum('om,nq,q,nq,nm->oq',dL_dpsi2,gamma,variance,mu,common_sum) + -np.einsum('om,nq,q,mq,nq->oq',dL_dpsi2,gamma2,variance2,Z,mu2)) + + return dL_dvar, dL_dgamma, dL_dmu, dL_dS, dL_dZ From 0b75aa8b0ff30b3b9945af1d9f6cf51adc4c1d5f Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Mon, 16 Jun 2014 10:14:41 +0100 Subject: [PATCH 20/55] add truncated linear kernel --- GPy/kern/__init__.py | 1 + GPy/kern/_src/linear.py | 2 +- GPy/kern/_src/trunclinear.py | 208 ++++++++++++++++++++++++++ GPy/plotting/matplot_dep/visualize.py | 13 +- 4 files changed, 219 insertions(+), 5 deletions(-) create mode 100644 GPy/kern/_src/trunclinear.py diff --git a/GPy/kern/__init__.py b/GPy/kern/__init__.py index 27e056b4..e28bd6ed 100644 --- a/GPy/kern/__init__.py +++ b/GPy/kern/__init__.py @@ -14,6 +14,7 @@ from _src.ODE_UYC import ODE_UYC from _src.ODE_st import ODE_st from _src.ODE_t import ODE_t from _src.poly import Poly +from _src.trunclinear import TruncLinear,TruncLinear_inf # TODO: put this in an init file somewhere #I'm commenting this out because the files were not added. JH. Remember to add the files before commiting diff --git a/GPy/kern/_src/linear.py b/GPy/kern/_src/linear.py index 9a1388c5..d64652f1 100644 --- a/GPy/kern/_src/linear.py +++ b/GPy/kern/_src/linear.py @@ -96,7 +96,7 @@ class Linear(Kern): def gradients_X(self, dL_dK, X, X2=None): if X2 is None: - return 2.*(((X[None,:, :] * self.variances)) * dL_dK[:, :, None]).sum(1) + return np.einsum('mq,nm->nq',X*self.variances,dL_dK)+np.einsum('nq,nm->mq',X*self.variances,dL_dK) else: return (((X2[None,:, :] * self.variances)) * dL_dK[:, :, None]).sum(1) diff --git a/GPy/kern/_src/trunclinear.py b/GPy/kern/_src/trunclinear.py new file mode 100644 index 00000000..540c5bc8 --- /dev/null +++ b/GPy/kern/_src/trunclinear.py @@ -0,0 +1,208 @@ +# Copyright (c) 2012, GPy authors (see AUTHORS.txt). +# Licensed under the BSD 3-clause license (see LICENSE.txt) + + +import numpy as np +from scipy import weave +from kern import Kern +from ...util.linalg import tdot +from ...util.misc import param_to_array +from ...core.parameterization import Param +from ...core.parameterization.transformations import Logexp +from ...util.caching import Cache_this +from ...core.parameterization import variational +from ...util.config import * + +class TruncLinear(Kern): + """ + Truncated Linear kernel + + .. math:: + + k(x,y) = \sum_{i=1}^input_dim \sigma^2_i \max(0, x_iy_i - \simga_q) + + :param input_dim: the number of input dimensions + :type input_dim: int + :param variances: the vector of variances :math:`\sigma^2_i` + :type variances: array or list of the appropriate size (or float if there + is only one variance parameter) + :param ARD: Auto Relevance Determination. If False, the kernel has only one + variance parameter \sigma^2, otherwise there is one variance + parameter per dimension. + :type ARD: Boolean + :rtype: kernel object + + """ + + def __init__(self, input_dim, variances=None, delta=None, ARD=False, active_dims=None, name='linear'): + super(TruncLinear, self).__init__(input_dim, active_dims, name) + self.ARD = ARD + if not ARD: + if variances is not None: + variances = np.asarray(variances) + delta = np.asarray(delta) + assert variances.size == 1, "Only one variance needed for non-ARD kernel" + else: + variances = np.ones(1) + delta = np.zeros(1) + else: + if variances is not None: + variances = np.asarray(variances) + delta = np.asarray(delta) + assert variances.size == self.input_dim, "bad number of variances, need one ARD variance per input_dim" + else: + variances = np.ones(self.input_dim) + delta = np.zeros(self.input_dim) + + self.variances = Param('variances', variances, Logexp()) + self.delta = Param('delta', delta) + self.add_parameter(self.variances) + self.add_parameter(self.delta) + + @Cache_this(limit=2) + def K(self, X, X2=None): + XX = self.variances*self._product(X, X2) + return XX.sum(axis=-1) + + @Cache_this(limit=2) + def _product(self, X, X2=None): + if X2 is None: + X2 = X + XX = np.einsum('nq,mq->nmq',X-self.delta,X2-self.delta) + XX[XX<0] = 0 + return XX + + def Kdiag(self, X): + return self.variances*(np.square(X-self.delta)).sum(axis=-1) + + def update_gradients_full(self, dL_dK, X, X2=None): + dK_dvar = self._product(X, X2) + if X2 is None: + X2=X + dK_ddelta = self.variances*(2*self.delta-X[:,None,:]-X2[None,:,:])*(dK_dvar>0) + if self.ARD: + self.variances.gradient[:] = np.einsum('nmq,nm->q',dK_dvar,dL_dK) + self.delta.gradient[:] = np.einsum('nmq,nm->q',dK_ddelta,dL_dK) + else: + self.variances.gradient[:] = np.einsum('nmq,nm->',dK_dvar,dL_dK) + self.delta.gradient[:] = np.einsum('nmq,nm->',dK_ddelta,dL_dK) + + def update_gradients_diag(self, dL_dKdiag, X): + if self.ARD: + self.variances.gradient[:] = np.einsum('nq,n->q',np.square(X-self.delta),dL_dKdiag) + self.delta.gradient[:] = np.einsum('nq,n->q',2*self.variances*(self.delta-X),dL_dKdiag) + else: + self.variances.gradient[:] = np.einsum('nq,n->',np.square(X-self.delta),dL_dKdiag) + self.delta.gradient[:] = np.einsum('nq,n->',2*self.variances*(self.delta-X),dL_dKdiag) + + def gradients_X(self, dL_dK, X, X2=None): + XX = self._product(X, X2) + if X2 is None: + Xp = (self.variances*(X-self.delta))*(XX>0) + else: + Xp = (self.variances*(X2-self.delta))*(XX>0) + if X2 is None: + return np.einsum('nmq,nm->nq',Xp,dL_dK)+np.einsum('mnq,nm->mq',Xp,dL_dK) + else: + return np.einsum('nmq,nm->nq',Xp,dL_dK) + + def gradients_X_diag(self, dL_dKdiag, X): + return 2.*self.variances*dL_dKdiag[:,None]*(X-self.delta) + + def input_sensitivity(self): + return np.ones(self.input_dim) * self.variances + +class TruncLinear_inf(Kern): + """ + Truncated Linear kernel + + .. math:: + + k(x,y) = \sum_{i=1}^input_dim \sigma^2_i \max(0, x_iy_i - \simga_q) + + :param input_dim: the number of input dimensions + :type input_dim: int + :param variances: the vector of variances :math:`\sigma^2_i` + :type variances: array or list of the appropriate size (or float if there + is only one variance parameter) + :param ARD: Auto Relevance Determination. If False, the kernel has only one + variance parameter \sigma^2, otherwise there is one variance + parameter per dimension. + :type ARD: Boolean + :rtype: kernel object + + """ + + def __init__(self, input_dim, interval, variances=None, ARD=False, active_dims=None, name='linear'): + super(TruncLinear_inf, self).__init__(input_dim, active_dims, name) + self.interval = interval + self.ARD = ARD + if not ARD: + if variances is not None: + variances = np.asarray(variances) + assert variances.size == 1, "Only one variance needed for non-ARD kernel" + else: + variances = np.ones(1) + else: + if variances is not None: + variances = np.asarray(variances) + assert variances.size == self.input_dim, "bad number of variances, need one ARD variance per input_dim" + else: + variances = np.ones(self.input_dim) + + self.variances = Param('variances', variances, Logexp()) + self.add_parameter(self.variances) + + +# @Cache_this(limit=2) + def K(self, X, X2=None): + tmp = self._product(X, X2) + return (self.variances*tmp).sum(axis=-1) + +# @Cache_this(limit=2) + def _product(self, X, X2=None): + if X2 is None: + X2 = X + X_X2 = X[:,None,:]-X2[None,:,:] + tmp = np.abs(X_X2**3)/6+np.einsum('nq,mq->nmq',X,X2)*(self.interval[1]-self.interval[0]) \ + -(X[:,None,:]+X2[None,:,:])*(self.interval[1]*self.interval[1]-self.interval[0]*self.interval[0])/2+(self.interval[1]**3-self.interval[0]**3)/3. + return tmp + + def Kdiag(self, X): + tmp = np.square(X)*(self.interval[1]-self.interval[0]) \ + -X*(self.interval[1]*self.interval[1]-self.interval[0]*self.interval[0])+(self.interval[1]**3-self.interval[0]**3)/3 + return (self.variances*tmp).sum(axis=-1) + + def update_gradients_full(self, dL_dK, X, X2=None): + dK_dvar = self._product(X, X2) + if self.ARD: + self.variances.gradient[:] = np.einsum('nmq,nm->q',dK_dvar,dL_dK) + else: + self.variances.gradient[:] = np.einsum('nmq,nm->',dK_dvar,dL_dK) + + def update_gradients_diag(self, dL_dKdiag, X): + tmp = np.square(X)*(self.interval[1]-self.interval[0]) \ + -X*(self.interval[1]*self.interval[1]-self.interval[0]*self.interval[0])+(self.interval[1]**3-self.interval[0]**3)/3 + if self.ARD: + self.variances.gradient[:] = np.einsum('nq,n->q',tmp,dL_dKdiag) + else: + self.variances.gradient[:] = np.einsum('nq,n->',tmp,dL_dKdiag) + + def gradients_X(self, dL_dK, X, X2=None): + XX = self._product(X, X2) + if X2 is None: + Xp = (self.variances*(X-self.delta))*(XX>0) + else: + Xp = (self.variances*(X2-self.delta))*(XX>0) + if X2 is None: + return np.einsum('nmq,nm->nq',Xp,dL_dK)+np.einsum('mnq,nm->mq',Xp,dL_dK) + else: + return np.einsum('nmq,nm->nq',Xp,dL_dK) + + def gradients_X_diag(self, dL_dKdiag, X): + return 2.*self.variances*dL_dKdiag[:,None]*(X-self.delta) + + def input_sensitivity(self): + return np.ones(self.input_dim) * self.variances + + diff --git a/GPy/plotting/matplot_dep/visualize.py b/GPy/plotting/matplot_dep/visualize.py index 557729da..12fee039 100644 --- a/GPy/plotting/matplot_dep/visualize.py +++ b/GPy/plotting/matplot_dep/visualize.py @@ -280,8 +280,11 @@ class image_show(matplotlib_show): :param preset_mean: the preset mean of a scaled image. :type preset_mean: double :param preset_std: the preset standard deviation of a scaled image. - :type preset_std: double""" - def __init__(self, vals, axes=None, dimensions=(16,16), transpose=False, order='C', invert=False, scale=False, palette=[], preset_mean=0., preset_std=1., select_image=0): + :type preset_std: double + :param cmap: the colormap for image visualization + :type cmap: matplotlib.cm""" + + def __init__(self, vals, axes=None, dimensions=(16,16), transpose=False, order='C', invert=False, scale=False, palette=[], preset_mean=0., preset_std=1., select_image=0, cmap=None): matplotlib_show.__init__(self, vals, axes) self.dimensions = dimensions self.transpose = transpose @@ -296,8 +299,10 @@ class image_show(matplotlib_show): self.set_image(self.vals) if not self.palette == []: # Can just show the image (self.set_image() took care of setting the palette) self.handle = self.axes.imshow(self.vals, interpolation='nearest') - else: # Use a boring gray map. - self.handle = self.axes.imshow(self.vals, cmap=plt.cm.gray, interpolation='nearest') # @UndefinedVariable + elif cmap==None: # Use a jet map. + self.handle = self.axes.imshow(self.vals, cmap=plt.cm.jet, interpolation='nearest') # @UndefinedVariable + else: # Use the selected map. + self.handle = self.axes.imshow(self.vals, cmap=cmap, interpolation='nearest') # @UndefinedVariable plt.show() def modify(self, vals): From db9b9bc857568a9cf6a200aa24f43a66eb1e573c Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Fri, 20 Jun 2014 11:56:21 +0100 Subject: [PATCH 21/55] rbf kernel gpu implementation in progress --- GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py | 454 ++++++++++++++++++++ GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py | 2 +- GPy/kern/_src/rbf.py | 17 +- GPy/kern/_src/trunclinear.py | 2 +- 4 files changed, 467 insertions(+), 8 deletions(-) create mode 100644 GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py diff --git a/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py new file mode 100644 index 00000000..7b604227 --- /dev/null +++ b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py @@ -0,0 +1,454 @@ +""" +The module for psi-statistics for RBF kernel +""" + +import numpy as np +from GPy.util.caching import Cacher +from . import PSICOMP_RBF +from ....util import gpu_init + +try: + import pycuda.gpuarray as gpuarray + from pycuda.compiler import SourceModule +except: + pass + +gpu_code = """ + // define THREADNUM + + #define IDX_NMQ(n,m,q) ((q*M+m)*N+n) + #define IDX_NQ(n,q) (q*N+n) + #define IDX_NM(n,m) (m*N+n) + #define IDX_MQ(m,q) (q*M+m) + #define IDX_MM(m1,m2) (m2*M+m1) + #define IDX_NQB(n,q,b) ((b*Q+q)*N+n) + #define IDX_QB(q,b) (b*Q+q) + + // Divide data evenly + __device__ void divide_data(int total_data, int psize, int pidx, int *start, int *end) { + int residue = (total_data)%psize; + if(pidx= blockDim.x) { + for(int i=blockDim.x+threadIdx.x; i=1;s=s/2) { + if(threadIdx.x < s) {array[threadIdx.x] += array[s+threadIdx.x];} + __syncthreads(); + } + } + + __global__ void psi1computations(double *psi1, double var, double *l, double *Z, double *mu, double *S, int N, int M, int Q) + { + int m_start, m_end; + divide_data(M, gridDim.x, blockIdx.x, &m_start, &m_end); + + for(int m=m_start; mnm',np.square(mu[:,None,:]-Z[None,:,:]),1./(S+lengthscale2)))/(-2.) + _psi1 = variance*np.exp(_psi1_log) + + return _psi1 + +def __psi2computations(variance, lengthscale, Z, mu, S): + """ + Z - MxQ + mu - NxQ + S - NxQ + gamma - NxQ + """ + # here are the "statistics" for psi2 + # Produced intermediate results: + # _psi2 MxM + + lengthscale2 = np.square(lengthscale) + + _psi2_logdenom = np.log(2.*S/lengthscale2+1.).sum(axis=-1)/(-2.) # N + _psi2_exp1 = (np.square(Z[:,None,:]-Z[None,:,:])/lengthscale2).sum(axis=-1)/(-4.) #MxM + Z_hat = (Z[:,None,:]+Z[None,:,:])/2. #MxMxQ + denom = 1./(2.*S+lengthscale2) + _psi2_exp2 = -(np.square(mu)*denom).sum(axis=-1)[:,None,None]+2.*np.einsum('nq,moq,nq->nmo',mu,Z_hat,denom)-np.einsum('moq,nq->nmo',np.square(Z_hat),denom) + _psi2 = variance*variance*np.exp(_psi2_logdenom[:,None,None]+_psi2_exp1[None,:,:]+_psi2_exp2) + + + return _psi2 + +def psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior): + ARD = (len(lengthscale)!=1) + + dvar_psi1, dl_psi1, dZ_psi1, dmu_psi1, dS_psi1 = _psi1compDer(dL_dpsi1, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance) + dvar_psi2, dl_psi2, dZ_psi2, dmu_psi2, dS_psi2 = _psi2compDer(dL_dpsi2, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance) + + dL_dvar = np.sum(dL_dpsi0) + dvar_psi1 + dvar_psi2 + + dL_dlengscale = dl_psi1 + dl_psi2 + if not ARD: + dL_dlengscale = dL_dlengscale.sum() + + dL_dmu = dmu_psi1 + dmu_psi2 + dL_dS = dS_psi1 + dS_psi2 + dL_dZ = dZ_psi1 + dZ_psi2 + + return dL_dvar, dL_dlengscale, dL_dZ, dL_dmu, dL_dS + +def _psi1compDer(dL_dpsi1, variance, lengthscale, Z, mu, S): + """ + dL_dpsi1 - NxM + Z - MxQ + mu - NxQ + S - NxQ + gamma - NxQ + """ + # here are the "statistics" for psi1 + # Produced intermediate results: dL_dparams w.r.t. psi1 + # _dL_dvariance 1 + # _dL_dlengthscale Q + # _dL_dZ MxQ + # _dL_dgamma NxQ + # _dL_dmu NxQ + # _dL_dS NxQ + + lengthscale2 = np.square(lengthscale) + + _psi1 = _psi1computations(variance, lengthscale, Z, mu, S) + Lpsi1 = dL_dpsi1*_psi1 + Zmu = Z[None,:,:]-mu[:,None,:] # NxMxQ + denom = 1./(S+lengthscale2) + Zmu2_denom = np.square(Zmu)*denom[:,None,:] #NxMxQ + _dL_dvar = Lpsi1.sum()/variance + _dL_dmu = np.einsum('nm,nmq,nq->nq',Lpsi1,Zmu,denom) + _dL_dS = np.einsum('nm,nmq,nq->nq',Lpsi1,(Zmu2_denom-1.),denom)/2. + _dL_dZ = -np.einsum('nm,nmq,nq->mq',Lpsi1,Zmu,denom) + _dL_dl = np.einsum('nm,nmq,nq->q',Lpsi1,(Zmu2_denom+(S/lengthscale2)[:,None,:]),denom*lengthscale) + + return _dL_dvar, _dL_dl, _dL_dZ, _dL_dmu, _dL_dS + +def _psi2compDer(dL_dpsi2, variance, lengthscale, Z, mu, S): + """ + Z - MxQ + mu - NxQ + S - NxQ + gamma - NxQ + dL_dpsi2 - MxM + """ + # here are the "statistics" for psi2 + # Produced the derivatives w.r.t. psi2: + # _dL_dvariance 1 + # _dL_dlengthscale Q + # _dL_dZ MxQ + # _dL_dgamma NxQ + # _dL_dmu NxQ + # _dL_dS NxQ + + lengthscale2 = np.square(lengthscale) + denom = 1./(2*S+lengthscale2) + denom2 = np.square(denom) + + _psi2 = _psi2computations(variance, lengthscale, Z, mu, S) # NxMxM + + Lpsi2 = dL_dpsi2[None,:,:]*_psi2 + Lpsi2sum = np.einsum('nmo->n',Lpsi2) #N + Lpsi2Z = np.einsum('nmo,oq->nq',Lpsi2,Z) #NxQ + Lpsi2Z2 = np.einsum('nmo,oq,oq->nq',Lpsi2,Z,Z) #NxQ + Lpsi2Z2p = np.einsum('nmo,mq,oq->nq',Lpsi2,Z,Z) #NxQ + Lpsi2Zhat = Lpsi2Z + Lpsi2Zhat2 = (Lpsi2Z2+Lpsi2Z2p)/2 + + _dL_dvar = Lpsi2sum.sum()*2/variance + _dL_dmu = (-2*denom) * (mu*Lpsi2sum[:,None]-Lpsi2Zhat) + _dL_dS = (2*np.square(denom))*(np.square(mu)*Lpsi2sum[:,None]-2*mu*Lpsi2Zhat+Lpsi2Zhat2) - denom*Lpsi2sum[:,None] + _dL_dZ = -np.einsum('nmo,oq->oq',Lpsi2,Z)/lengthscale2+np.einsum('nmo,oq->mq',Lpsi2,Z)/lengthscale2+ \ + 2*np.einsum('nmo,nq,nq->mq',Lpsi2,mu,denom) - np.einsum('nmo,nq,mq->mq',Lpsi2,denom,Z) - np.einsum('nmo,oq,nq->mq',Lpsi2,Z,denom) + _dL_dl = 2*lengthscale* ((S/lengthscale2*denom+np.square(mu*denom))*Lpsi2sum[:,None]+(Lpsi2Z2-Lpsi2Z2p)/(2*np.square(lengthscale2))- + (2*mu*denom2)*Lpsi2Zhat+denom2*Lpsi2Zhat2).sum(axis=0) + + return _dL_dvar, _dL_dl, _dL_dZ, _dL_dmu, _dL_dS + +_psi1computations = Cacher(__psi1computations, limit=1) +_psi2computations = Cacher(__psi2computations, limit=1) diff --git a/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py b/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py index e220e0d0..8f845d32 100644 --- a/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py +++ b/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py @@ -263,7 +263,7 @@ class PSICOMP_SSRBF(object): self.cublas_handle = gpu_init.cublas_handle self.gpuCache = None self.gpuCacheAll = None - + def _initGPUCache(self, N, M, Q): if self.gpuCache!=None and self.gpuCache['mu_gpu'].shape[0] == N: return diff --git a/GPy/kern/_src/rbf.py b/GPy/kern/_src/rbf.py index 221ca593..409c2e41 100644 --- a/GPy/kern/_src/rbf.py +++ b/GPy/kern/_src/rbf.py @@ -9,6 +9,7 @@ from stationary import Stationary from GPy.util.caching import Cache_this from ...core.parameterization import variational from psi_comp import PSICOMP_RBF +from psi_comp.rbf_psi_gpucomp import PSICOMP_RBF_GPU from ...util.config import * class RBF(Stationary): @@ -26,6 +27,8 @@ class RBF(Stationary): self.weave_options = {} self.group_spike_prob = False self.psicomp = PSICOMP_RBF() + if self.useGPU: + self.psicomp = PSICOMP_RBF_GPU() def K_of_r(self, r): return self.variance * np.exp(-0.5 * r**2) @@ -39,25 +42,27 @@ class RBF(Stationary): def psi0(self, Z, variational_posterior): if self.useGPU: - return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[0] + return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[0] else: return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[0] def psi1(self, Z, variational_posterior): if self.useGPU: - return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[1] + return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[1] else: return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[1] def psi2(self, Z, variational_posterior): if self.useGPU: - return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[2] + return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[2] else: return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[2] def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): if self.useGPU: - self.psicomp.update_gradients_expectations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) + dL_dvar, dL_dlengscale = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[:2] + self.variance.gradient = dL_dvar + self.lengthscale.gradient = dL_dlengscale else: dL_dvar, dL_dlengscale = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[:2] self.variance.gradient = dL_dvar @@ -65,12 +70,12 @@ class RBF(Stationary): def gradients_Z_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): if self.useGPU: - return self.psicomp.gradients_Z_expectations(dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) + return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[2] else: return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[2] def gradients_qX_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): if self.useGPU: - return self.psicomp.gradients_qX_expectations(dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior) + return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[3:] else: return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[3:] diff --git a/GPy/kern/_src/trunclinear.py b/GPy/kern/_src/trunclinear.py index 540c5bc8..76ed31f7 100644 --- a/GPy/kern/_src/trunclinear.py +++ b/GPy/kern/_src/trunclinear.py @@ -73,7 +73,7 @@ class TruncLinear(Kern): return XX def Kdiag(self, X): - return self.variances*(np.square(X-self.delta)).sum(axis=-1) + return (self.variances*np.square(X-self.delta)).sum(axis=-1) def update_gradients_full(self, dL_dK, X, X2=None): dK_dvar = self._product(X, X2) From e0412ebf5471b018a1ac091abf0376a5ca92e573 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Fri, 20 Jun 2014 14:09:26 +0100 Subject: [PATCH 22/55] rbf gpu psicomp pass gradcheck --- GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py | 166 +++++++++++++++++----- 1 file changed, 134 insertions(+), 32 deletions(-) diff --git a/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py index 7b604227..00e0d397 100644 --- a/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py +++ b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py @@ -17,6 +17,7 @@ gpu_code = """ // define THREADNUM #define IDX_NMQ(n,m,q) ((q*M+m)*N+n) + #define IDX_NMM(n,m1,m2) ((m2*M+m1)*N+n) #define IDX_NQ(n,q) (q*N+n) #define IDX_NM(n,m) (m*N+n) #define IDX_MQ(m,q) (q*M+m) @@ -66,15 +67,17 @@ gpu_code = """ double log_psi1 = 0; for(int q=0;q Date: Fri, 20 Jun 2014 15:02:00 +0100 Subject: [PATCH 23/55] rbf kernel gpu implementation ready --- GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py | 263 +++------------------- GPy/kern/_src/rbf.py | 39 +--- 2 files changed, 48 insertions(+), 254 deletions(-) diff --git a/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py index 00e0d397..d0d5bec5 100644 --- a/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py +++ b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py @@ -3,9 +3,10 @@ The module for psi-statistics for RBF kernel """ import numpy as np -from GPy.util.caching import Cacher +from ....util.caching import Cache_this from . import PSICOMP_RBF from ....util import gpu_init +from ....util.linalg_gpu import sum_axis try: import pycuda.gpuarray as gpuarray @@ -251,19 +252,25 @@ class PSICOMP_RBF_GPU(PSICOMP_RBF): 'psi1_gpu' :gpuarray.empty((N,M),np.float64,order='F'), 'psi2_gpu' :gpuarray.empty((M,M),np.float64,order='F'), 'psi2n_gpu' :gpuarray.empty((N,M,M),np.float64,order='F'), + 'dL_dpsi1_gpu' :gpuarray.empty((N,M),np.float64,order='F'), + 'dL_dpsi2_gpu' :gpuarray.empty((M,M),np.float64,order='F'), # derivatives 'dvar_gpu' :gpuarray.empty((self.blocknum,),np.float64, order='F'), 'dl_gpu' :gpuarray.empty((Q,self.blocknum),np.float64, order='F'), 'dZ_gpu' :gpuarray.empty((M,Q),np.float64, order='F'), 'dmu_gpu' :gpuarray.empty((N,Q,self.blocknum),np.float64, order='F'), 'dS_gpu' :gpuarray.empty((N,Q,self.blocknum),np.float64, order='F'), - # gradients - 'grad_l_gpu' :gpuarray.empty((Q,),np.float64,order='F'), - 'grad_Z_gpu' :gpuarray.empty((M,Q),np.float64,order='F'), + # grad + 'grad_l_gpu' :gpuarray.empty((Q,),np.float64, order='F'), + 'grad_mu_gpu' :gpuarray.empty((N,Q,),np.float64, order='F'), + 'grad_S_gpu' :gpuarray.empty((N,Q,),np.float64, order='F'), } def sync_params(self, lengthscale, Z, mu, S): - self.gpuCache['l_gpu'].set(np.asfortranarray(lengthscale)) + if len(lengthscale)==1: + self.gpuCache['l_gpu'].fill(lengthscale) + else: + self.gpuCache['l_gpu'].set(np.asfortranarray(lengthscale)) self.gpuCache['Z_gpu'].set(np.asfortranarray(Z)) self.gpuCache['mu_gpu'].set(np.asfortranarray(mu)) self.gpuCache['S_gpu'].set(np.asfortranarray(S)) @@ -274,23 +281,21 @@ class PSICOMP_RBF_GPU(PSICOMP_RBF): self.gpuCache['dZ_gpu'].fill(0.) self.gpuCache['dmu_gpu'].fill(0.) self.gpuCache['dS_gpu'].fill(0.) + self.gpuCache['grad_l_gpu'].fill(0.) + self.gpuCache['grad_mu_gpu'].fill(0.) + self.gpuCache['grad_S_gpu'].fill(0.) + + def get_dimensions(self, Z, variational_posterior): + return variational_posterior.mean.shape[0], Z.shape[0], Z.shape[1] -# @Cache_this(limit=1, ignore_args=(0,)) + @Cache_this(limit=1, ignore_args=(0,)) def psicomputations(self, variance, lengthscale, Z, variational_posterior): """ Z - MxQ mu - NxQ S - NxQ - gamma - NxQ """ - # here are the "statistics" for psi0, psi1 and psi2 - # Produced intermediate results: - # _psi1 NxM - mu = variational_posterior.mean - S = variational_posterior.variance - N = mu.shape[0] - M = Z.shape[0] - Q = Z.shape[1] + N,M,Q = self.get_dimensions(Z, variational_posterior) self._initGPUCache(N,M,Q) self.sync_params(lengthscale, Z, variational_posterior.mean, variational_posterior.variance) @@ -312,33 +317,11 @@ class PSICOMP_RBF_GPU(PSICOMP_RBF): else: return psi0_gpu.get(), psi1_gpu.get(), psi2_gpu.get() - psi0 = np.empty(mu.shape[0]) - psi0[:] = variance - - psi1 = _psi1computations(variance, lengthscale, Z, mu, S) - self.g_psi1computations(psi1_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1)) - psi1g = psi1_gpu.get() - print np.abs(psi1-psi1g).max() - - psi2 = _psi2computations(variance, lengthscale, Z, mu, S).sum(axis=0) - self.g_psi2computations(psi2_gpu, psi2n_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1)) - psi2g = psi2_gpu.get() - print np.abs(psi2-psi2g).max() - - return psi0, psi1, psi2 - -# @Cache_this(limit=1, ignore_args=(0,1,2,3)) + @Cache_this(limit=1, ignore_args=(0,1,2,3)) def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior): ARD = (len(lengthscale)!=1) - dvar_psi1, dl_psi1, dZ_psi1, dmu_psi1, dS_psi1 = _psi1compDer(dL_dpsi1, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance) - dvar_psi2, dl_psi2, dZ_psi2, dmu_psi2, dS_psi2 = _psi2compDer(dL_dpsi2, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance) - - mu = variational_posterior.mean - S = variational_posterior.variance - N = mu.shape[0] - M = Z.shape[0] - Q = Z.shape[1] + N,M,Q = self.get_dimensions(Z, variational_posterior) psi1_gpu = self.gpuCache['psi1_gpu'] psi2n_gpu = self.gpuCache['psi2n_gpu'] l_gpu = self.gpuCache['l_gpu'] @@ -350,207 +333,35 @@ class PSICOMP_RBF_GPU(PSICOMP_RBF): dZ_gpu = self.gpuCache['dZ_gpu'] dmu_gpu = self.gpuCache['dmu_gpu'] dS_gpu = self.gpuCache['dS_gpu'] + grad_l_gpu = self.gpuCache['grad_l_gpu'] + grad_mu_gpu = self.gpuCache['grad_mu_gpu'] + grad_S_gpu = self.gpuCache['grad_S_gpu'] if self.GPU_direct: dL_dpsi1_gpu = dL_dpsi1 dL_dpsi2_gpu = dL_dpsi2 else: - dL_dpsi1_gpu = gpuarray.to_gpu(np.asfortranarray(dL_dpsi1)) - dL_dpsi2_gpu = gpuarray.to_gpu(np.asfortranarray(dL_dpsi2)) + dL_dpsi1_gpu = self.gpuCache['dL_dpsi1_gpu'] + dL_dpsi2_gpu = self.gpuCache['dL_dpsi2_gpu'] + dL_dpsi1_gpu.set(np.asfortranarray(dL_dpsi1)) + dL_dpsi2_gpu.set(np.asfortranarray(dL_dpsi2)) self.reset_derivative() self.g_psi1compDer(dvar_gpu,dl_gpu,dZ_gpu,dmu_gpu,dS_gpu,dL_dpsi1_gpu,psi1_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1)) -# print np.abs(dvar_psi1-dvar_gpu.get().sum(axis=-1)).max() -# print np.abs(dl_psi1-dl_gpu.get().sum(axis=-1)).max() -# print np.abs(dmu_psi1-dmu_gpu.get().sum(axis=-1)).max() -# print np.abs(dS_psi1-dS_gpu.get().sum(axis=-1)).max() -# print np.abs(dZ_psi1-dZ_gpu.get()).max() - -# self.reset_derivative() self.g_psi2compDer(dvar_gpu,dl_gpu,dZ_gpu,dmu_gpu,dS_gpu,dL_dpsi2_gpu,psi2n_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1)) -# print np.abs(dvar_psi2-dvar_gpu.get().sum(axis=-1)).max() -# print np.abs(dl_psi2-dl_gpu.get().sum(axis=-1)).max() -# print np.abs(dmu_psi2-dmu_gpu.get().sum(axis=-1)).max() -# print np.abs(dS_psi2-dS_gpu.get().sum(axis=-1)).max() -# print np.abs(dZ_psi2-dZ_gpu.get()).max() - dL_dvar = np.sum(dL_dpsi0) + dvar_gpu.get().sum() - dL_dmu = dmu_gpu.get().sum(axis=-1) - dL_dS = dS_gpu.get().sum(axis=-1) + dL_dvar = np.sum(dL_dpsi0) + gpuarray.sum(dvar_gpu).get() + sum_axis(grad_mu_gpu,dmu_gpu,N*Q,self.blocknum) + dL_dmu = grad_mu_gpu.get() + sum_axis(grad_S_gpu,dS_gpu,N*Q,self.blocknum) + dL_dS = grad_S_gpu.get() dL_dZ = dZ_gpu.get() if ARD: - dL_dlengscale = dl_gpu.get().sum(axis=-1) + sum_axis(grad_l_gpu,dl_gpu,Q,self.blocknum) + dL_dlengscale = grad_l_gpu.get() else: - dL_dlengscale = dl_gpu.get().sum() + dL_dlengscale = gpuarray.sum(dl_gpu).get() -# print np.abs(dL_dlengscale - dl_psi1-dl_psi2).max() - -# -# dL_dvar = np.sum(dL_dpsi0) + dvar_psi1 + dvar_psi2 -# -# dL_dlengscale = dl_psi1 + dl_psi2 -# if not ARD: -# dL_dlengscale = dL_dlengscale.sum() -# -# dL_dmu = dmu_psi1 + dmu_psi2 -# dL_dS = dS_psi1 + dS_psi2 -# dL_dZ = dZ_psi1 + dZ_psi2 - return dL_dvar, dL_dlengscale, dL_dZ, dL_dmu, dL_dS -def psicomputations(variance, lengthscale, Z, variational_posterior): - """ - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - """ - # here are the "statistics" for psi0, psi1 and psi2 - # Produced intermediate results: - # _psi1 NxM - mu = variational_posterior.mean - S = variational_posterior.variance - - psi0 = np.empty(mu.shape[0]) - psi0[:] = variance - psi1 = _psi1computations(variance, lengthscale, Z, mu, S) - psi2 = _psi2computations(variance, lengthscale, Z, mu, S).sum(axis=0) - return psi0, psi1, psi2 - -def __psi1computations(variance, lengthscale, Z, mu, S): - """ - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - """ - # here are the "statistics" for psi1 - # Produced intermediate results: - # _psi1 NxM - - lengthscale2 = np.square(lengthscale) - - # psi1 - _psi1_logdenom = np.log(S/lengthscale2+1.).sum(axis=-1) # N - _psi1_log = (_psi1_logdenom[:,None]+np.einsum('nmq,nq->nm',np.square(mu[:,None,:]-Z[None,:,:]),1./(S+lengthscale2)))/(-2.) - _psi1 = variance*np.exp(_psi1_log) - - return _psi1 - -def __psi2computations(variance, lengthscale, Z, mu, S): - """ - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - """ - # here are the "statistics" for psi2 - # Produced intermediate results: - # _psi2 MxM - - lengthscale2 = np.square(lengthscale) - - _psi2_logdenom = np.log(2.*S/lengthscale2+1.).sum(axis=-1)/(-2.) # N - _psi2_exp1 = (np.square(Z[:,None,:]-Z[None,:,:])/lengthscale2).sum(axis=-1)/(-4.) #MxM - Z_hat = (Z[:,None,:]+Z[None,:,:])/2. #MxMxQ - denom = 1./(2.*S+lengthscale2) - _psi2_exp2 = -(np.square(mu)*denom).sum(axis=-1)[:,None,None]+2.*np.einsum('nq,moq,nq->nmo',mu,Z_hat,denom)-np.einsum('moq,nq->nmo',np.square(Z_hat),denom) - _psi2 = variance*variance*np.exp(_psi2_logdenom[:,None,None]+_psi2_exp1[None,:,:]+_psi2_exp2) - - - return _psi2 - -def psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior): - ARD = (len(lengthscale)!=1) - - dvar_psi1, dl_psi1, dZ_psi1, dmu_psi1, dS_psi1 = _psi1compDer(dL_dpsi1, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance) - dvar_psi2, dl_psi2, dZ_psi2, dmu_psi2, dS_psi2 = _psi2compDer(dL_dpsi2, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance) - - dL_dvar = np.sum(dL_dpsi0) + dvar_psi1 + dvar_psi2 - - dL_dlengscale = dl_psi1 + dl_psi2 - if not ARD: - dL_dlengscale = dL_dlengscale.sum() - - dL_dmu = dmu_psi1 + dmu_psi2 - dL_dS = dS_psi1 + dS_psi2 - dL_dZ = dZ_psi1 + dZ_psi2 - - return dL_dvar, dL_dlengscale, dL_dZ, dL_dmu, dL_dS - -def _psi1compDer(dL_dpsi1, variance, lengthscale, Z, mu, S): - """ - dL_dpsi1 - NxM - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - """ - # here are the "statistics" for psi1 - # Produced intermediate results: dL_dparams w.r.t. psi1 - # _dL_dvariance 1 - # _dL_dlengthscale Q - # _dL_dZ MxQ - # _dL_dgamma NxQ - # _dL_dmu NxQ - # _dL_dS NxQ - - lengthscale2 = np.square(lengthscale) - - _psi1 = _psi1computations(variance, lengthscale, Z, mu, S) - Lpsi1 = dL_dpsi1*_psi1 - Zmu = Z[None,:,:]-mu[:,None,:] # NxMxQ - denom = 1./(S+lengthscale2) - Zmu2_denom = np.square(Zmu)*denom[:,None,:] #NxMxQ - _dL_dvar = Lpsi1.sum()/variance - _dL_dmu = np.einsum('nm,nmq,nq->nq',Lpsi1,Zmu,denom) - _dL_dS = np.einsum('nm,nmq,nq->nq',Lpsi1,(Zmu2_denom-1.),denom)/2. - _dL_dZ = -np.einsum('nm,nmq,nq->mq',Lpsi1,Zmu,denom) - _dL_dl = np.einsum('nm,nmq,nq->q',Lpsi1,(Zmu2_denom+(S/lengthscale2)[:,None,:]),denom*lengthscale) - - return _dL_dvar, _dL_dl, _dL_dZ, _dL_dmu, _dL_dS - -def _psi2compDer(dL_dpsi2, variance, lengthscale, Z, mu, S): - """ - Z - MxQ - mu - NxQ - S - NxQ - gamma - NxQ - dL_dpsi2 - MxM - """ - # here are the "statistics" for psi2 - # Produced the derivatives w.r.t. psi2: - # _dL_dvariance 1 - # _dL_dlengthscale Q - # _dL_dZ MxQ - # _dL_dgamma NxQ - # _dL_dmu NxQ - # _dL_dS NxQ - - lengthscale2 = np.square(lengthscale) - denom = 1./(2*S+lengthscale2) - denom2 = np.square(denom) - - _psi2 = _psi2computations(variance, lengthscale, Z, mu, S) # NxMxM - - Lpsi2 = dL_dpsi2[None,:,:]*_psi2 - Lpsi2sum = np.einsum('nmo->n',Lpsi2) #N - Lpsi2Z = np.einsum('nmo,oq->nq',Lpsi2,Z) #NxQ - Lpsi2Z2 = np.einsum('nmo,oq,oq->nq',Lpsi2,Z,Z) #NxQ - Lpsi2Z2p = np.einsum('nmo,mq,oq->nq',Lpsi2,Z,Z) #NxQ - Lpsi2Zhat = Lpsi2Z - Lpsi2Zhat2 = (Lpsi2Z2+Lpsi2Z2p)/2 - - _dL_dvar = Lpsi2sum.sum()*2/variance - _dL_dmu = (-2*denom) * (mu*Lpsi2sum[:,None]-Lpsi2Zhat) - _dL_dS = (2*np.square(denom))*(np.square(mu)*Lpsi2sum[:,None]-2*mu*Lpsi2Zhat+Lpsi2Zhat2) - denom*Lpsi2sum[:,None] - _dL_dZ = -np.einsum('nmo,oq->oq',Lpsi2,Z)/lengthscale2+np.einsum('nmo,oq->mq',Lpsi2,Z)/lengthscale2+ \ - 2*np.einsum('nmo,nq,nq->mq',Lpsi2,mu,denom) - np.einsum('nmo,nq,mq->mq',Lpsi2,denom,Z) - np.einsum('nmo,oq,nq->mq',Lpsi2,Z,denom) - _dL_dl = 2*lengthscale* ((S/lengthscale2*denom+np.square(mu*denom))*Lpsi2sum[:,None]+(Lpsi2Z2-Lpsi2Z2p)/(2*np.square(lengthscale2))- - (2*mu*denom2)*Lpsi2Zhat+denom2*Lpsi2Zhat2).sum(axis=0) - - return _dL_dvar, _dL_dl, _dL_dZ, _dL_dmu, _dL_dS - -_psi1computations = Cacher(__psi1computations, limit=1) -_psi2computations = Cacher(__psi2computations, limit=1) diff --git a/GPy/kern/_src/rbf.py b/GPy/kern/_src/rbf.py index 409c2e41..a6f31ff2 100644 --- a/GPy/kern/_src/rbf.py +++ b/GPy/kern/_src/rbf.py @@ -29,6 +29,8 @@ class RBF(Stationary): self.psicomp = PSICOMP_RBF() if self.useGPU: self.psicomp = PSICOMP_RBF_GPU() + else: + self.psicomp = PSICOMP_RBF() def K_of_r(self, r): return self.variance * np.exp(-0.5 * r**2) @@ -41,41 +43,22 @@ class RBF(Stationary): #---------------------------------------# def psi0(self, Z, variational_posterior): - if self.useGPU: - return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[0] - else: - return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[0] + return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[0] def psi1(self, Z, variational_posterior): - if self.useGPU: - return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[1] - else: - return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[1] + return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[1] def psi2(self, Z, variational_posterior): - if self.useGPU: - return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[2] - else: - return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[2] + return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[2] def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): - if self.useGPU: - dL_dvar, dL_dlengscale = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[:2] - self.variance.gradient = dL_dvar - self.lengthscale.gradient = dL_dlengscale - else: - dL_dvar, dL_dlengscale = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[:2] - self.variance.gradient = dL_dvar - self.lengthscale.gradient = dL_dlengscale + dL_dvar, dL_dlengscale = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[:2] + self.variance.gradient = dL_dvar + self.lengthscale.gradient = dL_dlengscale def gradients_Z_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): - if self.useGPU: - return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[2] - else: - return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[2] + return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[2] def gradients_qX_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): - if self.useGPU: - return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[3:] - else: - return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[3:] + return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[3:] + From e486f3fd991caac90d87e0580f38f5552548e33a Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Fri, 20 Jun 2014 15:03:44 +0100 Subject: [PATCH 24/55] fix import issue on no-gpu machine --- GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py index d0d5bec5..777d4749 100644 --- a/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py +++ b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py @@ -6,11 +6,11 @@ import numpy as np from ....util.caching import Cache_this from . import PSICOMP_RBF from ....util import gpu_init -from ....util.linalg_gpu import sum_axis try: import pycuda.gpuarray as gpuarray from pycuda.compiler import SourceModule + from ....util.linalg_gpu import sum_axis except: pass From ca1edecce4627a3fb9579389f9e20a8383f9148e Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Fri, 20 Jun 2014 18:02:35 +0100 Subject: [PATCH 25/55] rbf gpu usable --- .../latent_function_inference/var_dtc_gpu.py | 194 ++++++++---------- .../var_dtc_parallel.py | 45 ++-- GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py | 17 +- GPy/models/bayesian_gplvm.py | 3 + GPy/util/linalg_gpu.py | 27 +++ 5 files changed, 140 insertions(+), 146 deletions(-) diff --git a/GPy/inference/latent_function_inference/var_dtc_gpu.py b/GPy/inference/latent_function_inference/var_dtc_gpu.py index f83a5bc0..3bd5c347 100644 --- a/GPy/inference/latent_function_inference/var_dtc_gpu.py +++ b/GPy/inference/latent_function_inference/var_dtc_gpu.py @@ -66,7 +66,6 @@ class VarDTC_GPU(LatentFunctionInference): 'beta_gpu' :gpuarray.empty((ndata,),np.float64,order='F'), 'YT_gpu' :gpuarray.to_gpu(np.asfortranarray(Y.T)), # DxN 'betaYT_gpu' :gpuarray.empty(Y.T.shape,np.float64,order='F'), # DxN - 'psi2_t_gpu' :gpuarray.empty((num_inducing*num_inducing*self.batchsize),np.float64,order='F'), # inference_minibatch 'dL_dpsi0_gpu' :gpuarray.empty((self.batchsize,),np.float64,order='F'), 'dL_dpsi1_gpu' :gpuarray.empty((self.batchsize,num_inducing),np.float64,order='F'), @@ -122,6 +121,89 @@ class VarDTC_GPU(LatentFunctionInference): else: return jitchol(tdot(Y)) + def gatherPsiStat(self, kern, X, Z, Y, beta, uncertain_inputs, het_noise): + num_inducing, input_dim = Z.shape[0], Z.shape[1] + num_data, output_dim = Y.shape + trYYT = self._trYYT + psi1Y_gpu = self.gpuCache['psi1Y_gpu'] + psi2_gpu = self.gpuCache['psi2_gpu'] + beta_gpu = self.gpuCache['beta_gpu'] + YT_gpu = self.gpuCache['YT_gpu'] + betaYT_gpu = self.gpuCache['betaYT_gpu'] + + beta_gpu.fill(beta) + betaYT_gpu.fill(0.) + cublas.cublasDaxpy(self.cublas_handle, betaYT_gpu.size, beta, YT_gpu.gpudata, 1, betaYT_gpu.gpudata, 1) + YRY_full = trYYT*beta + + if kern.useGPU: + psi1Y_gpu.fill(0.) + psi2_gpu.fill(0.) + psi0_full = 0 + + for n_start in xrange(0,num_data,self.batchsize): + n_end = min(self.batchsize+n_start, num_data) + ndata = n_end - n_start + X_slice = X[n_start:n_end] + betaYT_gpu_slice = betaYT_gpu[:,n_start:n_end] + + if uncertain_inputs: + psi0 = kern.psi0(Z, X_slice) + psi1p_gpu = kern.psi1(Z, X_slice) + psi2p_gpu = kern.psi2(Z, X_slice) + else: + psi0 = kern.Kdiag(X_slice) + psi1p_gpu = kern.K(X_slice, Z) + + cublas.cublasDgemm(self.cublas_handle, 'T', 'T', num_inducing, output_dim, ndata, 1.0, psi1p_gpu.gpudata, ndata, betaYT_gpu_slice.gpudata, output_dim, 1.0, psi1Y_gpu.gpudata, num_inducing) + + psi0_full += psi0.sum() + + if uncertain_inputs: + sum_axis(psi2_gpu,psi2p_gpu,1,1) + else: + cublas.cublasDgemm(self.cublas_handle, 'T', 'N', num_inducing, num_inducing, ndata, beta, psi1p_gpu.gpudata, ndata, psi1p_gpu.gpudata, ndata, 1.0, psi2_gpu.gpudata, num_inducing) + + psi0_full *= beta + if uncertain_inputs: + cublas.cublasDscal(self.cublas_handle, psi2_gpu.size, beta, psi2_gpu.gpudata, 1) + + else: + psi2_full = np.zeros((num_inducing,num_inducing)) + psi1Y_full = np.zeros((output_dim,num_inducing)) # DxM + psi0_full = 0. + YRY_full = 0. + + for n_start in xrange(0,num_data,self.batchsize): + n_end = min(self.batchsize+n_start, num_data) + Y_slice = Y[n_start:n_end] + X_slice = X[n_start:n_end] + + if het_noise: + b = beta[n_start] + YRY_full += np.inner(Y_slice, Y_slice)*b + else: + b = beta + + if uncertain_inputs: + psi0 = kern.psi0(Z, X_slice) + psi1 = kern.psi1(Z, X_slice) + psi2_full += kern.psi2(Z, X_slice)*b + else: + psi0 = kern.Kdiag(X_slice) + psi1 = kern.K(X_slice, Z) + psi2_full += np.dot(psi1.T,psi1)*b + + psi0_full += psi0.sum()*b + psi1Y_full += np.dot(Y_slice.T,psi1)*b # DxM + + if not het_noise: + YRY_full = trYYT*beta + psi1Y_gpu.set(psi1Y_full) + psi2_gpu.set(psi2_full) + + return psi0_full, YRY_full + def inference_likelihood(self, kern, X, Z, likelihood, Y): """ The first phase of inference: @@ -146,118 +228,10 @@ class VarDTC_GPU(LatentFunctionInference): else: uncertain_inputs = False - - trYYT = self._trYYT - psi1Y_gpu = self.gpuCache['psi1Y_gpu'] psi2_gpu = self.gpuCache['psi2_gpu'] - beta_gpu = self.gpuCache['beta_gpu'] - YT_gpu = self.gpuCache['YT_gpu'] - betaYT_gpu = self.gpuCache['betaYT_gpu'] - psi2_t_gpu = self.gpuCache['psi2_t_gpu'] - if het_noise: - beta_gpu.set(np.asfortranarray(beta)) - mul_bcast(betaYT_gpu,beta_gpu,YT_gpu,beta_gpu.size) - YRY_full = cublas.cublasDdot(self.cublas_handle, YT_gpu.size, betaYT_gpu.gpudata, 1, YT_gpu.gpudata, 1) - else: - beta_gpu.fill(beta) - betaYT_gpu.fill(0.) - cublas.cublasDaxpy(self.cublas_handle, betaYT_gpu.size, beta, YT_gpu.gpudata, 1, betaYT_gpu.gpudata, 1) - YRY_full = trYYT*beta - - if kern.useGPU: - psi1Y_gpu.fill(0.) - psi2_gpu.fill(0.) - psi0_full = 0 - - for n_start in xrange(0,num_data,self.batchsize): - n_end = min(self.batchsize+n_start, num_data) - ndata = n_end - n_start - X_slice = X[n_start:n_end] - beta_gpu_slice = beta_gpu[n_start:n_end] - betaYT_gpu_slice = betaYT_gpu[:,n_start:n_end] - if ndata==self.batchsize: - psi2_t_gpu_slice = psi2_t_gpu - else: - psi2_t_gpu_slice = psi2_t_gpu[:num_inducing*num_inducing*ndata] - if uncertain_inputs: - psi0p_gpu = kern.psi0(Z, X_slice) - psi1p_gpu = kern.psi1(Z, X_slice) - psi2p_gpu = kern.psi2(Z, X_slice) - else: - psi0p_gpu = kern.Kdiag(X_slice) - psi1p_gpu = kern.K(X_slice, Z) - - cublas.cublasDgemm(self.cublas_handle, 'T', 'T', num_inducing, output_dim, ndata, 1.0, psi1p_gpu.gpudata, ndata, betaYT_gpu_slice.gpudata, output_dim, 1.0, psi1Y_gpu.gpudata, num_inducing) - - if het_noise: - psi0_full += cublas.cublasDdot(self.cublas_handle, psi0p_gpu.size, beta_gpu_slice.gpudata, 1, psi0p_gpu.gpudata, 1) - else: - psi0_full += gpuarray.sum(psi0p_gpu).get() - - if uncertain_inputs: - if het_noise: - mul_bcast(psi2_t_gpu_slice,beta_gpu_slice,psi2p_gpu,beta_gpu_slice.size) - sum_axis(psi2_gpu,psi2_t_gpu_slice,1,ndata) - else: - sum_axis(psi2_gpu,psi2p_gpu,1,ndata) - else: - if het_noise: - psi1_t_gpu = psi2_t_gpu_slice[:,num_inducing*ndata] - mul_bcast(psi1_t_gpu,beta_gpu_slice,psi1p_gpu,beta_gpu_slice.size) - cublas.cublasDgemm(self.cublas_handle, 'T', 'N', num_inducing, num_inducing, ndata, 1.0, psi1p_gpu.gpudata, ndata, psi1_t_gpu.gpudata, ndata, 1.0, psi2_gpu.gpudata, num_inducing) - else: - cublas.cublasDgemm(self.cublas_handle, 'T', 'N', num_inducing, num_inducing, ndata, beta, psi1p_gpu.gpudata, ndata, psi1p_gpu.gpudata, ndata, 1.0, psi2_gpu.gpudata, num_inducing) - - if not het_noise: - psi0_full *= beta - if uncertain_inputs: - cublas.cublasDscal(self.cublas_handle, psi2_gpu.size, beta, psi2_gpu.gpudata, 1) - - else: - psi2_full = np.zeros((num_inducing,num_inducing),order='F') - psi1Y_full = np.zeros((num_inducing,output_dim),order='F') # MxD - psi0_full = 0 - - for n_start in xrange(0,num_data,self.batchsize): - n_end = min(self.batchsize+n_start, num_data) - Y_slice = Y[n_start:n_end] - X_slice = X[n_start:n_end] - if uncertain_inputs: - psi0 = kern.psi0(Z, X_slice) - psi1 = kern.psi1(Z, X_slice) - psi2 = kern.psi2(Z, X_slice) - else: - psi0 = kern.Kdiag(X_slice) - psi1 = kern.K(X_slice, Z) - - if het_noise: - beta_slice = beta[n_start:n_end] - psi0_full += (beta_slice*psi0).sum() - psi1Y_full += np.dot(psi1.T,beta_slice[:,None]*Y_slice) # MxD - else: - psi0_full += psi0.sum() - psi1Y_full += np.dot(psi1.T,Y_slice) # MxD - - if uncertain_inputs: - if het_noise: - psi2_full += np.einsum('n,nmo->mo',beta_slice,psi2) - else: - psi2_full += psi2 - else: - if het_noise: - psi2_full += np.einsum('n,nm,no->mo',beta_slice,psi1,psi1) - else: - psi2_full += np.outer(psi1.T, psi1.T) - - if not het_noise: - psi0_full *= beta - psi1Y_full *= beta - psi2_full *= beta - - psi1Y_gpu.set(psi1Y_full) - psi2_gpu.set(psi2_full) + psi0_full, YRY_full = self.gatherPsiStat(kern, X, Z, Y, beta, uncertain_inputs, het_noise) #====================================================================== # Compute Common Components diff --git a/GPy/inference/latent_function_inference/var_dtc_parallel.py b/GPy/inference/latent_function_inference/var_dtc_parallel.py index dc4f45d5..54bc11ac 100644 --- a/GPy/inference/latent_function_inference/var_dtc_parallel.py +++ b/GPy/inference/latent_function_inference/var_dtc_parallel.py @@ -73,13 +73,14 @@ class VarDTC_minibatch(LatentFunctionInference): else: return jitchol(tdot(Y)) - def gatherPsiStat(self, kern, X, Z, Y, beta, uncertain_inputs, het_noise): + def gatherPsiStat(self, kern, X, Z, Y, beta, uncertain_inputs): num_inducing = Z.shape[0] num_data, output_dim = Y.shape if self.batchsize == None: self.batchsize = num_data + het_noise = beta.size > 1 trYYT = self.get_trYYT(Y) @@ -88,46 +89,30 @@ class VarDTC_minibatch(LatentFunctionInference): psi0_full = 0. YRY_full = 0. - for n_start in xrange(0,num_data,self.batchsize): - + for n_start in xrange(0,num_data,self.batchsize): n_end = min(self.batchsize+n_start, num_data) - Y_slice = Y[n_start:n_end] X_slice = X[n_start:n_end] + if het_noise: + b = beta[n_start] + YRY_full += np.inner(Y_slice, Y_slice)*b + else: + b = beta + if uncertain_inputs: psi0 = kern.psi0(Z, X_slice) psi1 = kern.psi1(Z, X_slice) - psi2 = kern.psi2(Z, X_slice) + psi2_full += kern.psi2(Z, X_slice)*b else: psi0 = kern.Kdiag(X_slice) psi1 = kern.K(X_slice, Z) - psi2 = None - - if het_noise: - beta_slice = beta[n_start:n_end] - psi0_full += (beta_slice*psi0).sum() - psi1Y_full += np.dot(beta_slice*Y_slice.T,psi1) # DxM - YRY_full += (beta_slice*np.square(Y_slice).sum(axis=-1)).sum() - else: - psi0_full += psi0.sum() - psi1Y_full += np.dot(Y_slice.T,psi1) # DxM - - if uncertain_inputs: - if het_noise: - psi2_full += beta_slice*psi2 - else: - psi2_full += psi2 - else: - if het_noise: - psi2_full += beta_slice*np.outer(psi1,psi1) - else: - psi2_full += np.dot(psi1.T,psi1) + psi2_full += np.dot(psi1.T,psi1)*b + psi0_full += psi0.sum()*b + psi1Y_full += np.dot(Y_slice.T,psi1)*b # DxM + if not het_noise: - psi0_full *= beta - psi1Y_full *= beta - psi2_full *= beta YRY_full = trYYT*beta if self.mpi_comm != None: @@ -168,7 +153,7 @@ class VarDTC_minibatch(LatentFunctionInference): if het_noise: self.batchsize = 1 - psi0_full, psi1Y_full, psi2_full, YRY_full = self.gatherPsiStat(kern, X, Z, Y, beta, uncertain_inputs, het_noise) + psi0_full, psi1Y_full, psi2_full, YRY_full = self.gatherPsiStat(kern, X, Z, Y, beta, uncertain_inputs) #====================================================================== # Compute Common Components diff --git a/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py index 777d4749..270801ff 100644 --- a/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py +++ b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py @@ -248,7 +248,6 @@ class PSICOMP_RBF_GPU(PSICOMP_RBF): 'Z_gpu' :gpuarray.empty((M,Q),np.float64,order='F'), 'mu_gpu' :gpuarray.empty((N,Q),np.float64,order='F'), 'S_gpu' :gpuarray.empty((N,Q),np.float64,order='F'), - 'psi0_gpu' :gpuarray.empty((N,),np.float64,order='F'), 'psi1_gpu' :gpuarray.empty((N,M),np.float64,order='F'), 'psi2_gpu' :gpuarray.empty((M,M),np.float64,order='F'), 'psi2n_gpu' :gpuarray.empty((N,M,M),np.float64,order='F'), @@ -265,6 +264,10 @@ class PSICOMP_RBF_GPU(PSICOMP_RBF): 'grad_mu_gpu' :gpuarray.empty((N,Q,),np.float64, order='F'), 'grad_S_gpu' :gpuarray.empty((N,Q,),np.float64, order='F'), } + else: + assert N==self.gpuCache['mu_gpu'].shape[0] + assert M==self.gpuCache['Z_gpu'].shape[0] + assert Q==self.gpuCache['l_gpu'].shape[0] def sync_params(self, lengthscale, Z, mu, S): if len(lengthscale)==1: @@ -299,7 +302,6 @@ class PSICOMP_RBF_GPU(PSICOMP_RBF): self._initGPUCache(N,M,Q) self.sync_params(lengthscale, Z, variational_posterior.mean, variational_posterior.variance) - psi0_gpu = self.gpuCache['psi0_gpu'] psi1_gpu = self.gpuCache['psi1_gpu'] psi2_gpu = self.gpuCache['psi2_gpu'] psi2n_gpu = self.gpuCache['psi2n_gpu'] @@ -308,14 +310,15 @@ class PSICOMP_RBF_GPU(PSICOMP_RBF): mu_gpu = self.gpuCache['mu_gpu'] S_gpu = self.gpuCache['S_gpu'] - psi0_gpu.fill(variance) + psi0 = np.empty((N,)) + psi0[:] = variance self.g_psi1computations(psi1_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1)) self.g_psi2computations(psi2_gpu, psi2n_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1)) if self.GPU_direct: - return psi0_gpu, psi1_gpu, psi2_gpu + return psi0, psi1_gpu, psi2_gpu else: - return psi0_gpu.get(), psi1_gpu.get(), psi2_gpu.get() + return psi0, psi1_gpu.get(), psi2_gpu.get() @Cache_this(limit=1, ignore_args=(0,1,2,3)) def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior): @@ -340,17 +343,19 @@ class PSICOMP_RBF_GPU(PSICOMP_RBF): if self.GPU_direct: dL_dpsi1_gpu = dL_dpsi1 dL_dpsi2_gpu = dL_dpsi2 + dL_dpsi0_sum = gpuarray.sum(dL_dpsi0).get() else: dL_dpsi1_gpu = self.gpuCache['dL_dpsi1_gpu'] dL_dpsi2_gpu = self.gpuCache['dL_dpsi2_gpu'] dL_dpsi1_gpu.set(np.asfortranarray(dL_dpsi1)) dL_dpsi2_gpu.set(np.asfortranarray(dL_dpsi2)) + dL_dpsi0_sum = dL_dpsi0.sum() self.reset_derivative() self.g_psi1compDer(dvar_gpu,dl_gpu,dZ_gpu,dmu_gpu,dS_gpu,dL_dpsi1_gpu,psi1_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1)) self.g_psi2compDer(dvar_gpu,dl_gpu,dZ_gpu,dmu_gpu,dS_gpu,dL_dpsi2_gpu,psi2n_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1)) - dL_dvar = np.sum(dL_dpsi0) + gpuarray.sum(dvar_gpu).get() + dL_dvar = dL_dpsi0_sum + gpuarray.sum(dvar_gpu).get() sum_axis(grad_mu_gpu,dmu_gpu,N*Q,self.blocknum) dL_dmu = grad_mu_gpu.get() sum_axis(grad_S_gpu,dS_gpu,N*Q,self.blocknum) diff --git a/GPy/models/bayesian_gplvm.py b/GPy/models/bayesian_gplvm.py index b77768ae..65ab2469 100644 --- a/GPy/models/bayesian_gplvm.py +++ b/GPy/models/bayesian_gplvm.py @@ -62,6 +62,9 @@ class BayesianGPLVM(SparseGP): else: from ..inference.latent_function_inference.var_dtc import VarDTC inference_method = VarDTC() + + if kernel.useGPU and isinstance(inference_method, VarDTC_GPU): + kernel.psicomp.GPU_direct = True SparseGP.__init__(self, X, Y, Z, kernel, likelihood, inference_method, name, **kwargs) self.add_parameter(self.X, index=0) diff --git a/GPy/util/linalg_gpu.py b/GPy/util/linalg_gpu.py index cf6d483e..1b9b0594 100644 --- a/GPy/util/linalg_gpu.py +++ b/GPy/util/linalg_gpu.py @@ -12,6 +12,9 @@ from ..util import gpu_init try: from pycuda.reduction import ReductionKernel from pycuda.elementwise import ElementwiseKernel + import scikits.cuda.linalg as culinalg + from scikits.cuda import cublas + from scikits.cuda.cula import culaExceptions # log|A| for A is a low triangle matrix # logDiagSum(A, A.shape[0]+1) @@ -60,3 +63,27 @@ try: except: pass + +def jitchol(A, L, cublas_handle, maxtries=5): + try: + cublas.cublasDcopy(cublas_handle, A.size, A.gpudata, 1, L.gpudata, 1) + culinalg.cho_factor(L,'L') + except culaExceptions: + + + diagA = np.diag(A) + if np.any(diagA <= 0.): + raise linalg.LinAlgError, "not pd: non-positive diagonal elements" + jitter = diagA.mean() * 1e-6 + while maxtries > 0 and np.isfinite(jitter): + print 'Warning: adding jitter of {:.10e}'.format(jitter) + try: + return linalg.cholesky(A + np.eye(A.shape[0]).T * jitter, lower=True) + except: + jitter *= 10 + finally: + maxtries -= 1 + raise linalg.LinAlgError, "not positive definite, even with jitter." + + + From 59172435e2c6b138e81bdcaa9e8f1ff9b82ebc2d Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Fri, 20 Jun 2014 21:51:51 +0100 Subject: [PATCH 26/55] fix the remaining problem of cache.py --- GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py | 59 ++++++++++++++++++----- GPy/kern/_src/stationary.py | 1 - GPy/util/caching.py | 21 +++++--- 3 files changed, 61 insertions(+), 20 deletions(-) diff --git a/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py index 270801ff..43fffcd1 100644 --- a/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py +++ b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py @@ -58,7 +58,23 @@ gpu_code = """ } } - __global__ void psi1computations(double *psi1, double var, double *l, double *Z, double *mu, double *S, int N, int M, int Q) + __global__ void compDenom(double *log_denom1, double *log_denom2, double *l, double *S, int N, int Q) + { + int n_start, n_end; + divide_data(N, gridDim.x, blockIdx.x, &n_start, &n_end); + + for(int i=n_start*Q+threadIdx.x; i Date: Mon, 23 Jun 2014 12:02:36 +0100 Subject: [PATCH 27/55] fix pickle for RBF GPU kernel --- GPy/kern/_src/rbf.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/GPy/kern/_src/rbf.py b/GPy/kern/_src/rbf.py index a6f31ff2..4aa3ed74 100644 --- a/GPy/kern/_src/rbf.py +++ b/GPy/kern/_src/rbf.py @@ -38,6 +38,15 @@ class RBF(Stationary): def dK_dr(self, r): return -r*self.K_of_r(r) + def __getstate__(self): + dc = super(RBF, self).__getstate__() + if self.useGPU: + dc['psicomp'] = PSICOMP_RBF() + return dc + + def __setstate__(self, state): + return super(RBF, self).__setstate__(state) + #---------------------------------------# # PSI statistics # #---------------------------------------# From cf2bf3bbe2c59910a7a45df1270154983d0be01f Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Mon, 23 Jun 2014 14:23:43 +0100 Subject: [PATCH 28/55] var_dtc_parallel make YY.T speed up --- GPy/core/gp.py | 2 +- .../var_dtc_parallel.py | 29 ++++++++++++------- GPy/models/ss_gplvm.py | 5 ++-- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/GPy/core/gp.py b/GPy/core/gp.py index f4b02128..3825a48e 100644 --- a/GPy/core/gp.py +++ b/GPy/core/gp.py @@ -277,4 +277,4 @@ class GP(Model): """ self.inference_method.on_optimization_start() super(GP, self).optimize(optimizer, start, **kwargs) - self.inference_method.on_optimization_end() \ No newline at end of file + self.inference_method.on_optimization_end() diff --git a/GPy/inference/latent_function_inference/var_dtc_parallel.py b/GPy/inference/latent_function_inference/var_dtc_parallel.py index 54bc11ac..450e767a 100644 --- a/GPy/inference/latent_function_inference/var_dtc_parallel.py +++ b/GPy/inference/latent_function_inference/var_dtc_parallel.py @@ -39,14 +39,15 @@ class VarDTC_minibatch(LatentFunctionInference): self.midRes = {} self.batch_pos = 0 # the starting position of the current mini-batch + self.Y_speedup = False # Replace Y with the cholesky factor of YY.T, but the posterior inference will be wrong def __getstate__(self): # has to be overridden, as Cacher objects cannot be pickled. - return self.batchsize, self.limit + return self.batchsize, self.limit, self.Y_speedup def __setstate__(self, state): # has to be overridden, as Cacher objects cannot be pickled. - self.batchsize, self.limit = state + self.batchsize, self.limit, self.Y_speedup = state self.mpi_comm = None self.midRes = {} self.batch_pos = 0 @@ -75,14 +76,16 @@ class VarDTC_minibatch(LatentFunctionInference): def gatherPsiStat(self, kern, X, Z, Y, beta, uncertain_inputs): - num_inducing = Z.shape[0] - num_data, output_dim = Y.shape - - if self.batchsize == None: - self.batchsize = num_data het_noise = beta.size > 1 trYYT = self.get_trYYT(Y) + if self.Y_speedup and not het_noise: + Y = self.get_YYTfactor(Y) + + num_inducing = Z.shape[0] + num_data, output_dim = Y.shape + if self.batchsize == None: + self.batchsize = num_data psi2_full = np.zeros((num_inducing,num_inducing)) psi1Y_full = np.zeros((output_dim,num_inducing)) # DxM @@ -200,8 +203,11 @@ class VarDTC_minibatch(LatentFunctionInference): #====================================================================== # Compute the Posterior distribution of inducing points p(u|Y) #====================================================================== - - post = Posterior(woodbury_inv=KmmInvPsi2P, woodbury_vector=v, K=Kmm, mean=None, cov=None, K_chol=Lm) + + if not self.Y_speedup or het_noise: + post = Posterior(woodbury_inv=KmmInvPsi2P, woodbury_vector=v, K=Kmm, mean=None, cov=None, K_chol=Lm) + else: + post = None #====================================================================== # Compute dL_dthetaL for uncertian input and non-heter noise @@ -232,7 +238,10 @@ class VarDTC_minibatch(LatentFunctionInference): het_noise = beta.size > 1 # VVT_factor is a matrix such that tdot(VVT_factor) = VVT...this is for efficiency! #self.YYTfactor = beta*self.get_YYTfactor(Y) - YYT_factor = Y + if self.Y_speedup and not het_noise: + YYT_factor = self.get_YYTfactor(Y) + else: + YYT_factor = Y n_start = self.batch_pos n_end = min(self.batchsize+n_start, num_data) diff --git a/GPy/models/ss_gplvm.py b/GPy/models/ss_gplvm.py index b10991ab..ab32e462 100644 --- a/GPy/models/ss_gplvm.py +++ b/GPy/models/ss_gplvm.py @@ -31,7 +31,8 @@ class SSGPLVM(SparseGP): self.mpi_comm = mpi_comm self.__IN_OPTIMIZATION__ = False - + self.group_spike = group_spike + if X == None: from ..util.initialization import initialize_latent X, fracs = initialize_latent(init, input_dim, Y) @@ -50,7 +51,7 @@ class SSGPLVM(SparseGP): gamma[:] = 0.5 if group_spike: - gamma[:] = gamma.mean(axis=0) + gamma[:] = gamma[:,0] if Z is None: Z = np.random.permutation(X.copy())[:num_inducing] From 36d53f815cb7b88b776dcc4c31168c9447efcdf0 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Mon, 23 Jun 2014 14:36:28 +0100 Subject: [PATCH 29/55] merge ties branch into psi2 --- GPy/core/parameterization/index_operations.py | 47 +++++++++++++++---- GPy/core/parameterization/parameter_core.py | 30 ++++++++++++ 2 files changed, 68 insertions(+), 9 deletions(-) diff --git a/GPy/core/parameterization/index_operations.py b/GPy/core/parameterization/index_operations.py index 7fc76d07..78c0d2b9 100644 --- a/GPy/core/parameterization/index_operations.py +++ b/GPy/core/parameterization/index_operations.py @@ -22,15 +22,44 @@ def extract_properties_to_index(index, props): class ParameterIndexOperations(object): - ''' - Index operations for storing param index _properties - This class enables index with slices retrieved from object.__getitem__ calls. - Adding an index will add the selected indexes by the slice of an indexarray - indexing a shape shaped array to the flattened index array. Remove will - remove the selected slice indices from the flattened array. - You can give an offset to set an offset for the given indices in the - index array, for multi-param handling. - ''' + """ + This object wraps a dictionary, whos keys are _operations_ that we'd like + to apply to a parameter array, and whose values are np integer arrays which + index the parameter array appropriately. + + A model instance will contain one instance of this class for each thing + that needs indexing (i.e. constraints, ties and priors). Parameters within + the model constain instances of the ParameterIndexOperationsView class, + which can map from a 'local' index (starting 0) to this global index. + + Here's an illustration: + + #======================================================================= + model : 0 1 2 3 4 5 6 7 8 9 + key1: 4 5 + key2: 7 8 + + param1: 0 1 2 3 4 5 + key1: 2 3 + key2: 5 + + param2: 0 1 2 3 4 + key1: 0 + key2: 2 3 + #======================================================================= + + The views of this global index have a subset of the keys in this global + (model) index. + + Adding a new key (e.g. a constraint) to a view will cause the view to pass + the new key to the global index, along with the local index and an offset. + This global index then stores the key and the appropriate global index + (which can be seen by the view). + + See also: + ParameterIndexOperationsView + + """ _offset = 0 def __init__(self, constraints=None): self._properties = IntArrayDict() diff --git a/GPy/core/parameterization/parameter_core.py b/GPy/core/parameterization/parameter_core.py index dab5c2d0..1f8b347d 100644 --- a/GPy/core/parameterization/parameter_core.py +++ b/GPy/core/parameterization/parameter_core.py @@ -495,6 +495,35 @@ class Indexable(Nameable, Observable): #=========================================================================== # Constrain operations -> done #=========================================================================== + + def tie(self, name): + #remove any constraints + old_const = self.constraints.properties()[:] + self.unconstrain() + + #set these parameters to be 'fixed' as in, not optimized + self._highest_parent_._set_fixed(self, self._raveled_index()) + + #see if a tie exists with that name + if name in self._highest_parent_.ties: + t = self._highest_parent_.ties[name] + else: + #create a tie object + value = np.atleast_1d(self.param_array)[0]*1 + import ties_and_remappings + t = ties_and_remappings.Tie(value=value, name=name) + + #add the new tie object to the global index + self._highest_parent_.ties[name] = t + self._highest_parent_.add_parameter(t) + + #constrain the tie as we were constrained + if len(old_const)==1: + t.constrain(old_const[0]) + + + self.constraints.add(t, self._raveled_index()) + t.add_tied_parameter(self) def constrain(self, transform, warning=True, trigger_parent=True): """ @@ -784,6 +813,7 @@ class Parameterizable(OptimizationHandlable): self._param_array_ = None self._added_names_ = set() self.__visited = False # for traversing in reverse order we need to know if we were here already + self.ties = {} @property def param_array(self): From fa6b5343cc649bac10fefba7f72df7087ecb1b9c Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Mon, 23 Jun 2014 14:37:11 +0100 Subject: [PATCH 30/55] merge ties branch into psi2 --- .../parameterization/ties_and_remappings.py | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 GPy/core/parameterization/ties_and_remappings.py diff --git a/GPy/core/parameterization/ties_and_remappings.py b/GPy/core/parameterization/ties_and_remappings.py new file mode 100644 index 00000000..75b46a95 --- /dev/null +++ b/GPy/core/parameterization/ties_and_remappings.py @@ -0,0 +1,80 @@ +# Copyright (c) 2014, James Hensman, Max Zwiessele +# Licensed under the BSD 3-clause license (see LICENSE.txt) + +import numpy as np +from parameterized import Parameterized +from param import Param + +class Remapping(Parameterized): + def mapping(self): + """ + The return value of this function gives the values which the re-mapped + parameters should take. Implement in sub-classes. + """ + raise NotImplementedError + + def callback(self): + raise NotImplementedError + + def __str__(self): + return self.name + + def parameters_changed(self): + #ensure all out parameters have the correct value, as specified by our mapping + index = self._highest_parent_.constraints[self] + self._highest_parent_.param_array[index] = self.mapping() + [p.notify_observers(which=self) for p in self.tied_parameters] + +class Fix(Remapping): + pass + + + + +class Tie(Remapping): + def __init__(self, value, name): + super(Tie, self).__init__(name) + self.tied_parameters = [] + self.value = Param('val', value) + self.add_parameter(self.value) + + def add_tied_parameter(self, p): + self.tied_parameters.append(p) + p.add_observer(self, self.callback) + self.parameters_changed() + + def callback(self, param=None, which=None): + """ + This gets called whenever any of the tied parameters changes. we spend + considerable effort working out whhat has changed ant to what value. + Then we store that value in self.value, and broadcast it everywhere + with parameters_changed. + """ + if which is self:return + index = self._highest_parent_.constraints[self] + if len(index)==0: + return # nothing to tie together, this tie exists without any tied parameters + vals = self._highest_parent_.param_array[index] + uvals = np.unique(vals) + if len(uvals)==1: + #all of the tied things are at the same value + self.value[...] = uvals[0] + elif len(uvals)==2: + #only *one* of the tied things has changed. it must be different to self.value + newval = uvals[uvals != self.value*1] + self.value[...] = newval + else: + #more than one of the tied things changed. panic. + raise ValueError, "something is wrong with the tieing" + + def mapping(self): + return self.value + + def collate_gradient(self): + index = self._highest_parent_.constraints[self] + self.value.gradient = np.sum(self._highest_parent_.gradient[index]) + + + + + From 567612b3a9b864738c9b56bd109cc819324c58fd Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Mon, 23 Jun 2014 19:39:35 +0100 Subject: [PATCH 31/55] tie framework works roughly --- GPy/core/parameterization/parameter_core.py | 49 +++++++++++-------- .../parameterization/ties_and_remappings.py | 4 ++ 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/GPy/core/parameterization/parameter_core.py b/GPy/core/parameterization/parameter_core.py index 1f8b347d..b99ae25a 100644 --- a/GPy/core/parameterization/parameter_core.py +++ b/GPy/core/parameterization/parameter_core.py @@ -423,12 +423,15 @@ class Indexable(Nameable, Observable): if np.all(self._fixes_): self._fixes_ = None # ==UNFIXED def _connect_fixes(self): - fixed_indices = self.constraints[__fixed__] - if fixed_indices.size > 0: - self._ensure_fixes() - self._fixes_[fixed_indices] = FIXED - else: - self._fixes_ = None + from ties_and_remappings import Tie + self._ensure_fixes() +# for c, ind in self.constraints.iteritems(): +# if c == __fixed__ or isinstance(c,Tie): +# self._fixes_[ind] = FIXED + [np.put(self._fixes_, ind, FIXED) for c, ind in self.constraints.iteritems() + if c == __fixed__ or isinstance(c,Tie)] + if np.all(self._fixes_): self._fixes_ = None + if self.constraints[__fixed__]==0: del self.constraints[__fixed__] #=========================================================================== @@ -501,29 +504,25 @@ class Indexable(Nameable, Observable): old_const = self.constraints.properties()[:] self.unconstrain() - #set these parameters to be 'fixed' as in, not optimized - self._highest_parent_._set_fixed(self, self._raveled_index()) - #see if a tie exists with that name if name in self._highest_parent_.ties: t = self._highest_parent_.ties[name] else: #create a tie object value = np.atleast_1d(self.param_array)[0]*1 - import ties_and_remappings - t = ties_and_remappings.Tie(value=value, name=name) + from ties_and_remappings import Tie + t = Tie(value=value, name=name) #add the new tie object to the global index self._highest_parent_.ties[name] = t self._highest_parent_.add_parameter(t) - #constrain the tie as we were constrained if len(old_const)==1: t.constrain(old_const[0]) - - + self.constraints.add(t, self._raveled_index()) t.add_tied_parameter(self) + self._highest_parent_._connect_fixes() def constrain(self, transform, warning=True, trigger_parent=True): """ @@ -649,10 +648,12 @@ class OptimizationHandlable(Indexable): def _get_params_transformed(self): # transformed parameters (apply un-transformation rules) p = self.param_array.copy() - [np.put(p, ind, c.finv(p[ind])) for c, ind in self.constraints.iteritems() if c != __fixed__] + from ties_and_remappings import Tie + [np.put(p, ind, c.finv(p[ind])) for c, ind in self.constraints.iteritems() if c != __fixed__ and not isinstance(c,Tie)] if self.has_parent() and self.constraints[__fixed__].size != 0: fixes = np.ones(self.size).astype(bool) - fixes[self.constraints[__fixed__]] = FIXED + [np.put(fixes,ind,FIXED) for c, ind in self.constraints.iteritems() + if c == __fixed__ or isinstance(c,Tie)] return p[fixes] elif self._has_fixes(): return p[self._fixes_] @@ -664,15 +665,21 @@ class OptimizationHandlable(Indexable): This means, the optimizer sees p, whereas the model sees transformed(p), such that, the parameters the model sees are in the right domain. """ + from ties_and_remappings import Tie if not(p is self.param_array): if self.has_parent() and self.constraints[__fixed__].size != 0: fixes = np.ones(self.size).astype(bool) - fixes[self.constraints[__fixed__]] = FIXED +# fixes[self.constraints[__fixed__]] = FIXED + for c, ind in self.constraints.iteritems(): + if c == __fixed__ or isinstance(c,Tie): + fixes[ind] = FIXED self.param_array.flat[fixes] = p elif self._has_fixes(): self.param_array.flat[self._fixes_] = p else: self.param_array.flat = p - [np.put(self.param_array, ind, c.f(self.param_array.flat[ind])) - for c, ind in self.constraints.iteritems() if c != __fixed__] + [np.put(self.param_array, ind, c.f(self.param_array.flat[ind])) + for c, ind in self.constraints.iteritems() if c != __fixed__ and not isinstance(c,Tie)] + [np.put(self.param_array, ind, c.val) + for c, ind in self.constraints.iteritems() if isinstance(c,Tie)] self._trigger_params_changed() def _trigger_params_changed(self, trigger_parent=True): @@ -699,7 +706,9 @@ class OptimizationHandlable(Indexable): """ if self.has_parent(): return g - [np.put(g, i, g[i] * c.gradfactor(self.param_array[i])) for c, i in self.constraints.iteritems() if c != __fixed__] + from ties_and_remappings import Tie + [np.put(g, self._raveled_index_for(c.val), g[i].sum()) for c, i in self.constraints.iteritems() if isinstance(c,Tie)] + [np.put(g, i, g[i] * c.gradfactor(self.param_array[i])) for c, i in self.constraints.iteritems() if c != __fixed__ and not isinstance(c,Tie)] if self._has_fixes(): return g[self._fixes_] return g diff --git a/GPy/core/parameterization/ties_and_remappings.py b/GPy/core/parameterization/ties_and_remappings.py index 75b46a95..a4cf4e6c 100644 --- a/GPy/core/parameterization/ties_and_remappings.py +++ b/GPy/core/parameterization/ties_and_remappings.py @@ -54,6 +54,7 @@ class Tie(Remapping): index = self._highest_parent_.constraints[self] if len(index)==0: return # nothing to tie together, this tie exists without any tied parameters + self.value.gradient[:] = self._highest_parent_.gradient[index].sum() vals = self._highest_parent_.param_array[index] uvals = np.unique(vals) if len(uvals)==1: @@ -66,6 +67,9 @@ class Tie(Remapping): else: #more than one of the tied things changed. panic. raise ValueError, "something is wrong with the tieing" + def parameters_changed(self): + super(Tie,self).parameters_changed() + self.value.gradient[:] = self._highest_parent_.gradient[self._highest_parent_.constraints[self]].sum() def mapping(self): return self.value From 3f36a245d1d45bf6a8d573973173db602742f32e Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Tue, 24 Jun 2014 12:49:04 +0100 Subject: [PATCH 32/55] fix the problem of multiple ties on the same param array object --- GPy/core/parameterization/parameter_core.py | 7 ++++--- GPy/core/parameterization/ties_and_remappings.py | 8 +++++--- GPy/core/parameterization/variational.py | 7 +------ GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py | 6 +++--- GPy/models/ss_gplvm.py | 14 +++++--------- 5 files changed, 18 insertions(+), 24 deletions(-) diff --git a/GPy/core/parameterization/parameter_core.py b/GPy/core/parameterization/parameter_core.py index b99ae25a..e0dadf18 100644 --- a/GPy/core/parameterization/parameter_core.py +++ b/GPy/core/parameterization/parameter_core.py @@ -500,8 +500,9 @@ class Indexable(Nameable, Observable): #=========================================================================== def tie(self, name): + from ties_and_remappings import Tie #remove any constraints - old_const = self.constraints.properties()[:] + old_const = [c for c in self.constraints.properties() if not isinstance(c,Tie)] self.unconstrain() #see if a tie exists with that name @@ -510,14 +511,14 @@ class Indexable(Nameable, Observable): else: #create a tie object value = np.atleast_1d(self.param_array)[0]*1 - from ties_and_remappings import Tie t = Tie(value=value, name=name) #add the new tie object to the global index self._highest_parent_.ties[name] = t self._highest_parent_.add_parameter(t) + #constrain the tie as we were constrained - if len(old_const)==1: + if len(old_const)>0: t.constrain(old_const[0]) self.constraints.add(t, self._raveled_index()) diff --git a/GPy/core/parameterization/ties_and_remappings.py b/GPy/core/parameterization/ties_and_remappings.py index a4cf4e6c..61c40a8e 100644 --- a/GPy/core/parameterization/ties_and_remappings.py +++ b/GPy/core/parameterization/ties_and_remappings.py @@ -46,7 +46,7 @@ class Tie(Remapping): def callback(self, param=None, which=None): """ This gets called whenever any of the tied parameters changes. we spend - considerable effort working out whhat has changed ant to what value. + considerable effort working out what has changed and to what value. Then we store that value in self.value, and broadcast it everywhere with parameters_changed. """ @@ -54,11 +54,13 @@ class Tie(Remapping): index = self._highest_parent_.constraints[self] if len(index)==0: return # nothing to tie together, this tie exists without any tied parameters - self.value.gradient[:] = self._highest_parent_.gradient[index].sum() + self.collate_gradient() vals = self._highest_parent_.param_array[index] uvals = np.unique(vals) if len(uvals)==1: #all of the tied things are at the same value + if (self.value==uvals[0]).all(): + return # DO NOT DO ANY CHANGES IF THE TIED PART IS NOT CHANGED! self.value[...] = uvals[0] elif len(uvals)==2: #only *one* of the tied things has changed. it must be different to self.value @@ -69,7 +71,7 @@ class Tie(Remapping): raise ValueError, "something is wrong with the tieing" def parameters_changed(self): super(Tie,self).parameters_changed() - self.value.gradient[:] = self._highest_parent_.gradient[self._highest_parent_.constraints[self]].sum() + self.collate_gradient() def mapping(self): return self.value diff --git a/GPy/core/parameterization/variational.py b/GPy/core/parameterization/variational.py index b9811af8..e70b9b71 100644 --- a/GPy/core/parameterization/variational.py +++ b/GPy/core/parameterization/variational.py @@ -40,7 +40,6 @@ class SpikeAndSlabPrior(VariationalPrior): self.pi = Param('pi', pi, Logistic(1e-10,1.-1e-10)) self.variance = Param('variance',variance) self.add_parameters(self.pi) - self.group_spike_prob = False def KL_divergence(self, variational_posterior): mu = variational_posterior.mean @@ -56,11 +55,7 @@ class SpikeAndSlabPrior(VariationalPrior): S = variational_posterior.variance gamma = variational_posterior.binary_prob - if self.group_spike_prob: - gamma_grad = np.log((1-self.pi)/self.pi*gamma/(1.-gamma))+(np.square(mu)+S-np.log(S)-1.)/2. - gamma.gradient -= gamma_grad.mean(axis=0) - else: - gamma.gradient -= np.log((1-self.pi)/self.pi*gamma/(1.-gamma))+(np.square(mu)+S-np.log(S)-1.)/2. + gamma.gradient -= np.log((1-self.pi)/self.pi*gamma/(1.-gamma))+(np.square(mu)+S-np.log(S)-1.)/2. mu.gradient -= gamma*mu S.gradient -= (1. - (1. / (S))) * gamma /2. self.pi.gradient = (gamma/self.pi - (1.-gamma)/(1.-self.pi)).sum(axis=0) diff --git a/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py index 43fffcd1..92387945 100644 --- a/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py +++ b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py @@ -242,14 +242,14 @@ gpu_code = """ class PSICOMP_RBF_GPU(PSICOMP_RBF): - def __init__(self, GPU_direct=False): + def __init__(self, threadnum=128, blocknum=15, GPU_direct=False): assert gpu_init.initSuccess, "GPU initialization failed!" self.GPU_direct = GPU_direct self.cublas_handle = gpu_init.cublas_handle self.gpuCache = None - self.threadnum = 128 - self.blocknum = 15 + self.threadnum = threadnum + self.blocknum = blocknum module = SourceModule("#define THREADNUM "+str(self.threadnum)+"\n"+gpu_code) self.g_psi1computations = module.get_function('psi1computations') self.g_psi1computations.prepare('PPdPPPPiii') diff --git a/GPy/models/ss_gplvm.py b/GPy/models/ss_gplvm.py index ab32e462..fa469b3b 100644 --- a/GPy/models/ss_gplvm.py +++ b/GPy/models/ss_gplvm.py @@ -44,15 +44,12 @@ class SSGPLVM(SparseGP): if X_variance is None: # The variance of the variational approximation (S) X_variance = np.random.uniform(0,.1,X.shape) - gamma = np.empty_like(X, order='F') # The posterior probabilities of the binary variable in the variational approximation + gamma = np.empty_like(X) # The posterior probabilities of the binary variable in the variational approximation gamma[:] = 0.5 + 0.1 * np.random.randn(X.shape[0], input_dim) gamma[gamma>1.-1e-9] = 1.-1e-9 gamma[gamma<1e-9] = 1e-9 gamma[:] = 0.5 - - if group_spike: - gamma[:] = gamma[:,0] - + if Z is None: Z = np.random.permutation(X.copy())[:num_inducing] assert Z.shape[1] == X.shape[1] @@ -73,14 +70,13 @@ class SSGPLVM(SparseGP): X = SpikeAndSlabPosterior(X, X_variance, gamma) - if group_spike: - kernel.group_spike_prob = True - self.variational_prior.group_spike_prob = True - SparseGP.__init__(self, X, Y, Z, kernel, likelihood, inference_method, name, **kwargs) self.add_parameter(self.X, index=0) self.add_parameter(self.variational_prior) + if self.group_spike: + [self.X.gamma[:,i].tie('tieGamma'+str(i)) for i in xrange(self.X.gamma.shape[1])] # Tie columns together + if mpi_comm != None: from ..util.mpi import divide_data Y_start, Y_end, Y_list = divide_data(Y.shape[0], mpi_comm) From 9c6bfae0b94f9a3bf94a2af616412168a9b44edd Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Tue, 24 Jun 2014 15:10:53 +0100 Subject: [PATCH 33/55] RBF for SSGPLVM gpu implemented --- GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py | 1 - GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py | 926 +++++++++----------- GPy/kern/_src/rbf.py | 4 - GPy/models/ss_gplvm.py | 7 +- 4 files changed, 435 insertions(+), 503 deletions(-) diff --git a/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py index 92387945..d8fec65b 100644 --- a/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py +++ b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py @@ -112,7 +112,6 @@ gpu_code = """ double Snq = S[IDX_NQ(n,q)]; double lq = l[q]*l[q]; log_psi2_n += dZ*dZ/(-4.*lq)-muZhat*muZhat/(2.*Snq+lq) + log_denom2[IDX_NQ(n,q)]/(-2.); - //log_psi2_n += log(2.*Snq/lq+1)/(-2.); } double exp_psi2_n = exp(log_psi2_n); psi2n[IDX_NMM(n,m1,m2)] = var*var*exp_psi2_n; diff --git a/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py b/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py index 8f845d32..989767c2 100644 --- a/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py +++ b/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py @@ -1,538 +1,476 @@ -# Copyright (c) 2012, GPy authors (see AUTHORS.txt). -# Licensed under the BSD 3-clause license (see LICENSE.txt) """ -The package for the psi statistics computation on GPU +The module for psi-statistics for RBF kernel for Spike-and-Slab GPLVM """ import numpy as np -from GPy.util.caching import Cache_this - +from ....util.caching import Cache_this +from . import PSICOMP_RBF from ....util import gpu_init try: import pycuda.gpuarray as gpuarray - from scikits.cuda import cublas - from pycuda.reduction import ReductionKernel - from pycuda.elementwise import ElementwiseKernel - from ....util import linalg_gpu - - - # The kernel form computing psi1 het_noise - comp_psi1 = ElementwiseKernel( - "double *psi1, double var, double *l, double *Z, double *mu, double *S, double *logGamma, double *log1Gamma, double *logpsi1denom, int N, int M, int Q", - "psi1[i] = comp_psi1_element(var, l, Z, mu, S, logGamma, log1Gamma, logpsi1denom, N, M, Q, i)", - "comp_psi1", - preamble=""" - #define IDX_NMQ(n,m,q) ((q*M+m)*N+n) - #define IDX_NQ(n,q) (q*N+n) - #define IDX_MQ(m,q) (q*M+m) - #define LOGEXPSUM(a,b) (a>=b?a+log(1.0+exp(b-a)):b+log(1.0+exp(a-b))) - - __device__ double comp_psi1_element(double var, double *l, double *Z, double *mu, double *S, double *logGamma, double *log1Gamma, double *logpsi1denom, int N, int M, int Q, int idx) - { - int n = idx%N; - int m = idx/N; - double psi1_exp=0; - for(int q=0;q=b?a+log(1.0+exp(b-a)):b+log(1.0+exp(a-b))) - - __device__ double comp_psi2_element(double var, double *l, double *Z, double *mu, double *S, double *logGamma, double *log1Gamma, double *logpsi2denom, int N, int M, int Q, int idx) - { - // psi2 (n,m1,m2) - int m2 = idx/M; - int m1 = idx%M; - - double psi2=0; - for(int n=0;n=b?a+log(1.0+exp(b-a)):b+log(1.0+exp(a-b))) - - __device__ double comp_dpsi1_dvar_element(double *psi1_neq, double *psi1exp1, double *psi1exp2, double *l, double *Z, double *mu, double *S, double *logGamma, double *log1Gamma, double *logpsi1denom, int N, int M, int Q, int idx) - { - int n = idx%N; - int m = idx/N; - - double psi1_sum = 0; - for(int q=0;q=b?a+log(1.0+exp(b-a)):b+log(1.0+exp(a-b))) - - __device__ double comp_dpsi2_dvar_element(double *psi2_neq, double *psi2exp1, double *psi2exp2, double var, double *l, double *Z, double *mu, double *S, double *logGamma, double *log1Gamma, double *logpsi2denom, int N, int M, int Q, int idx) - { - // psi2 (n,m1,m2) - int m2 = idx/(M*N); - int m1 = (idx%(M*N))/N; - int n = idx%N; - - double psi2_sum=0; - for(int q=0;q= blockDim.x) { + for(int i=blockDim.x+threadIdx.x; i=1;s=s/2) { + if(threadIdx.x < s) {array[threadIdx.x] += array[s+threadIdx.x];} + __syncthreads(); + } + } + + __global__ void compDenom(double *log_denom1, double *log_denom2, double *log_gamma, double*log_gamma1, double *gamma, double *l, double *S, int N, int Q) + { + int n_start, n_end; + divide_data(N, gridDim.x, blockIdx.x, &n_start, &n_end); + + for(int i=n_start*Q+threadIdx.x; iexp2)?exp1+log1p(exp(exp2-exp1)):exp2+log1p(exp(exp1-exp2)); + } + psi1[IDX_NM(n,m)] = var*exp(log_psi1); + } + } + } + + __global__ void psi2computations(double *psi2, double *psi2n, double *log_denom2, double *log_gamma, double*log_gamma1, double var, double *l, double *Z, double *mu, double *S, int N, int M, int Q) + { + int psi2_idx_start, psi2_idx_end; + __shared__ double psi2_local[THREADNUM]; + divide_data((M+1)*M/2, gridDim.x, blockIdx.x, &psi2_idx_start, &psi2_idx_end); + + for(int psi2_idx=psi2_idx_start; psi2_idxexp2)?exp1+log1p(exp(exp2-exp1)):exp2+log1p(exp(exp1-exp2)); + } + double exp_psi2_n = exp(log_psi2_n); + psi2n[IDX_NMM(n,m1,m2)] = var*var*exp_psi2_n; + if(m1!=m2) { psi2n[IDX_NMM(n,m2,m1)] = var*var*exp_psi2_n;} + psi2_local[threadIdx.x] += exp_psi2_n; + } + __syncthreads(); + reduce_sum(psi2_local, THREADNUM); + if(threadIdx.x==0) { + psi2[IDX_MM(m1,m2)] = var*var*psi2_local[0]; + if(m1!=m2) { psi2[IDX_MM(m2,m1)] = var*var*psi2_local[0]; } + } + __syncthreads(); + } + } + + __global__ void psi1compDer(double *dvar, double *dl, double *dZ, double *dmu, double *dS, double *dgamma, double *dL_dpsi1, double *psi1, double *log_denom1, double *log_gamma, double*log_gamma1, double var, double *l, double *Z, double *mu, double *S, double *gamma, int N, int M, int Q) + { + int m_start, m_end; + __shared__ double g_local[THREADNUM]; + divide_data(M, gridDim.x, blockIdx.x, &m_start, &m_end); + int P = int(ceil(double(N)/THREADNUM)); + + double dvar_local = 0; + for(int q=0;qexp2) { + d_exp1 = 1.; + d_exp2 = exp(exp2-exp1); + } else { + d_exp1 = exp(exp1-exp2); + d_exp2 = 1.; + } + double exp_sum = d_exp1+d_exp2; + + dmu_local += lpsi1*Zmu*d_exp1/(denom*exp_sum); + dS_local += lpsi1*(Zmu2_denom-1.)*d_exp1/(denom*exp_sum); + dgamma_local += lpsi1*(d_exp1/gnq-d_exp2/(1.-gnq))/exp_sum; + dl_local += lpsi1*((Zmu2_denom+Snq/lq)/denom*d_exp1+Zmq*Zmq/(lq*lq)*d_exp2)/(2.*exp_sum); + g_local[threadIdx.x] = lpsi1*(-Zmu/denom*d_exp1-Zmq/lq*d_exp2)/exp_sum; + } + __syncthreads(); + reduce_sum(g_local, pexp2) { + d_exp1 = 1.; + d_exp2 = exp(exp2-exp1); + } else { + d_exp1 = exp(exp1-exp2); + d_exp2 = 1.; + } + double exp_sum = d_exp1+d_exp2; + + dmu_local += lpsi2*muZhat/denom*d_exp1/exp_sum; + dS_local += lpsi2*(2.*muZhat2_denom-1.)/denom*d_exp1/exp_sum; + dgamma_local += lpsi2*(d_exp1/gnq-d_exp2/(1.-gnq))/exp_sum; + dl_local += lpsi2*(((Snq/lq+muZhat2_denom)/denom+dZ*dZ/(4.*lq*lq))*d_exp1+Z2/(2.*lq*lq)*d_exp2)/exp_sum; + g_local[threadIdx.x] += 2.*lpsi2*((muZhat/denom-dZ/(2*lq))*d_exp1-Zm1q/lq*d_exp2)/exp_sum; + } + } + __syncthreads(); + reduce_sum(g_local, p reallocate - self._releaseMemory() - - if self.gpuCacheAll == None: - self.gpuCacheAll = { + self.threadnum = threadnum + self.blocknum = blocknum + module = SourceModule("#define THREADNUM "+str(self.threadnum)+"\n"+gpu_code) + self.g_psi1computations = module.get_function('psi1computations') + self.g_psi1computations.prepare('PPPPdPPPPiii') + self.g_psi2computations = module.get_function('psi2computations') + self.g_psi2computations.prepare('PPPPPdPPPPiii') + self.g_psi1compDer = module.get_function('psi1compDer') + self.g_psi1compDer.prepare('PPPPPPPPPPPdPPPPPiii') + self.g_psi2compDer = module.get_function('psi2compDer') + self.g_psi2compDer.prepare('PPPPPPPPPPPdPPPPPiii') + self.g_compDenom = module.get_function('compDenom') + self.g_compDenom.prepare('PPPPPPPii') + + def _initGPUCache(self, N, M, Q): + if self.gpuCache == None: + self.gpuCache = { 'l_gpu' :gpuarray.empty((Q,),np.float64,order='F'), 'Z_gpu' :gpuarray.empty((M,Q),np.float64,order='F'), 'mu_gpu' :gpuarray.empty((N,Q),np.float64,order='F'), 'S_gpu' :gpuarray.empty((N,Q),np.float64,order='F'), 'gamma_gpu' :gpuarray.empty((N,Q),np.float64,order='F'), - 'logGamma_gpu' :gpuarray.empty((N,Q),np.float64,order='F'), - 'log1Gamma_gpu' :gpuarray.empty((N,Q),np.float64,order='F'), - 'logpsi1denom_gpu' :gpuarray.empty((N,Q),np.float64,order='F'), - 'logpsi2denom_gpu' :gpuarray.empty((N,Q),np.float64,order='F'), - 'psi0_gpu' :gpuarray.empty((N,),np.float64,order='F'), 'psi1_gpu' :gpuarray.empty((N,M),np.float64,order='F'), 'psi2_gpu' :gpuarray.empty((M,M),np.float64,order='F'), - # derivatives psi1 - 'psi1_neq_gpu' :gpuarray.empty((N,M,Q),np.float64, order='F'), - 'psi1exp1_gpu' :gpuarray.empty((N,M,Q),np.float64, order='F'), - 'psi1exp2_gpu' :gpuarray.empty((N,M,Q),np.float64, order='F'), - 'dpsi1_dvar_gpu' :gpuarray.empty((N,M),np.float64, order='F'), - 'dpsi1_dl_gpu' :gpuarray.empty((N,M,Q),np.float64, order='F'), - 'dpsi1_dZ_gpu' :gpuarray.empty((N,M,Q),np.float64, order='F'), - 'dpsi1_dgamma_gpu' :gpuarray.empty((N,M,Q),np.float64, order='F'), - 'dpsi1_dmu_gpu' :gpuarray.empty((N,M,Q),np.float64, order='F'), - 'dpsi1_dS_gpu' :gpuarray.empty((N,M,Q),np.float64, order='F'), - # derivatives psi2 - 'psi2_neq_gpu' :gpuarray.empty((N,M,M,Q),np.float64, order='F'), - 'psi2exp1_gpu' :gpuarray.empty((N,M,M,Q),np.float64, order='F'), - 'psi2exp2_gpu' :gpuarray.empty((M,M,Q),np.float64, order='F'), - 'dpsi2_dvar_gpu' :gpuarray.empty((N,M,M),np.float64, order='F'), - 'dpsi2_dl_gpu' :gpuarray.empty((N,M,M,Q),np.float64, order='F'), - 'dpsi2_dZ_gpu' :gpuarray.empty((N,M,M,Q),np.float64, order='F'), - 'dpsi2_dgamma_gpu' :gpuarray.empty((N,M,M,Q),np.float64, order='F'), - 'dpsi2_dmu_gpu' :gpuarray.empty((N,M,M,Q),np.float64, order='F'), - 'dpsi2_dS_gpu' :gpuarray.empty((N,M,M,Q),np.float64, order='F'), - # gradients - 'grad_l_gpu' :gpuarray.empty((Q,),np.float64,order='F'), - 'grad_Z_gpu' :gpuarray.empty((M,Q),np.float64,order='F'), - 'grad_mu_gpu' :gpuarray.empty((N,Q),np.float64,order='F'), - 'grad_S_gpu' :gpuarray.empty((N,Q),np.float64,order='F'), - 'grad_gamma_gpu' :gpuarray.empty((N,Q),np.float64,order='F'), + 'psi2n_gpu' :gpuarray.empty((N,M,M),np.float64,order='F'), + 'dL_dpsi1_gpu' :gpuarray.empty((N,M),np.float64,order='F'), + 'dL_dpsi2_gpu' :gpuarray.empty((M,M),np.float64,order='F'), + 'log_denom1_gpu' :gpuarray.empty((N,Q),np.float64,order='F'), + 'log_denom2_gpu' :gpuarray.empty((N,Q),np.float64,order='F'), + 'log_gamma_gpu' :gpuarray.empty((N,Q),np.float64,order='F'), + 'log_gamma1_gpu' :gpuarray.empty((N,Q),np.float64,order='F'), + # derivatives + 'dvar_gpu' :gpuarray.empty((self.blocknum,),np.float64, order='F'), + 'dl_gpu' :gpuarray.empty((Q,self.blocknum),np.float64, order='F'), + 'dZ_gpu' :gpuarray.empty((M,Q),np.float64, order='F'), + 'dmu_gpu' :gpuarray.empty((N,Q,self.blocknum),np.float64, order='F'), + 'dS_gpu' :gpuarray.empty((N,Q,self.blocknum),np.float64, order='F'), + 'dgamma_gpu' :gpuarray.empty((N,Q,self.blocknum),np.float64, order='F'), + # grad + 'grad_l_gpu' :gpuarray.empty((Q,),np.float64, order='F'), + 'grad_mu_gpu' :gpuarray.empty((N,Q,),np.float64, order='F'), + 'grad_S_gpu' :gpuarray.empty((N,Q,),np.float64, order='F'), + 'grad_gamma_gpu' :gpuarray.empty((N,Q,),np.float64, order='F'), } - self.gpuCache = self.gpuCacheAll - elif self.gpuCacheAll['mu_gpu'].shape[0]==N: - self.gpuCache = self.gpuCacheAll else: - # remap to a smaller cache - self.gpuCache = self.gpuCacheAll.copy() - Nlist=['mu_gpu','S_gpu','gamma_gpu','logGamma_gpu','log1Gamma_gpu','logpsi1denom_gpu','logpsi2denom_gpu','psi0_gpu','psi1_gpu','psi2_gpu', - 'psi1_neq_gpu','psi1exp1_gpu','psi1exp2_gpu','dpsi1_dvar_gpu','dpsi1_dl_gpu','dpsi1_dZ_gpu','dpsi1_dgamma_gpu','dpsi1_dmu_gpu', - 'dpsi1_dS_gpu','psi2_neq_gpu','psi2exp1_gpu','dpsi2_dvar_gpu','dpsi2_dl_gpu','dpsi2_dZ_gpu','dpsi2_dgamma_gpu','dpsi2_dmu_gpu','dpsi2_dS_gpu','grad_mu_gpu','grad_S_gpu','grad_gamma_gpu',] - oldN = self.gpuCacheAll['mu_gpu'].shape[0] - for v in Nlist: - u = self.gpuCacheAll[v] - self.gpuCache[v] = u.ravel()[:u.size/oldN*N].reshape(*((N,)+u.shape[1:])) + assert N==self.gpuCache['mu_gpu'].shape[0] + assert M==self.gpuCache['Z_gpu'].shape[0] + assert Q==self.gpuCache['l_gpu'].shape[0] - def _releaseMemory(self): - if self.gpuCacheAll!=None: - [v.gpudata.free() for v in self.gpuCacheAll.values()] - self.gpuCacheAll = None - self.gpuCache = None + def sync_params(self, lengthscale, Z, mu, S, gamma): + if len(lengthscale)==1: + self.gpuCache['l_gpu'].fill(lengthscale) + else: + self.gpuCache['l_gpu'].set(np.asfortranarray(lengthscale)) + self.gpuCache['Z_gpu'].set(np.asfortranarray(Z)) + self.gpuCache['mu_gpu'].set(np.asfortranarray(mu)) + self.gpuCache['S_gpu'].set(np.asfortranarray(S)) + self.gpuCache['gamma_gpu'].set(np.asfortranarray(gamma)) + N,Q = self.gpuCache['S_gpu'].shape + # t=self.g_compDenom(self.gpuCache['log_denom1_gpu'],self.gpuCache['log_denom2_gpu'],self.gpuCache['l_gpu'],self.gpuCache['S_gpu'], np.int32(N), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True) + # print 'g_compDenom '+str(t) + self.g_compDenom.prepared_call((self.blocknum,1),(self.threadnum,1,1), self.gpuCache['log_denom1_gpu'].gpudata,self.gpuCache['log_denom2_gpu'].gpudata,self.gpuCache['log_gamma_gpu'].gpudata,self.gpuCache['log_gamma1_gpu'].gpudata,self.gpuCache['gamma_gpu'].gpudata,self.gpuCache['l_gpu'].gpudata,self.gpuCache['S_gpu'].gpudata, np.int32(N), np.int32(Q)) + + def reset_derivative(self): + self.gpuCache['dvar_gpu'].fill(0.) + self.gpuCache['dl_gpu'].fill(0.) + self.gpuCache['dZ_gpu'].fill(0.) + self.gpuCache['dmu_gpu'].fill(0.) + self.gpuCache['dS_gpu'].fill(0.) + self.gpuCache['dgamma_gpu'].fill(0.) + self.gpuCache['grad_l_gpu'].fill(0.) + self.gpuCache['grad_mu_gpu'].fill(0.) + self.gpuCache['grad_S_gpu'].fill(0.) + self.gpuCache['grad_gamma_gpu'].fill(0.) - def estimateMemoryOccupation(self, N, M, Q): - """ - Estimate the best batch size. - N - the number of total datapoints - M - the number of inducing points - Q - the number of hidden (input) dimensions - return: the constant memory size, the memory occupation of batchsize=1 - unit: GB - """ - return (2.*Q+2.*M*Q+M*M*Q)*8./1024./1024./1024., (1.+2.*M+10.*Q+2.*M*M+8.*M*Q+7.*M*M*Q)*8./1024./1024./1024. + def get_dimensions(self, Z, variational_posterior): + return variational_posterior.mean.shape[0], Z.shape[0], Z.shape[1] - @Cache_this(limit=1,ignore_args=(0,)) - def psicomputations(self, variance, lengthscale, Z, mu, S, gamma): - """Compute Psi statitsitcs""" - if isinstance(lengthscale, np.ndarray) and len(lengthscale)>1: - ARD = True - else: - ARD = False - - N = mu.shape[0] - M = Z.shape[0] - Q = mu.shape[1] - + @Cache_this(limit=1, ignore_args=(0,)) + def psicomputations(self, variance, lengthscale, Z, variational_posterior): + """ + Z - MxQ + mu - NxQ + S - NxQ + """ + N,M,Q = self.get_dimensions(Z, variational_posterior) self._initGPUCache(N,M,Q) - l_gpu = self.gpuCache['l_gpu'] - Z_gpu = self.gpuCache['Z_gpu'] - mu_gpu = self.gpuCache['mu_gpu'] - S_gpu = self.gpuCache['S_gpu'] - gamma_gpu = self.gpuCache['gamma_gpu'] - logGamma_gpu = self.gpuCache['logGamma_gpu'] - log1Gamma_gpu = self.gpuCache['log1Gamma_gpu'] - logpsi1denom_gpu = self.gpuCache['logpsi1denom_gpu'] - logpsi2denom_gpu = self.gpuCache['logpsi2denom_gpu'] - psi0_gpu = self.gpuCache['psi0_gpu'] + self.sync_params(lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) + psi1_gpu = self.gpuCache['psi1_gpu'] psi2_gpu = self.gpuCache['psi2_gpu'] - - if ARD: - l_gpu.set(np.asfortranarray(lengthscale**2)) - else: - l_gpu.fill(lengthscale*lengthscale) - Z_gpu.set(np.asfortranarray(Z)) - mu_gpu.set(np.asfortranarray(mu)) - S_gpu.set(np.asfortranarray(S)) - gamma_gpu.set(np.asfortranarray(gamma)) - linalg_gpu.log(gamma_gpu,logGamma_gpu) - linalg_gpu.logOne(gamma_gpu,log1Gamma_gpu) - comp_logpsidenom(logpsi1denom_gpu, S_gpu,l_gpu,1.0,N) - comp_logpsidenom(logpsi2denom_gpu, S_gpu,l_gpu,2.0,N) - - psi0_gpu.fill(variance) - comp_psi1(psi1_gpu, variance, l_gpu, Z_gpu, mu_gpu, S_gpu, logGamma_gpu, log1Gamma_gpu, logpsi1denom_gpu, N, M, Q) - comp_psi2(psi2_gpu, variance, l_gpu, Z_gpu, mu_gpu, S_gpu, logGamma_gpu, log1Gamma_gpu, logpsi2denom_gpu, N, M, Q) - - return psi0_gpu, psi1_gpu, psi2_gpu - - @Cache_this(limit=1,ignore_args=(0,)) - def _psiDercomputations(self, variance, lengthscale, Z, mu, S, gamma): - """Compute the derivatives w.r.t. Psi statistics""" - N, M, Q = mu.shape[0],Z.shape[0], mu.shape[1] - - self._initGPUCache(N,M,Q) + psi2n_gpu = self.gpuCache['psi2n_gpu'] l_gpu = self.gpuCache['l_gpu'] Z_gpu = self.gpuCache['Z_gpu'] mu_gpu = self.gpuCache['mu_gpu'] S_gpu = self.gpuCache['S_gpu'] gamma_gpu = self.gpuCache['gamma_gpu'] - logGamma_gpu = self.gpuCache['logGamma_gpu'] - log1Gamma_gpu = self.gpuCache['log1Gamma_gpu'] - logpsi1denom_gpu = self.gpuCache['logpsi1denom_gpu'] - logpsi2denom_gpu = self.gpuCache['logpsi2denom_gpu'] + log_denom1_gpu = self.gpuCache['log_denom1_gpu'] + log_denom2_gpu = self.gpuCache['log_denom2_gpu'] + log_gamma_gpu = self.gpuCache['log_gamma_gpu'] + log_gamma1_gpu = self.gpuCache['log_gamma1_gpu'] - psi1_neq_gpu = self.gpuCache['psi1_neq_gpu'] - psi1exp1_gpu = self.gpuCache['psi1exp1_gpu'] - psi1exp2_gpu = self.gpuCache['psi1exp2_gpu'] - dpsi1_dvar_gpu = self.gpuCache['dpsi1_dvar_gpu'] - dpsi1_dl_gpu = self.gpuCache['dpsi1_dl_gpu'] - dpsi1_dZ_gpu = self.gpuCache['dpsi1_dZ_gpu'] - dpsi1_dgamma_gpu = self.gpuCache['dpsi1_dgamma_gpu'] - dpsi1_dmu_gpu = self.gpuCache['dpsi1_dmu_gpu'] - dpsi1_dS_gpu = self.gpuCache['dpsi1_dS_gpu'] - - psi2_neq_gpu = self.gpuCache['psi2_neq_gpu'] - psi2exp1_gpu = self.gpuCache['psi2exp1_gpu'] - psi2exp2_gpu = self.gpuCache['psi2exp2_gpu'] - dpsi2_dvar_gpu = self.gpuCache['dpsi2_dvar_gpu'] - dpsi2_dl_gpu = self.gpuCache['dpsi2_dl_gpu'] - dpsi2_dZ_gpu = self.gpuCache['dpsi2_dZ_gpu'] - dpsi2_dgamma_gpu = self.gpuCache['dpsi2_dgamma_gpu'] - dpsi2_dmu_gpu = self.gpuCache['dpsi2_dmu_gpu'] - dpsi2_dS_gpu = self.gpuCache['dpsi2_dS_gpu'] - - #========================================================================================================== - # Assuming the l_gpu, Z_gpu, mu_gpu, S_gpu, gamma_gpu, logGamma_gpu, log1Gamma_gpu, - # logpsi1denom_gpu, logpsi2denom_gpu has been synchonized. - #========================================================================================================== - - # psi1 derivatives - comp_dpsi1_dvar(dpsi1_dvar_gpu, psi1_neq_gpu, psi1exp1_gpu,psi1exp2_gpu, l_gpu, Z_gpu, mu_gpu, S_gpu, logGamma_gpu, log1Gamma_gpu, logpsi1denom_gpu, N, M, Q) - comp_psi1_der(dpsi1_dl_gpu,dpsi1_dmu_gpu,dpsi1_dS_gpu,dpsi1_dgamma_gpu, dpsi1_dZ_gpu, psi1_neq_gpu,psi1exp1_gpu,psi1exp2_gpu, variance, l_gpu, Z_gpu, mu_gpu, S_gpu, gamma_gpu, N, M, Q) - - # psi2 derivatives - comp_dpsi2_dvar(dpsi2_dvar_gpu, psi2_neq_gpu, psi2exp1_gpu,psi2exp2_gpu, variance, l_gpu, Z_gpu, mu_gpu, S_gpu, logGamma_gpu, log1Gamma_gpu, logpsi2denom_gpu, N, M, Q) - comp_psi2_der(dpsi2_dl_gpu,dpsi2_dmu_gpu,dpsi2_dS_gpu,dpsi2_dgamma_gpu, dpsi2_dZ_gpu, psi2_neq_gpu,psi2exp1_gpu,psi2exp2_gpu, variance, l_gpu, Z_gpu, mu_gpu, S_gpu, gamma_gpu, N, M, Q) - - def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior): - mu = variational_posterior.mean - S = variational_posterior.variance - gamma = variational_posterior.binary_prob - self._psiDercomputations(variance, lengthscale, Z, mu, S, gamma) - N, M, Q = mu.shape[0],Z.shape[0], mu.shape[1] - - if isinstance(lengthscale, np.ndarray) and len(lengthscale)>1: - ARD = True + psi0 = np.empty((N,)) + psi0[:] = variance + self.g_psi1computations.prepared_call((self.blocknum,1),(self.threadnum,1,1),psi1_gpu.gpudata, log_denom1_gpu.gpudata, log_gamma_gpu.gpudata, log_gamma1_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata, np.int32(N), np.int32(M), np.int32(Q)) + self.g_psi2computations.prepared_call((self.blocknum,1),(self.threadnum,1,1),psi2_gpu.gpudata, psi2n_gpu.gpudata, log_denom2_gpu.gpudata, log_gamma_gpu.gpudata, log_gamma1_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata, np.int32(N), np.int32(M), np.int32(Q)) + # t = self.g_psi1computations(psi1_gpu, log_denom1_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True) + # print 'g_psi1computations '+str(t) + # t = self.g_psi2computations(psi2_gpu, psi2n_gpu, log_denom2_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True) + # print 'g_psi2computations '+str(t) + + if self.GPU_direct: + return psi0, psi1_gpu, psi2_gpu else: - ARD = False - - dpsi1_dvar_gpu = self.gpuCache['dpsi1_dvar_gpu'] - dpsi2_dvar_gpu = self.gpuCache['dpsi2_dvar_gpu'] - dpsi1_dl_gpu = self.gpuCache['dpsi1_dl_gpu'] - dpsi2_dl_gpu = self.gpuCache['dpsi2_dl_gpu'] - psi1_comb_gpu = self.gpuCache['psi1_neq_gpu'] - psi2_comb_gpu = self.gpuCache['psi2_neq_gpu'] + return psi0, psi1_gpu.get(), psi2_gpu.get() + + @Cache_this(limit=1, ignore_args=(0,1,2,3)) + def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior): + ARD = (len(lengthscale)!=1) + + N,M,Q = self.get_dimensions(Z, variational_posterior) + psi1_gpu = self.gpuCache['psi1_gpu'] + psi2n_gpu = self.gpuCache['psi2n_gpu'] + l_gpu = self.gpuCache['l_gpu'] + Z_gpu = self.gpuCache['Z_gpu'] + mu_gpu = self.gpuCache['mu_gpu'] + S_gpu = self.gpuCache['S_gpu'] + gamma_gpu = self.gpuCache['gamma_gpu'] + dvar_gpu = self.gpuCache['dvar_gpu'] + dl_gpu = self.gpuCache['dl_gpu'] + dZ_gpu = self.gpuCache['dZ_gpu'] + dmu_gpu = self.gpuCache['dmu_gpu'] + dS_gpu = self.gpuCache['dS_gpu'] + dgamma_gpu = self.gpuCache['dgamma_gpu'] grad_l_gpu = self.gpuCache['grad_l_gpu'] - - # variance - variance.gradient = gpuarray.sum(dL_dpsi0).get() \ - + cublas.cublasDdot(self.cublas_handle, dL_dpsi1.size, dL_dpsi1.gpudata, 1, dpsi1_dvar_gpu.gpudata, 1) \ - + cublas.cublasDdot(self.cublas_handle, dL_dpsi2.size, dL_dpsi2.gpudata, 1, dpsi2_dvar_gpu.gpudata, 1) - - # lengscale - if ARD: - grad_l_gpu.fill(0.) - linalg_gpu.mul_bcast(psi1_comb_gpu, dL_dpsi1, dpsi1_dl_gpu, dL_dpsi1.size) - linalg_gpu.sum_axis(grad_l_gpu, psi1_comb_gpu, 1, N*M) - linalg_gpu.mul_bcast(psi2_comb_gpu, dL_dpsi2, dpsi2_dl_gpu, dL_dpsi2.size) - linalg_gpu.sum_axis(grad_l_gpu, psi2_comb_gpu, 1, N*M*M) - lengthscale.gradient = grad_l_gpu.get() - else: - linalg_gpu.mul_bcast(psi1_comb_gpu, dL_dpsi1, dpsi1_dl_gpu, dL_dpsi1.size) - linalg_gpu.mul_bcast(psi2_comb_gpu, dL_dpsi2, dpsi2_dl_gpu, dL_dpsi2.size) - lengthscale.gradient = gpuarray.sum(psi1_comb_gpu).get() + gpuarray.sum(psi2_comb_gpu).get() - - def gradients_Z_expectations(self, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior): - mu = variational_posterior.mean - S = variational_posterior.variance - gamma = variational_posterior.binary_prob - self._psiDercomputations(variance, lengthscale, Z, mu, S, gamma) - N, M, Q = mu.shape[0],Z.shape[0], mu.shape[1] - - dpsi1_dZ_gpu = self.gpuCache['dpsi1_dZ_gpu'] - dpsi2_dZ_gpu = self.gpuCache['dpsi2_dZ_gpu'] - psi1_comb_gpu = self.gpuCache['psi1_neq_gpu'] - psi2_comb_gpu = self.gpuCache['psi2_neq_gpu'] - grad_Z_gpu = self.gpuCache['grad_Z_gpu'] - - grad_Z_gpu.fill(0.) - linalg_gpu.mul_bcast(psi1_comb_gpu, dL_dpsi1, dpsi1_dZ_gpu, dL_dpsi1.size) - linalg_gpu.sum_axis(grad_Z_gpu, psi1_comb_gpu, 1, N) - linalg_gpu.mul_bcast(psi2_comb_gpu, dL_dpsi2, dpsi2_dZ_gpu, dL_dpsi2.size) - linalg_gpu.sum_axis(grad_Z_gpu, psi2_comb_gpu, 1, N*M) - return grad_Z_gpu.get() - - def gradients_qX_expectations(self, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior): - mu = variational_posterior.mean - S = variational_posterior.variance - gamma = variational_posterior.binary_prob - self._psiDercomputations(variance, lengthscale, Z, mu, S, gamma) - N, M, Q = mu.shape[0],Z.shape[0], mu.shape[1] - - dpsi1_dmu_gpu = self.gpuCache['dpsi1_dmu_gpu'] - dpsi2_dmu_gpu = self.gpuCache['dpsi2_dmu_gpu'] - dpsi1_dS_gpu = self.gpuCache['dpsi1_dS_gpu'] - dpsi2_dS_gpu = self.gpuCache['dpsi2_dS_gpu'] - dpsi1_dgamma_gpu = self.gpuCache['dpsi1_dgamma_gpu'] - dpsi2_dgamma_gpu = self.gpuCache['dpsi2_dgamma_gpu'] - psi1_comb_gpu = self.gpuCache['psi1_neq_gpu'] - psi2_comb_gpu = self.gpuCache['psi2_neq_gpu'] grad_mu_gpu = self.gpuCache['grad_mu_gpu'] grad_S_gpu = self.gpuCache['grad_S_gpu'] grad_gamma_gpu = self.gpuCache['grad_gamma_gpu'] + log_denom1_gpu = self.gpuCache['log_denom1_gpu'] + log_denom2_gpu = self.gpuCache['log_denom2_gpu'] + log_gamma_gpu = self.gpuCache['log_gamma_gpu'] + log_gamma1_gpu = self.gpuCache['log_gamma1_gpu'] - # mu gradients - grad_mu_gpu.fill(0.) - linalg_gpu.mul_bcast(psi1_comb_gpu, dL_dpsi1, dpsi1_dmu_gpu, dL_dpsi1.size) - linalg_gpu.sum_axis(grad_mu_gpu, psi1_comb_gpu, N, M) - linalg_gpu.mul_bcast(psi2_comb_gpu, dL_dpsi2, dpsi2_dmu_gpu, dL_dpsi2.size) - linalg_gpu.sum_axis(grad_mu_gpu, psi2_comb_gpu, N, M*M) + if self.GPU_direct: + dL_dpsi1_gpu = dL_dpsi1 + dL_dpsi2_gpu = dL_dpsi2 + dL_dpsi0_sum = gpuarray.sum(dL_dpsi0).get() + else: + dL_dpsi1_gpu = self.gpuCache['dL_dpsi1_gpu'] + dL_dpsi2_gpu = self.gpuCache['dL_dpsi2_gpu'] + dL_dpsi1_gpu.set(np.asfortranarray(dL_dpsi1)) + dL_dpsi2_gpu.set(np.asfortranarray(dL_dpsi2)) + dL_dpsi0_sum = dL_dpsi0.sum() - # S gradients - grad_S_gpu.fill(0.) - linalg_gpu.mul_bcast(psi1_comb_gpu, dL_dpsi1, dpsi1_dS_gpu, dL_dpsi1.size) - linalg_gpu.sum_axis(grad_S_gpu, psi1_comb_gpu, N, M) - linalg_gpu.mul_bcast(psi2_comb_gpu, dL_dpsi2, dpsi2_dS_gpu, dL_dpsi2.size) - linalg_gpu.sum_axis(grad_S_gpu, psi2_comb_gpu, N, M*M) + self.reset_derivative() + # t=self.g_psi1compDer(dvar_gpu,dl_gpu,dZ_gpu,dmu_gpu,dS_gpu,dL_dpsi1_gpu,psi1_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True) + # print 'g_psi1compDer '+str(t) + # t=self.g_psi2compDer(dvar_gpu,dl_gpu,dZ_gpu,dmu_gpu,dS_gpu,dL_dpsi2_gpu,psi2n_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True) + # print 'g_psi2compDer '+str(t) + self.g_psi1compDer.prepared_call((self.blocknum,1),(self.threadnum,1,1),dvar_gpu.gpudata,dl_gpu.gpudata,dZ_gpu.gpudata,dmu_gpu.gpudata,dS_gpu.gpudata,dgamma_gpu.gpudata,dL_dpsi1_gpu.gpudata,psi1_gpu.gpudata, log_denom1_gpu.gpudata, log_gamma_gpu.gpudata, log_gamma1_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata,gamma_gpu.gpudata,np.int32(N), np.int32(M), np.int32(Q)) + self.g_psi2compDer.prepared_call((self.blocknum,1),(self.threadnum,1,1),dvar_gpu.gpudata,dl_gpu.gpudata,dZ_gpu.gpudata,dmu_gpu.gpudata,dS_gpu.gpudata,dgamma_gpu.gpudata,dL_dpsi2_gpu.gpudata,psi2n_gpu.gpudata, log_denom2_gpu.gpudata, log_gamma_gpu.gpudata, log_gamma1_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata,gamma_gpu.gpudata,np.int32(N), np.int32(M), np.int32(Q)) + + dL_dvar = dL_dpsi0_sum + gpuarray.sum(dvar_gpu).get() + sum_axis(grad_mu_gpu,dmu_gpu,N*Q,self.blocknum) + dL_dmu = grad_mu_gpu.get() + sum_axis(grad_S_gpu,dS_gpu,N*Q,self.blocknum) + dL_dS = grad_S_gpu.get() + sum_axis(grad_gamma_gpu,dgamma_gpu,N*Q,self.blocknum) + dL_dgamma = grad_gamma_gpu.get() + dL_dZ = dZ_gpu.get() + if ARD: + sum_axis(grad_l_gpu,dl_gpu,Q,self.blocknum) + dL_dlengscale = grad_l_gpu.get() + else: + dL_dlengscale = gpuarray.sum(dl_gpu).get() + + return dL_dvar, dL_dlengscale, dL_dZ, dL_dmu, dL_dS, dL_dgamma + - # gamma gradients - grad_gamma_gpu.fill(0.) - linalg_gpu.mul_bcast(psi1_comb_gpu, dL_dpsi1, dpsi1_dgamma_gpu, dL_dpsi1.size) - linalg_gpu.sum_axis(grad_gamma_gpu, psi1_comb_gpu, N, M) - linalg_gpu.mul_bcast(psi2_comb_gpu, dL_dpsi2, dpsi2_dgamma_gpu, dL_dpsi2.size) - linalg_gpu.sum_axis(grad_gamma_gpu, psi2_comb_gpu, N, M*M) - - return grad_mu_gpu.get(), grad_S_gpu.get(), grad_gamma_gpu.get() diff --git a/GPy/kern/_src/rbf.py b/GPy/kern/_src/rbf.py index 4aa3ed74..90c9100b 100644 --- a/GPy/kern/_src/rbf.py +++ b/GPy/kern/_src/rbf.py @@ -3,11 +3,7 @@ import numpy as np -from scipy import weave -from ...util.misc import param_to_array from stationary import Stationary -from GPy.util.caching import Cache_this -from ...core.parameterization import variational from psi_comp import PSICOMP_RBF from psi_comp.rbf_psi_gpucomp import PSICOMP_RBF_GPU from ...util.config import * diff --git a/GPy/models/ss_gplvm.py b/GPy/models/ss_gplvm.py index fa469b3b..c23e3524 100644 --- a/GPy/models/ss_gplvm.py +++ b/GPy/models/ss_gplvm.py @@ -2,17 +2,14 @@ # Licensed under the BSD 3-clause license (see LICENSE.txt) import numpy as np -import itertools -from matplotlib import pyplot from ..core.sparse_gp import SparseGP from .. import kern from ..likelihoods import Gaussian -from ..inference.optimization import SCG -from ..util import linalg from ..core.parameterization.variational import SpikeAndSlabPrior, SpikeAndSlabPosterior from ..inference.latent_function_inference.var_dtc_parallel import update_gradients, VarDTC_minibatch from ..inference.latent_function_inference.var_dtc_gpu import VarDTC_GPU +from ..kern._src.psi_comp.ssrbf_psi_gpucomp import PSICOMP_SSRBF_GPU class SSGPLVM(SparseGP): """ @@ -62,6 +59,8 @@ class SSGPLVM(SparseGP): if kernel is None: kernel = kern.RBF(input_dim, lengthscale=fracs, ARD=True) # + kern.white(input_dim) + if kernel.useGPU: + kernel.psicomp = PSICOMP_SSRBF_GPU() if inference_method is None: inference_method = VarDTC_minibatch(mpi_comm=mpi_comm) From 525d9329ff1f0daab34cb4c2aca4732406394a58 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Tue, 24 Jun 2014 17:28:01 +0100 Subject: [PATCH 34/55] a little optimization of gpu code --- GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py b/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py index 989767c2..60fa8259 100644 --- a/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py +++ b/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py @@ -157,9 +157,10 @@ gpu_code = """ double dmu_local = 0; double dS_local = 0; double dgamma_local = 0; - double Snq,mu_nq,gnq,log_gnq,log_gnq1; + double Snq,mu_nq,gnq,log_gnq,log_gnq1,log_de; if(nexp2) { @@ -227,9 +228,10 @@ gpu_code = """ double dmu_local = 0; double dS_local = 0; double dgamma_local = 0; - double Snq,mu_nq,gnq,log_gnq,log_gnq1; + double Snq,mu_nq,gnq,log_gnq,log_gnq1,log_de; if(nexp2) { From 08ed72b2f234ed292e6628469e5c7b45b7532bb8 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Tue, 24 Jun 2014 18:01:16 +0100 Subject: [PATCH 35/55] fix the speed problem of the tie framework --- GPy/core/parameterization/ties_and_remappings.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/GPy/core/parameterization/ties_and_remappings.py b/GPy/core/parameterization/ties_and_remappings.py index 61c40a8e..b2330d59 100644 --- a/GPy/core/parameterization/ties_and_remappings.py +++ b/GPy/core/parameterization/ties_and_remappings.py @@ -70,7 +70,12 @@ class Tie(Remapping): #more than one of the tied things changed. panic. raise ValueError, "something is wrong with the tieing" def parameters_changed(self): - super(Tie,self).parameters_changed() + #ensure all out parameters have the correct value, as specified by our mapping + index = self._highest_parent_.constraints[self] + if (self._highest_parent_.param_array[index]==self.value).all(): + return # STOP TRIGGER THE UPDATE LOOP MULTIPLE TIMES!!! + self._highest_parent_.param_array[index] = self.mapping() + [p.notify_observers(which=self) for p in self.tied_parameters] self.collate_gradient() def mapping(self): From cf3380867375488945d1e2dfd0b4682bd12d9262 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Wed, 25 Jun 2014 13:57:25 +0100 Subject: [PATCH 36/55] fix the SSGPLVM with MPI --- GPy/core/parameterization/parameter_core.py | 3 --- .../parameterization/ties_and_remappings.py | 4 ++-- GPy/core/parameterization/variational.py | 3 +++ .../var_dtc_parallel.py | 6 ++--- GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py | 11 ++------- GPy/models/ss_gplvm.py | 23 +++++++++---------- GPy/util/gpu_init.py | 8 ++++++- 7 files changed, 28 insertions(+), 30 deletions(-) diff --git a/GPy/core/parameterization/parameter_core.py b/GPy/core/parameterization/parameter_core.py index e0dadf18..cdf3f534 100644 --- a/GPy/core/parameterization/parameter_core.py +++ b/GPy/core/parameterization/parameter_core.py @@ -425,9 +425,6 @@ class Indexable(Nameable, Observable): def _connect_fixes(self): from ties_and_remappings import Tie self._ensure_fixes() -# for c, ind in self.constraints.iteritems(): -# if c == __fixed__ or isinstance(c,Tie): -# self._fixes_[ind] = FIXED [np.put(self._fixes_, ind, FIXED) for c, ind in self.constraints.iteritems() if c == __fixed__ or isinstance(c,Tie)] if np.all(self._fixes_): self._fixes_ = None diff --git a/GPy/core/parameterization/ties_and_remappings.py b/GPy/core/parameterization/ties_and_remappings.py index b2330d59..da46acaa 100644 --- a/GPy/core/parameterization/ties_and_remappings.py +++ b/GPy/core/parameterization/ties_and_remappings.py @@ -59,7 +59,7 @@ class Tie(Remapping): uvals = np.unique(vals) if len(uvals)==1: #all of the tied things are at the same value - if (self.value==uvals[0]).all(): + if np.all(self.value==uvals[0]): return # DO NOT DO ANY CHANGES IF THE TIED PART IS NOT CHANGED! self.value[...] = uvals[0] elif len(uvals)==2: @@ -72,7 +72,7 @@ class Tie(Remapping): def parameters_changed(self): #ensure all out parameters have the correct value, as specified by our mapping index = self._highest_parent_.constraints[self] - if (self._highest_parent_.param_array[index]==self.value).all(): + if np.all(self._highest_parent_.param_array[index]==self.value): return # STOP TRIGGER THE UPDATE LOOP MULTIPLE TIMES!!! self._highest_parent_.param_array[index] = self.mapping() [p.notify_observers(which=self) for p in self.tied_parameters] diff --git a/GPy/core/parameterization/variational.py b/GPy/core/parameterization/variational.py index e70b9b71..8a6f397a 100644 --- a/GPy/core/parameterization/variational.py +++ b/GPy/core/parameterization/variational.py @@ -150,6 +150,9 @@ class SpikeAndSlabPosterior(VariationalPosterior): n.parameters[dc['mean']._parent_index_] = dc['mean'] n.parameters[dc['variance']._parent_index_] = dc['variance'] n.parameters[dc['binary_prob']._parent_index_] = dc['binary_prob'] + n._gradient_array_ = None + oversize = self.size - self.mean.size - self.variance.size + n.size = n.mean.size + n.variance.size + oversize n.ndim = n.mean.ndim n.shape = n.mean.shape n.num_data = n.mean.shape[0] diff --git a/GPy/inference/latent_function_inference/var_dtc_parallel.py b/GPy/inference/latent_function_inference/var_dtc_parallel.py index 450e767a..ed385bc8 100644 --- a/GPy/inference/latent_function_inference/var_dtc_parallel.py +++ b/GPy/inference/latent_function_inference/var_dtc_parallel.py @@ -328,7 +328,7 @@ def update_gradients(model, mpi_comm=None): X = model.X else: Y = model.Y_local - X = model.X_local + X = model.X[model.N_range[0]:model.N_range[1]] model._log_marginal_likelihood, dL_dKmm, model.posterior = model.inference_method.inference_likelihood(model.kern, X, model.Z, model.likelihood, Y) @@ -350,7 +350,7 @@ def update_gradients(model, mpi_comm=None): if mpi_comm ==None: X_slice = model.X[n_range[0]:n_range[1]] else: - X_slice = model.X[model.Y_range[0]+n_range[0]:model.Y_range[0]+n_range[1]] + X_slice = model.X[model.N_range[0]+n_range[0]:model.N_range[0]+n_range[1]] #gradients w.r.t. kernel model.kern.update_gradients_expectations(variational_posterior=X_slice, Z=model.Z, dL_dpsi0=grad_dict['dL_dpsi0'], dL_dpsi1=grad_dict['dL_dpsi1'], dL_dpsi2=grad_dict['dL_dpsi2']) @@ -396,7 +396,7 @@ def update_gradients(model, mpi_comm=None): KL_div_all = np.array(KL_div) mpi_comm.Allreduce([np.float64(KL_div), MPI.DOUBLE], [KL_div_all, MPI.DOUBLE]) KL_div = KL_div_all - [mpi_comm.Allgatherv([pp.copy(), MPI.DOUBLE], [pa, (model.Y_list*pa.shape[-1], None), MPI.DOUBLE]) for pp,pa in zip(model.get_X_gradients(X),model.get_X_gradients(model.X))] + [mpi_comm.Allgatherv([pp.copy(), MPI.DOUBLE], [pa, (model.N_list*pa.shape[-1], None), MPI.DOUBLE]) for pp,pa in zip(model.get_X_gradients(X),model.get_X_gradients(model.X))] from ...models import SSGPLVM if isinstance(model, SSGPLVM): grad_pi = np.array(model.variational_prior.pi.gradient) diff --git a/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py b/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py index 60fa8259..9c699daa 100644 --- a/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py +++ b/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py @@ -339,7 +339,7 @@ class PSICOMP_SSRBF_GPU(PSICOMP_RBF): 'grad_l_gpu' :gpuarray.empty((Q,),np.float64, order='F'), 'grad_mu_gpu' :gpuarray.empty((N,Q,),np.float64, order='F'), 'grad_S_gpu' :gpuarray.empty((N,Q,),np.float64, order='F'), - 'grad_gamma_gpu' :gpuarray.empty((N,Q,),np.float64, order='F'), + 'grad_gamma_gpu' :gpuarray.empty((N,Q,),np.float64, order='F'), } else: assert N==self.gpuCache['mu_gpu'].shape[0] @@ -356,8 +356,6 @@ class PSICOMP_SSRBF_GPU(PSICOMP_RBF): self.gpuCache['S_gpu'].set(np.asfortranarray(S)) self.gpuCache['gamma_gpu'].set(np.asfortranarray(gamma)) N,Q = self.gpuCache['S_gpu'].shape - # t=self.g_compDenom(self.gpuCache['log_denom1_gpu'],self.gpuCache['log_denom2_gpu'],self.gpuCache['l_gpu'],self.gpuCache['S_gpu'], np.int32(N), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True) - # print 'g_compDenom '+str(t) self.g_compDenom.prepared_call((self.blocknum,1),(self.threadnum,1,1), self.gpuCache['log_denom1_gpu'].gpudata,self.gpuCache['log_denom2_gpu'].gpudata,self.gpuCache['log_gamma_gpu'].gpudata,self.gpuCache['log_gamma1_gpu'].gpudata,self.gpuCache['gamma_gpu'].gpudata,self.gpuCache['l_gpu'].gpudata,self.gpuCache['S_gpu'].gpudata, np.int32(N), np.int32(Q)) def reset_derivative(self): @@ -393,7 +391,6 @@ class PSICOMP_SSRBF_GPU(PSICOMP_RBF): Z_gpu = self.gpuCache['Z_gpu'] mu_gpu = self.gpuCache['mu_gpu'] S_gpu = self.gpuCache['S_gpu'] - gamma_gpu = self.gpuCache['gamma_gpu'] log_denom1_gpu = self.gpuCache['log_denom1_gpu'] log_denom2_gpu = self.gpuCache['log_denom2_gpu'] log_gamma_gpu = self.gpuCache['log_gamma_gpu'] @@ -403,11 +400,7 @@ class PSICOMP_SSRBF_GPU(PSICOMP_RBF): psi0[:] = variance self.g_psi1computations.prepared_call((self.blocknum,1),(self.threadnum,1,1),psi1_gpu.gpudata, log_denom1_gpu.gpudata, log_gamma_gpu.gpudata, log_gamma1_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata, np.int32(N), np.int32(M), np.int32(Q)) self.g_psi2computations.prepared_call((self.blocknum,1),(self.threadnum,1,1),psi2_gpu.gpudata, psi2n_gpu.gpudata, log_denom2_gpu.gpudata, log_gamma_gpu.gpudata, log_gamma1_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata, np.int32(N), np.int32(M), np.int32(Q)) - # t = self.g_psi1computations(psi1_gpu, log_denom1_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True) - # print 'g_psi1computations '+str(t) - # t = self.g_psi2computations(psi2_gpu, psi2n_gpu, log_denom2_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True) - # print 'g_psi2computations '+str(t) - + if self.GPU_direct: return psi0, psi1_gpu, psi2_gpu else: diff --git a/GPy/models/ss_gplvm.py b/GPy/models/ss_gplvm.py index c23e3524..817c11c0 100644 --- a/GPy/models/ss_gplvm.py +++ b/GPy/models/ss_gplvm.py @@ -45,7 +45,6 @@ class SSGPLVM(SparseGP): gamma[:] = 0.5 + 0.1 * np.random.randn(X.shape[0], input_dim) gamma[gamma>1.-1e-9] = 1.-1e-9 gamma[gamma<1e-9] = 1e-9 - gamma[:] = 0.5 if Z is None: Z = np.random.permutation(X.copy())[:num_inducing] @@ -72,20 +71,19 @@ class SSGPLVM(SparseGP): SparseGP.__init__(self, X, Y, Z, kernel, likelihood, inference_method, name, **kwargs) self.add_parameter(self.X, index=0) self.add_parameter(self.variational_prior) + + if mpi_comm != None: + from ..util.mpi import divide_data + N_start, N_end, N_list = divide_data(Y.shape[0], mpi_comm) + self.N_range = (N_start, N_end) + self.N_list = np.array(N_list) + self.Y_local = self.Y[N_start:N_end] + print 'MPI RANK: '+str(self.mpi_comm.rank)+' with datasize: '+str(self.N_range) + mpi_comm.Bcast(self.param_array, root=0) if self.group_spike: [self.X.gamma[:,i].tie('tieGamma'+str(i)) for i in xrange(self.X.gamma.shape[1])] # Tie columns together - if mpi_comm != None: - from ..util.mpi import divide_data - Y_start, Y_end, Y_list = divide_data(Y.shape[0], mpi_comm) - self.Y_local = self.Y[Y_start:Y_end] - self.X_local = self.X[Y_start:Y_end] - self.Y_range = (Y_start, Y_end) - self.Y_list = np.array(Y_list) - print self.mpi_comm.rank, self.Y_range - mpi_comm.Bcast(self.param_array, root=0) - def set_X_gradients(self, X, X_grad): """Set the gradients of the posterior distribution of X in its specific form.""" X.mean.gradient, X.variance.gradient, X.binary_prob.gradient = X_grad @@ -124,9 +122,10 @@ class SSGPLVM(SparseGP): dc = super(SSGPLVM, self).__getstate__() dc['mpi_comm'] = None if self.mpi_comm != None: + del dc['N_range'] + del dc['N_list'] del dc['Y_local'] del dc['X_local'] - del dc['Y_range'] return dc def __setstate__(self, state): diff --git a/GPy/util/gpu_init.py b/GPy/util/gpu_init.py index 917d8158..98f711e8 100644 --- a/GPy/util/gpu_init.py +++ b/GPy/util/gpu_init.py @@ -13,4 +13,10 @@ try: cublas_handle = cublas.cublasCreate() initSuccess = True except: - initSuccess = False \ No newline at end of file + initSuccess = False + +def initGPU(gpuid=None): + if gpuid==None: + return pycuda.tools.make_default_context() + else: + return pycuda.driver.Device(gpuid).make_context() \ No newline at end of file From bdf478956de92dc629829cd5013d6642edfac98e Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Thu, 26 Jun 2014 12:17:45 +0100 Subject: [PATCH 37/55] adapt gpu initialization multiple gpu cards --- GPy/kern/_src/rbf.py | 2 ++ GPy/util/gpu_init.py | 30 +++++++++++++++++++++++++----- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/GPy/kern/_src/rbf.py b/GPy/kern/_src/rbf.py index 90c9100b..bd768769 100644 --- a/GPy/kern/_src/rbf.py +++ b/GPy/kern/_src/rbf.py @@ -6,6 +6,7 @@ import numpy as np from stationary import Stationary from psi_comp import PSICOMP_RBF from psi_comp.rbf_psi_gpucomp import PSICOMP_RBF_GPU +from ...util.gpu_init import initGPU from ...util.config import * class RBF(Stationary): @@ -24,6 +25,7 @@ class RBF(Stationary): self.group_spike_prob = False self.psicomp = PSICOMP_RBF() if self.useGPU: + initGPU() self.psicomp = PSICOMP_RBF_GPU() else: self.psicomp = PSICOMP_RBF() diff --git a/GPy/util/gpu_init.py b/GPy/util/gpu_init.py index 98f711e8..e433e5fa 100644 --- a/GPy/util/gpu_init.py +++ b/GPy/util/gpu_init.py @@ -6,17 +6,37 @@ providing CUBLAS handle: cublas_handle """ try: - import pycuda.autoinit from scikits.cuda import cublas import scikits.cuda.linalg as culinalg culinalg.init() cublas_handle = cublas.cublasCreate() - initSuccess = True except: - initSuccess = False + +gpu_initialized = False +gpu_device = None +gpu_context = None def initGPU(gpuid=None): + if gpu_initialized: + return if gpuid==None: - return pycuda.tools.make_default_context() + try: + import pycuda.autoinit + gpu_initialized = True + except: + pass else: - return pycuda.driver.Device(gpuid).make_context() \ No newline at end of file + try: + import pycuda.driver + pycuda.driver.init() + if gpuid>=pycuda.driver.Device.count(): + return + gpu_device = pycuda.driver.Device(gpuid) + gpu_context = gpu_device.make_context() + gpu_initialized = True + except: + pass + +def closeGPU(): + if gpu_context is not None: + gpu_context.detach() From c3482e7a94f82416e7abc4436656b46866f942f7 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Thu, 26 Jun 2014 13:46:47 +0100 Subject: [PATCH 38/55] fix the gpu initialization for multiple cards --- GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py | 1 - GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py | 1 - GPy/kern/_src/rbf.py | 2 - GPy/util/gpu_init.py | 56 ++++++++++++--------- GPy/util/parallel.py | 14 ++++++ 5 files changed, 45 insertions(+), 29 deletions(-) create mode 100644 GPy/util/parallel.py diff --git a/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py index d8fec65b..623c45c4 100644 --- a/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py +++ b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py @@ -242,7 +242,6 @@ gpu_code = """ class PSICOMP_RBF_GPU(PSICOMP_RBF): def __init__(self, threadnum=128, blocknum=15, GPU_direct=False): - assert gpu_init.initSuccess, "GPU initialization failed!" self.GPU_direct = GPU_direct self.cublas_handle = gpu_init.cublas_handle self.gpuCache = None diff --git a/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py b/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py index 9c699daa..0c28794c 100644 --- a/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py +++ b/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py @@ -292,7 +292,6 @@ gpu_code = """ class PSICOMP_SSRBF_GPU(PSICOMP_RBF): def __init__(self, threadnum=128, blocknum=15, GPU_direct=False): - assert gpu_init.initSuccess, "GPU initialization failed!" self.GPU_direct = GPU_direct self.cublas_handle = gpu_init.cublas_handle self.gpuCache = None diff --git a/GPy/kern/_src/rbf.py b/GPy/kern/_src/rbf.py index bd768769..90c9100b 100644 --- a/GPy/kern/_src/rbf.py +++ b/GPy/kern/_src/rbf.py @@ -6,7 +6,6 @@ import numpy as np from stationary import Stationary from psi_comp import PSICOMP_RBF from psi_comp.rbf_psi_gpucomp import PSICOMP_RBF_GPU -from ...util.gpu_init import initGPU from ...util.config import * class RBF(Stationary): @@ -25,7 +24,6 @@ class RBF(Stationary): self.group_spike_prob = False self.psicomp = PSICOMP_RBF() if self.useGPU: - initGPU() self.psicomp = PSICOMP_RBF_GPU() else: self.psicomp = PSICOMP_RBF() diff --git a/GPy/util/gpu_init.py b/GPy/util/gpu_init.py index e433e5fa..845d38a1 100644 --- a/GPy/util/gpu_init.py +++ b/GPy/util/gpu_init.py @@ -5,37 +5,43 @@ Global variables: initSuccess providing CUBLAS handle: cublas_handle """ +gpu_initialized = False +gpu_device = None +gpu_context = None +MPI_enabled = False + +try: + from mpi4py import MPI + MPI_enabled = True +except: + pass + +try: + if MPI_enabled and MPI.COMM_WORLD.size>1: + from .parallel import get_id_within_node + gpuid = get_id_within_node() + import pycuda.driver + pycuda.driver.init() + if gpuid>=pycuda.driver.Device.count(): + print '['+MPI.Get_processor_name()+'] more processes than the GPU numbers!' + MPI.COMM_WORLD.Abort() + raise + gpu_device = pycuda.driver.Device(gpuid) + gpu_context = gpu_device.make_context() + gpu_initialized = True + else: + import pycuda.autoinit + gpu_initialized = True +except: + pass + try: from scikits.cuda import cublas import scikits.cuda.linalg as culinalg culinalg.init() cublas_handle = cublas.cublasCreate() except: - -gpu_initialized = False -gpu_device = None -gpu_context = None - -def initGPU(gpuid=None): - if gpu_initialized: - return - if gpuid==None: - try: - import pycuda.autoinit - gpu_initialized = True - except: - pass - else: - try: - import pycuda.driver - pycuda.driver.init() - if gpuid>=pycuda.driver.Device.count(): - return - gpu_device = pycuda.driver.Device(gpuid) - gpu_context = gpu_device.make_context() - gpu_initialized = True - except: - pass + pass def closeGPU(): if gpu_context is not None: diff --git a/GPy/util/parallel.py b/GPy/util/parallel.py new file mode 100644 index 00000000..fd8791d4 --- /dev/null +++ b/GPy/util/parallel.py @@ -0,0 +1,14 @@ +""" +The module of tools for parallelization (MPI) +""" + +try: + from mpi4py import MPI +except: + pass + +def get_id_within_node(comm=MPI.COMM_WORLD): + rank = comm.rank + nodename = MPI.Get_processor_name() + nodelist = comm.allgather(nodename) + return len([i for i in nodelist[:rank] if i==nodename]) From 92fecbb4efd4a4cb81725b4be7281fce57830479 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Thu, 26 Jun 2014 14:01:58 +0100 Subject: [PATCH 39/55] remove dependence of cublas from rbf kernel --- GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py | 1 - GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py | 1 - 2 files changed, 2 deletions(-) diff --git a/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py index 623c45c4..c159a0b6 100644 --- a/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py +++ b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py @@ -243,7 +243,6 @@ class PSICOMP_RBF_GPU(PSICOMP_RBF): def __init__(self, threadnum=128, blocknum=15, GPU_direct=False): self.GPU_direct = GPU_direct - self.cublas_handle = gpu_init.cublas_handle self.gpuCache = None self.threadnum = threadnum diff --git a/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py b/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py index 0c28794c..c430e217 100644 --- a/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py +++ b/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py @@ -293,7 +293,6 @@ class PSICOMP_SSRBF_GPU(PSICOMP_RBF): def __init__(self, threadnum=128, blocknum=15, GPU_direct=False): self.GPU_direct = GPU_direct - self.cublas_handle = gpu_init.cublas_handle self.gpuCache = None self.threadnum = threadnum From 1512590a8ceef1b3883ed7f85c96536205e18d50 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Thu, 26 Jun 2014 14:12:19 +0100 Subject: [PATCH 40/55] remove dependence of scikits.cuda from rbf kernel --- GPy/util/gpu_init.py | 2 +- GPy/util/linalg_gpu.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/GPy/util/gpu_init.py b/GPy/util/gpu_init.py index 845d38a1..03d07d77 100644 --- a/GPy/util/gpu_init.py +++ b/GPy/util/gpu_init.py @@ -17,7 +17,7 @@ except: pass try: - if MPI_enabled and MPI.COMM_WORLD.size>1: + if MPI_enabled: #and MPI.COMM_WORLD.size>1: from .parallel import get_id_within_node gpuid = get_id_within_node() import pycuda.driver diff --git a/GPy/util/linalg_gpu.py b/GPy/util/linalg_gpu.py index 1b9b0594..d969d14f 100644 --- a/GPy/util/linalg_gpu.py +++ b/GPy/util/linalg_gpu.py @@ -12,9 +12,6 @@ from ..util import gpu_init try: from pycuda.reduction import ReductionKernel from pycuda.elementwise import ElementwiseKernel - import scikits.cuda.linalg as culinalg - from scikits.cuda import cublas - from scikits.cuda.cula import culaExceptions # log|A| for A is a low triangle matrix # logDiagSum(A, A.shape[0]+1) @@ -64,6 +61,13 @@ try: except: pass +try: + import scikits.cuda.linalg as culinalg + from scikits.cuda import cublas + from scikits.cuda.cula import culaExceptions +except: + pass + def jitchol(A, L, cublas_handle, maxtries=5): try: cublas.cublasDcopy(cublas_handle, A.size, A.gpudata, 1, L.gpudata, 1) From 8a4a4e56a90ec52ddd2b40a4a4659d3292e7adab Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Thu, 26 Jun 2014 14:39:02 +0100 Subject: [PATCH 41/55] fix pickle for ssgplvm and bgplvm with mpi --- GPy/models/bayesian_gplvm.py | 14 +++++++------- GPy/models/ss_gplvm.py | 1 - 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/GPy/models/bayesian_gplvm.py b/GPy/models/bayesian_gplvm.py index 65ab2469..8f61a661 100644 --- a/GPy/models/bayesian_gplvm.py +++ b/GPy/models/bayesian_gplvm.py @@ -71,11 +71,11 @@ class BayesianGPLVM(SparseGP): if mpi_comm != None: from ..util.mpi import divide_data - Y_start, Y_end, Y_list = divide_data(Y.shape[0], mpi_comm) - self.Y_local = self.Y[Y_start:Y_end] - self.X_local = self.X[Y_start:Y_end] - self.Y_range = (Y_start, Y_end) - self.Y_list = np.array(Y_list) + N_start, N_end, N_list = divide_data(Y.shape[0], mpi_comm) + self.N_range = (N_start, N_end) + self.N_list = np.array(N_list) + self.Y_local = self.Y[N_start:N_end] + print 'MPI RANK: '+str(self.mpi_comm.rank)+' with datasize: '+str(self.N_range) mpi_comm.Bcast(self.param_array, root=0) def set_X_gradients(self, X, X_grad): @@ -184,9 +184,9 @@ class BayesianGPLVM(SparseGP): dc = super(BayesianGPLVM, self).__getstate__() dc['mpi_comm'] = None if self.mpi_comm != None: + del dc['N_range'] + del dc['N_list'] del dc['Y_local'] - del dc['X_local'] - del dc['Y_range'] return dc def __setstate__(self, state): diff --git a/GPy/models/ss_gplvm.py b/GPy/models/ss_gplvm.py index 817c11c0..2fa3cb2a 100644 --- a/GPy/models/ss_gplvm.py +++ b/GPy/models/ss_gplvm.py @@ -125,7 +125,6 @@ class SSGPLVM(SparseGP): del dc['N_range'] del dc['N_list'] del dc['Y_local'] - del dc['X_local'] return dc def __setstate__(self, state): From 35eb0a015f5136e259cd53a73c0b2451fa369d89 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Fri, 27 Jun 2014 10:03:25 +0100 Subject: [PATCH 42/55] add ss_mrd model --- GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py | 5 +++++ GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py | 7 ++++++- GPy/kern/_src/rbf.py | 9 +++++++++ GPy/models/__init__.py | 1 + GPy/models/ss_gplvm.py | 2 +- GPy/models/ss_mrd.py | 22 +++++++++++++++++++++ GPy/util/gpu_init.py | 2 +- 7 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 GPy/models/ss_mrd.py diff --git a/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py index c159a0b6..73c2c33b 100644 --- a/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py +++ b/GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py @@ -258,6 +258,11 @@ class PSICOMP_RBF_GPU(PSICOMP_RBF): self.g_psi2compDer.prepare('PPPPPPPdPPPPiii') self.g_compDenom = module.get_function('compDenom') self.g_compDenom.prepare('PPPPii') + + def __deepcopy__(self, memo): + s = PSICOMP_RBF_GPU(threadnum=self.threadnum, blocknum=self.blocknum, GPU_direct=self.GPU_direct) + memo[id(self)] = s + return s def _initGPUCache(self, N, M, Q): if self.gpuCache == None: diff --git a/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py b/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py index c430e217..1a9d2058 100644 --- a/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py +++ b/GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py @@ -308,7 +308,12 @@ class PSICOMP_SSRBF_GPU(PSICOMP_RBF): self.g_psi2compDer.prepare('PPPPPPPPPPPdPPPPPiii') self.g_compDenom = module.get_function('compDenom') self.g_compDenom.prepare('PPPPPPPii') - + + def __deepcopy__(self, memo): + s = PSICOMP_SSRBF_GPU(threadnum=self.threadnum, blocknum=self.blocknum, GPU_direct=self.GPU_direct) + memo[id(self)] = s + return s + def _initGPUCache(self, N, M, Q): if self.gpuCache == None: self.gpuCache = { diff --git a/GPy/kern/_src/rbf.py b/GPy/kern/_src/rbf.py index 90c9100b..471c3305 100644 --- a/GPy/kern/_src/rbf.py +++ b/GPy/kern/_src/rbf.py @@ -43,6 +43,15 @@ class RBF(Stationary): def __setstate__(self, state): return super(RBF, self).__setstate__(state) +# def copy(self): +# k = super(RBF, self).copy() +# # Make sure the copy of the kernel instance has different instance of psicomp +# if k.useGPU: +# k.psicomp = PSICOMP_RBF_GPU() +# else: +# k.psicomp = PSICOMP_RBF() +# return k + #---------------------------------------# # PSI statistics # #---------------------------------------# diff --git a/GPy/models/__init__.py b/GPy/models/__init__.py index 299d5e65..1fb781d5 100644 --- a/GPy/models/__init__.py +++ b/GPy/models/__init__.py @@ -17,3 +17,4 @@ from ss_gplvm import SSGPLVM from gp_coregionalized_regression import GPCoregionalizedRegression from sparse_gp_coregionalized_regression import SparseGPCoregionalizedRegression from gp_heteroscedastic_regression import GPHeteroscedasticRegression +from ss_mrd import SSMRD diff --git a/GPy/models/ss_gplvm.py b/GPy/models/ss_gplvm.py index 2fa3cb2a..df35d549 100644 --- a/GPy/models/ss_gplvm.py +++ b/GPy/models/ss_gplvm.py @@ -24,7 +24,7 @@ class SSGPLVM(SparseGP): """ def __init__(self, Y, input_dim, X=None, X_variance=None, init='PCA', num_inducing=10, - Z=None, kernel=None, inference_method=None, likelihood=None, name='Spike-and-Slab GPLVM', group_spike=False, mpi_comm=None, **kwargs): + Z=None, kernel=None, inference_method=None, likelihood=None, name='Spike_and_Slab GPLVM', group_spike=False, mpi_comm=None, **kwargs): self.mpi_comm = mpi_comm self.__IN_OPTIMIZATION__ = False diff --git a/GPy/models/ss_mrd.py b/GPy/models/ss_mrd.py new file mode 100644 index 00000000..f959eb59 --- /dev/null +++ b/GPy/models/ss_mrd.py @@ -0,0 +1,22 @@ +""" +The Maniforld Relevance Determination model with the spike-and-slab prior +""" + +from ..core import Model +from .ss_gplvm import SSGPLVM + +class SSMRD(Model): + + def __init__(self, Ylist, input_dim, X=None, X_variance=None, + initx = 'PCA', initz = 'permute', + num_inducing=10, Z=None, kernel=None, + inference_method=None, likelihoods=None, name='ss_mrd', Ynames=None): + super(SSMRD, self).__init__(name) + + self.updates = False + self.models = [SSGPLVM(y, input_dim, X=X, X_variance=X_variance, num_inducing=num_inducing,Z=Z,init=initx, + kernel=kernel if kernel else None,inference_method=inference_method,likelihood=likelihoods, + name='model_'+str(i)) for i,y in enumerate(Ylist)] + self.add_parameters(*(self.models)) + self.updates = True + diff --git a/GPy/util/gpu_init.py b/GPy/util/gpu_init.py index 03d07d77..845d38a1 100644 --- a/GPy/util/gpu_init.py +++ b/GPy/util/gpu_init.py @@ -17,7 +17,7 @@ except: pass try: - if MPI_enabled: #and MPI.COMM_WORLD.size>1: + if MPI_enabled and MPI.COMM_WORLD.size>1: from .parallel import get_id_within_node gpuid = get_id_within_node() import pycuda.driver From 2c88528ebd96ecb8ce53a9614bdfc3d0a41ccc61 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Fri, 27 Jun 2014 17:25:08 +0100 Subject: [PATCH 43/55] ss_mrd with parameter tied --- GPy/models/ss_mrd.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/GPy/models/ss_mrd.py b/GPy/models/ss_mrd.py index f959eb59..612ea350 100644 --- a/GPy/models/ss_mrd.py +++ b/GPy/models/ss_mrd.py @@ -19,4 +19,13 @@ class SSMRD(Model): name='model_'+str(i)) for i,y in enumerate(Ylist)] self.add_parameters(*(self.models)) self.updates = True + + [[self.models[j].X.mean.flat[i:i+1].tie('mean_'+str(i)) for j in xrange(len(self.models))] for i in xrange(self.models[0].X.mean.size)] + [[self.models[j].X.variance.flat[i:i+1].tie('var_'+str(i)) for j in xrange(len(self.models))] for i in xrange(self.models[0].X.variance.size)] + def parameters_changed(self): + super(SSMRD, self).parameters_changed() + self._log_marginal_likelihood = sum([m._log_marginal_likelihood for m in self.models]) + + def log_likelihood(self): + return self._log_marginal_likelihood \ No newline at end of file From 129985998c5b92d33298d324cbe6acf2412f011b Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Tue, 1 Jul 2014 13:16:57 +0100 Subject: [PATCH 44/55] fix ss_mrd and fix white and bias kernel --- GPy/core/parameterization/parameter_core.py | 2 +- GPy/kern/_src/add.py | 11 ++++++----- GPy/kern/_src/static.py | 14 +++++++------- GPy/models/ss_mrd.py | 11 +++++++---- 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/GPy/core/parameterization/parameter_core.py b/GPy/core/parameterization/parameter_core.py index cdf3f534..bf5b74ac 100644 --- a/GPy/core/parameterization/parameter_core.py +++ b/GPy/core/parameterization/parameter_core.py @@ -520,7 +520,7 @@ class Indexable(Nameable, Observable): self.constraints.add(t, self._raveled_index()) t.add_tied_parameter(self) - self._highest_parent_._connect_fixes() + self._highest_parent_._connect_fixes() def constrain(self, transform, warning=True, trigger_parent=True): """ diff --git a/GPy/kern/_src/add.py b/GPy/kern/_src/add.py index 12f5d444..d63c9ab8 100644 --- a/GPy/kern/_src/add.py +++ b/GPy/kern/_src/add.py @@ -88,17 +88,18 @@ class Add(CombinationKernel): # rbf X bias #elif isinstance(p1, (Bias, Fixed)) and isinstance(p2, (RBF, RBFInv)): elif isinstance(p1, Bias) and isinstance(p2, (RBF, Linear)): - tmp = p2.psi1(Z, variational_posterior) - psi2 += p1.variance * (tmp[:, :, None] + tmp[:, None, :]) + tmp = p2.psi1(Z, variational_posterior).sum(axis=0) + psi2 += p1.variance * (tmp[:,None]+tmp[None,:]) #(tmp[:, :, None] + tmp[:, None, :]) #elif isinstance(p2, (Bias, Fixed)) and isinstance(p1, (RBF, RBFInv)): elif isinstance(p2, Bias) and isinstance(p1, (RBF, Linear)): - tmp = p1.psi1(Z, variational_posterior) - psi2 += p2.variance * (tmp[:, :, None] + tmp[:, None, :]) + tmp = p1.psi1(Z, variational_posterior).sum(axis=0) + psi2 += p2.variance * (tmp[:,None]+tmp[None,:]) #(tmp[:, :, None] + tmp[:, None, :]) elif isinstance(p2, (RBF, Linear)) and isinstance(p1, (RBF, Linear)): assert np.intersect1d(p1.active_dims, p2.active_dims).size == 0, "only non overlapping kernel dimensions allowed so far" tmp1 = p1.psi1(Z, variational_posterior) tmp2 = p2.psi1(Z, variational_posterior) - psi2 += (tmp1[:, :, None] * tmp2[:, None, :]) + (tmp2[:, :, None] * tmp1[:, None, :]) + psi2 += np.einsum('nm,no->mo',tmp1,tmp2)+np.einsum('nm,no->mo',tmp2,tmp1) + #(tmp1[:, :, None] * tmp2[:, None, :]) + (tmp2[:, :, None] * tmp1[:, None, :]) else: raise NotImplementedError, "psi2 cannot be computed for this kernel" return psi2 diff --git a/GPy/kern/_src/static.py b/GPy/kern/_src/static.py index 68884937..f8ce7fd1 100644 --- a/GPy/kern/_src/static.py +++ b/GPy/kern/_src/static.py @@ -39,7 +39,7 @@ class Static(Kern): def psi2(self, Z, variational_posterior): K = self.K(variational_posterior.mean, Z) - return K[:,:,None]*K[:,None,:] # NB. more efficient implementations on inherriting classes + return np.einsum('ij,ik->jk',K,K) #K[:,:,None]*K[:,None,:] # NB. more efficient implementations on inherriting classes class White(Static): @@ -53,7 +53,7 @@ class White(Static): return np.zeros((X.shape[0], X2.shape[0])) def psi2(self, Z, variational_posterior): - return np.zeros((variational_posterior.shape[0], Z.shape[0], Z.shape[0]), dtype=np.float64) + return np.zeros((Z.shape[0], Z.shape[0]), dtype=np.float64) def update_gradients_full(self, dL_dK, X, X2=None): self.variance.gradient = np.trace(dL_dK) @@ -82,12 +82,12 @@ class Bias(Static): self.variance.gradient = dL_dKdiag.sum() def psi2(self, Z, variational_posterior): - ret = np.empty((variational_posterior.shape[0], Z.shape[0], Z.shape[0]), dtype=np.float64) - ret[:] = self.variance**2 + ret = np.empty((Z.shape[0], Z.shape[0]), dtype=np.float64) + ret[:] = self.variance*self.variance*variational_posterior.shape[0] return ret def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): - self.variance.gradient = dL_dpsi0.sum() + dL_dpsi1.sum() + 2.*self.variance*dL_dpsi2.sum() + self.variance.gradient = dL_dpsi0.sum() + dL_dpsi1.sum() + 2.*self.variance*dL_dpsi2.sum()*variational_posterior.shape[0] class Fixed(Static): def __init__(self, input_dim, covariance_matrix, variance=1., active_dims=None, name='fixed'): @@ -97,7 +97,7 @@ class Fixed(Static): :param variance: the variance of the kernel :type variance: float """ - super(Bias, self).__init__(input_dim, variance, active_dims, name) + super(Fixed, self).__init__(input_dim, variance, active_dims, name) self.fixed_K = covariance_matrix def K(self, X, X2): return self.variance * self.fixed_K @@ -112,7 +112,7 @@ class Fixed(Static): self.variance.gradient = np.einsum('i,i', dL_dKdiag, self.fixed_K) def psi2(self, Z, variational_posterior): - return np.zeros((variational_posterior.shape[0], Z.shape[0], Z.shape[0]), dtype=np.float64) + return np.zeros((Z.shape[0], Z.shape[0]), dtype=np.float64) def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): self.variance.gradient = dL_dpsi0.sum() diff --git a/GPy/models/ss_mrd.py b/GPy/models/ss_mrd.py index 612ea350..036ac095 100644 --- a/GPy/models/ss_mrd.py +++ b/GPy/models/ss_mrd.py @@ -15,14 +15,17 @@ class SSMRD(Model): self.updates = False self.models = [SSGPLVM(y, input_dim, X=X, X_variance=X_variance, num_inducing=num_inducing,Z=Z,init=initx, - kernel=kernel if kernel else None,inference_method=inference_method,likelihood=likelihoods, + kernel=kernel.copy() if kernel else None,inference_method=inference_method,likelihood=likelihoods, name='model_'+str(i)) for i,y in enumerate(Ylist)] self.add_parameters(*(self.models)) + + [[[self.models[m].X.mean[i,j:j+1].tie('mean_'+str(i)+'_'+str(j)) for m in xrange(len(self.models))] for j in xrange(self.models[0].X.mean.shape[1])] + for i in xrange(self.models[0].X.mean.shape[0])] + [[[self.models[m].X.variance[i,j:j+1].tie('var_'+str(i)+'_'+str(j)) for m in xrange(len(self.models))] for j in xrange(self.models[0].X.variance.shape[1])] + for i in xrange(self.models[0].X.variance.shape[0])] + self.updates = True - [[self.models[j].X.mean.flat[i:i+1].tie('mean_'+str(i)) for j in xrange(len(self.models))] for i in xrange(self.models[0].X.mean.size)] - [[self.models[j].X.variance.flat[i:i+1].tie('var_'+str(i)) for j in xrange(len(self.models))] for i in xrange(self.models[0].X.variance.size)] - def parameters_changed(self): super(SSMRD, self).parameters_changed() self._log_marginal_likelihood = sum([m._log_marginal_likelihood for m in self.models]) From 216de32c0c784c06dc88b6bf62a9f577a460f135 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Tue, 1 Jul 2014 18:01:33 +0100 Subject: [PATCH 45/55] fix add kernel and VarDTC_minibatch speed tuning --- .../latent_function_inference/var_dtc_parallel.py | 12 +++++++++--- GPy/kern/_src/add.py | 7 +++++-- GPy/kern/_src/psi_comp/__init__.py | 8 ++++---- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/GPy/inference/latent_function_inference/var_dtc_parallel.py b/GPy/inference/latent_function_inference/var_dtc_parallel.py index ed385bc8..03f5d478 100644 --- a/GPy/inference/latent_function_inference/var_dtc_parallel.py +++ b/GPy/inference/latent_function_inference/var_dtc_parallel.py @@ -94,8 +94,12 @@ class VarDTC_minibatch(LatentFunctionInference): for n_start in xrange(0,num_data,self.batchsize): n_end = min(self.batchsize+n_start, num_data) - Y_slice = Y[n_start:n_end] - X_slice = X[n_start:n_end] + if (n_end-n_start)==num_data: + Y_slice = Y + X_slice = X + else: + Y_slice = Y[n_start:n_end] + X_slice = X[n_start:n_end] if het_noise: b = beta[n_start] @@ -347,7 +351,9 @@ def update_gradients(model, mpi_comm=None): while not isEnd: isEnd, n_range, grad_dict = model.inference_method.inference_minibatch(model.kern, X, model.Z, model.likelihood, Y) if isinstance(model.X, VariationalPosterior): - if mpi_comm ==None: + if (n_range[1]-n_range[0])==X.shape[0]: + X_slice = X + elif mpi_comm ==None: X_slice = model.X[n_range[0]:n_range[1]] else: X_slice = model.X[model.N_range[0]+n_range[0]:model.N_range[0]+n_range[1]] diff --git a/GPy/kern/_src/add.py b/GPy/kern/_src/add.py index d63c9ab8..cda80209 100644 --- a/GPy/kern/_src/add.py +++ b/GPy/kern/_src/add.py @@ -63,13 +63,16 @@ class Add(CombinationKernel): target = np.zeros(X.shape) [target.__iadd__(p.gradients_X_diag(dL_dKdiag, X)) for p in self.parts] return target - + + @Cache_this(limit=2, force_kwargs=['which_parts']) def psi0(self, Z, variational_posterior): return reduce(np.add, (p.psi0(Z, variational_posterior) for p in self.parts)) - + + @Cache_this(limit=2, force_kwargs=['which_parts']) def psi1(self, Z, variational_posterior): return reduce(np.add, (p.psi1(Z, variational_posterior) for p in self.parts)) + @Cache_this(limit=2, force_kwargs=['which_parts']) def psi2(self, Z, variational_posterior): psi2 = reduce(np.add, (p.psi2(Z, variational_posterior) for p in self.parts)) #return psi2 diff --git a/GPy/kern/_src/psi_comp/__init__.py b/GPy/kern/_src/psi_comp/__init__.py index 38094394..1395ad98 100644 --- a/GPy/kern/_src/psi_comp/__init__.py +++ b/GPy/kern/_src/psi_comp/__init__.py @@ -10,7 +10,7 @@ import sslinear_psi_comp class PSICOMP_RBF(Pickleable): - @Cache_this(limit=1, ignore_args=(0,)) + @Cache_this(limit=2, ignore_args=(0,)) def psicomputations(self, variance, lengthscale, Z, variational_posterior): if isinstance(variational_posterior, variational.NormalPosterior): return rbf_psi_comp.psicomputations(variance, lengthscale, Z, variational_posterior) @@ -19,7 +19,7 @@ class PSICOMP_RBF(Pickleable): else: raise ValueError, "unknown distriubtion received for psi-statistics" - @Cache_this(limit=1, ignore_args=(0,1,2,3)) + @Cache_this(limit=2, ignore_args=(0,1,2,3)) def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior): if isinstance(variational_posterior, variational.NormalPosterior): return rbf_psi_comp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior) @@ -30,7 +30,7 @@ class PSICOMP_RBF(Pickleable): class PSICOMP_Linear(Pickleable): - @Cache_this(limit=1, ignore_args=(0,)) + @Cache_this(limit=2, ignore_args=(0,)) def psicomputations(self, variance, Z, variational_posterior): if isinstance(variational_posterior, variational.NormalPosterior): raise NotImplementedError @@ -39,7 +39,7 @@ class PSICOMP_Linear(Pickleable): else: raise ValueError, "unknown distriubtion received for psi-statistics" - @Cache_this(limit=1, ignore_args=(0,1,2,3)) + @Cache_this(limit=2, ignore_args=(0,1,2,3)) def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior): if isinstance(variational_posterior, variational.NormalPosterior): raise NotImplementedError From 1c165db84580077851390c9803260292fff54379 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Wed, 2 Jul 2014 10:52:47 +0100 Subject: [PATCH 46/55] simplify the interface of using mpi --- GPy/models/bayesian_gplvm.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/GPy/models/bayesian_gplvm.py b/GPy/models/bayesian_gplvm.py index 8f61a661..a55f7e60 100644 --- a/GPy/models/bayesian_gplvm.py +++ b/GPy/models/bayesian_gplvm.py @@ -57,11 +57,13 @@ class BayesianGPLVM(SparseGP): if np.any(np.isnan(Y)): from ..inference.latent_function_inference.var_dtc import VarDTCMissingData inference_method = VarDTCMissingData() - elif mpi_comm != None: + elif mpi_comm is not None: inference_method = VarDTC_minibatch(mpi_comm=mpi_comm) else: from ..inference.latent_function_inference.var_dtc import VarDTC inference_method = VarDTC() + if isinstance(inference_method,VarDTC_minibatch): + inference_method.mpi_comm = mpi_comm if kernel.useGPU and isinstance(inference_method, VarDTC_GPU): kernel.psicomp.GPU_direct = True From f16db004383a75b25b2d4093f07eec261e0368cb Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Wed, 2 Jul 2014 14:21:27 +0100 Subject: [PATCH 47/55] fix sparse_gplvm --- GPy/kern/_src/rbf.py | 9 -------- GPy/models/sparse_gplvm.py | 44 +++++++------------------------------- 2 files changed, 8 insertions(+), 45 deletions(-) diff --git a/GPy/kern/_src/rbf.py b/GPy/kern/_src/rbf.py index 471c3305..90c9100b 100644 --- a/GPy/kern/_src/rbf.py +++ b/GPy/kern/_src/rbf.py @@ -43,15 +43,6 @@ class RBF(Stationary): def __setstate__(self, state): return super(RBF, self).__setstate__(state) -# def copy(self): -# k = super(RBF, self).copy() -# # Make sure the copy of the kernel instance has different instance of psicomp -# if k.useGPU: -# k.psicomp = PSICOMP_RBF_GPU() -# else: -# k.psicomp = PSICOMP_RBF() -# return k - #---------------------------------------# # PSI statistics # #---------------------------------------# diff --git a/GPy/models/sparse_gplvm.py b/GPy/models/sparse_gplvm.py index 638da63e..b1b3c2db 100644 --- a/GPy/models/sparse_gplvm.py +++ b/GPy/models/sparse_gplvm.py @@ -11,7 +11,7 @@ from GPy.models.gplvm import GPLVM # from ..core import model # from ..util.linalg import pdinv, PCA -class SparseGPLVM(SparseGPRegression, GPLVM): +class SparseGPLVM(SparseGPRegression): """ Sparse Gaussian Process Latent Variable Model @@ -23,40 +23,12 @@ class SparseGPLVM(SparseGPRegression, GPLVM): :type init: 'PCA'|'random' """ - def __init__(self, Y, input_dim, kernel=None, init='PCA', num_inducing=10): - X = self.initialise_latent(init, input_dim, Y) + def __init__(self, Y, input_dim, X=None, kernel=None, init='PCA', num_inducing=10): + if X is None: + from ..util.initialization import initialize_latent + X, fracs = initialize_latent(init, input_dim, Y) SparseGPRegression.__init__(self, X, Y, kernel=kernel, num_inducing=num_inducing) - self.ensure_default_constraints() - def _get_param_names(self): - return (sum([['X_%i_%i' % (n, q) for q in range(self.input_dim)] for n in range(self.num_data)], []) - + SparseGPRegression._get_param_names(self)) - - def _get_params(self): - return np.hstack((self.X.flatten(), SparseGPRegression._get_params(self))) - - def _set_params(self, x): - self.X = x[:self.X.size].reshape(self.num_data, self.input_dim).copy() - SparseGPRegression._set_params(self, x[self.X.size:]) - - def log_likelihood(self): - return SparseGPRegression.log_likelihood(self) - - def dL_dX(self): - dL_dX = self.kern.dKdiag_dX(self.dL_dpsi0, self.X) - dL_dX += self.kern.gradients_X(self.dL_dpsi1, self.X, self.Z) - - return dL_dX - - def _log_likelihood_gradients(self): - return np.hstack((self.dL_dX().flatten(), SparseGPRegression._log_likelihood_gradients(self))) - - def plot(self): - GPLVM.plot(self) - # passing Z without a small amout of jitter will induce the white kernel where we don;t want it! - mu, var, upper, lower = SparseGPRegression.predict(self, self.Z + np.random.randn(*self.Z.shape) * 0.0001) - pb.plot(mu[:, 0] , mu[:, 1], 'ko') - - def plot_latent(self, *args, **kwargs): - input_1, input_2 = GPLVM.plot_latent(*args, **kwargs) - pb.plot(m.Z[:, input_1], m.Z[:, input_2], '^w') + def parameters_changed(self): + super(SparseGPLVM, self).parameters_changed() + self.X.gradient = self.kern.gradients_X(self.grad_dict['dL_dKnm'], self.X, self.Z) From a7d04387315af55dd1afdccbae60a335a4a0de21 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Wed, 2 Jul 2014 14:23:56 +0100 Subject: [PATCH 48/55] add plot_latent to sparse_gplvm --- GPy/models/sparse_gplvm.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/GPy/models/sparse_gplvm.py b/GPy/models/sparse_gplvm.py index b1b3c2db..4642e158 100644 --- a/GPy/models/sparse_gplvm.py +++ b/GPy/models/sparse_gplvm.py @@ -32,3 +32,16 @@ class SparseGPLVM(SparseGPRegression): def parameters_changed(self): super(SparseGPLVM, self).parameters_changed() self.X.gradient = self.kern.gradients_X(self.grad_dict['dL_dKnm'], self.X, self.Z) + + def plot_latent(self, labels=None, which_indices=None, + resolution=50, ax=None, marker='o', s=40, + fignum=None, plot_inducing=True, legend=True, + plot_limits=None, + aspect='auto', updates=False, predict_kwargs={}, imshow_kwargs={}): + assert "matplotlib" in sys.modules, "matplotlib package has not been imported." + from ..plotting.matplot_dep import dim_reduction_plots + + return dim_reduction_plots.plot_latent(self, labels, which_indices, + resolution, ax, marker, s, + fignum, plot_inducing, legend, + plot_limits, aspect, updates, predict_kwargs, imshow_kwargs) From 872e97af847d24e447f335f956fcd8dcbb4a6931 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Thu, 3 Jul 2014 16:31:22 +0100 Subject: [PATCH 49/55] fix the problem starting multiple process with limited number of GPUs --- GPy/util/gpu_init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPy/util/gpu_init.py b/GPy/util/gpu_init.py index 845d38a1..b6a4a164 100644 --- a/GPy/util/gpu_init.py +++ b/GPy/util/gpu_init.py @@ -24,7 +24,7 @@ try: pycuda.driver.init() if gpuid>=pycuda.driver.Device.count(): print '['+MPI.Get_processor_name()+'] more processes than the GPU numbers!' - MPI.COMM_WORLD.Abort() + #MPI.COMM_WORLD.Abort() raise gpu_device = pycuda.driver.Device(gpuid) gpu_context = gpu_device.make_context() From dd6823446d72059626fb0af62607bc25c4f1292c Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Mon, 28 Jul 2014 11:51:35 +0100 Subject: [PATCH 50/55] ssgplvm simulation example --- GPy/examples/dimensionality_reduction.py | 25 ++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/GPy/examples/dimensionality_reduction.py b/GPy/examples/dimensionality_reduction.py index a216eec6..6647c79c 100644 --- a/GPy/examples/dimensionality_reduction.py +++ b/GPy/examples/dimensionality_reduction.py @@ -289,6 +289,31 @@ def bgplvm_simulation(optimize=True, verbose=1, m.kern.plot_ARD('BGPLVM Simulation ARD Parameters') return m +def ssgplvm_simulation(optimize=True, verbose=1, + plot=True, plot_sim=False, + max_iters=2e4, + ): + from GPy import kern + from GPy.models import SSGPLVM + + D1, D2, D3, N, num_inducing, Q = 13, 5, 8, 45, 3, 9 + _, _, Ylist = _simulate_sincos(D1, D2, D3, N, num_inducing, Q, plot_sim) + Y = Ylist[0] + k = kern.Linear(Q, ARD=True)# + kern.white(Q, _np.exp(-2)) # + kern.bias(Q) + #k = kern.RBF(Q, ARD=True, lengthscale=10.) + m = SSGPLVM(Y, Q, init="pca", num_inducing=num_inducing, kernel=k) + m.X.variance[:] = _np.random.uniform(0,.01,m.X.shape) + m.likelihood.variance = .1 + + if optimize: + print "Optimizing model:" + m.optimize('scg', messages=verbose, max_iters=max_iters, + gtol=.05) + if plot: + m.X.plot("SSGPLVM Latent Space 1D") + m.kern.plot_ARD('SSGPLVM Simulation ARD Parameters') + return m + def bgplvm_simulation_missing_data(optimize=True, verbose=1, plot=True, plot_sim=False, max_iters=2e4, From 6acb9b09b5ed4848988711a81f6a5909106f72a8 Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Mon, 11 Aug 2014 14:12:43 +0100 Subject: [PATCH 51/55] generalize the spike-and-slab prior with pi (N,Q) --- GPy/core/parameterization/variational.py | 25 ++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/GPy/core/parameterization/variational.py b/GPy/core/parameterization/variational.py index 8a6f397a..6c251d21 100644 --- a/GPy/core/parameterization/variational.py +++ b/GPy/core/parameterization/variational.py @@ -34,31 +34,36 @@ class NormalPrior(VariationalPrior): variational_posterior.variance.gradient -= (1. - (1. / (variational_posterior.variance))) * 0.5 class SpikeAndSlabPrior(VariationalPrior): - def __init__(self, pi, variance = 1.0, name='SpikeAndSlabPrior', **kw): + def __init__(self, pi=None, learnPi=False, variance = 1.0, name='SpikeAndSlabPrior', **kw): super(VariationalPrior, self).__init__(name=name, **kw) - assert variance==1.0, "Not Implemented!" self.pi = Param('pi', pi, Logistic(1e-10,1.-1e-10)) self.variance = Param('variance',variance) - self.add_parameters(self.pi) + if learnPi: + self.add_parameters(self.pi) def KL_divergence(self, variational_posterior): mu = variational_posterior.mean S = variational_posterior.variance gamma = variational_posterior.binary_prob - var_mean = np.square(mu) - var_S = (S - np.log(S)) + var_mean = np.square(mu)/self.variance + var_S = (S/self.variance - np.log(S)) var_gamma = (gamma*np.log(gamma/self.pi)).sum()+((1-gamma)*np.log((1-gamma)/(1-self.pi))).sum() - return var_gamma+ 0.5 * (gamma* (var_mean + var_S -1)).sum() + return var_gamma+ (gamma* (np.log(self.variance)-1. +var_mean + var_S)).sum()/2. def update_gradients_KL(self, variational_posterior): mu = variational_posterior.mean S = variational_posterior.variance gamma = variational_posterior.binary_prob - gamma.gradient -= np.log((1-self.pi)/self.pi*gamma/(1.-gamma))+(np.square(mu)+S-np.log(S)-1.)/2. - mu.gradient -= gamma*mu - S.gradient -= (1. - (1. / (S))) * gamma /2. - self.pi.gradient = (gamma/self.pi - (1.-gamma)/(1.-self.pi)).sum(axis=0) + gamma.gradient -= np.log((1-self.pi)/self.pi*gamma/(1.-gamma))+((np.square(mu)+S)/self.variance-np.log(S)+np.log(self.variance)-1.)/2. + mu.gradient -= gamma*mu/self.variance + S.gradient -= (1./self.variance - 1./S) * gamma /2. + if len(self.pi)==1: + self.pi.gradient = (gamma/self.pi - (1.-gamma)/(1.-self.pi)).sum() + if len(self.pi.shape)==1: + self.pi.gradient = (gamma/self.pi - (1.-gamma)/(1.-self.pi)).sum(axis=0) + else: + self.pi.gradient = (gamma/self.pi - (1.-gamma)/(1.-self.pi)) class VariationalPosterior(Parameterized): def __init__(self, means=None, variances=None, name='latent space', *a, **kw): From 703dbcabe255e841c5697b7e4b08f3fb2ed963aa Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Mon, 11 Aug 2014 14:57:50 +0100 Subject: [PATCH 52/55] implement the linear kernel with psi2 format --- GPy/kern/_src/linear.py | 185 ++-------------------- GPy/kern/_src/psi_comp/__init__.py | 5 +- GPy/kern/_src/psi_comp/linear_psi_comp.py | 74 +++++++++ 3 files changed, 89 insertions(+), 175 deletions(-) create mode 100644 GPy/kern/_src/psi_comp/linear_psi_comp.py diff --git a/GPy/kern/_src/linear.py b/GPy/kern/_src/linear.py index d64652f1..0577b5c9 100644 --- a/GPy/kern/_src/linear.py +++ b/GPy/kern/_src/linear.py @@ -103,197 +103,36 @@ class Linear(Kern): def gradients_X_diag(self, dL_dKdiag, X): return 2.*self.variances*dL_dKdiag[:,None]*X + def input_sensitivity(self): + return np.ones(self.input_dim) * self.variances + #---------------------------------------# # PSI statistics # #---------------------------------------# def psi0(self, Z, variational_posterior): - if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - return self.psicomp.psicomputations(self.variances, Z, variational_posterior)[0] - else: - return np.sum(self.variances * self._mu2S(variational_posterior), 1) + return self.psicomp.psicomputations(self.variances, Z, variational_posterior)[0] def psi1(self, Z, variational_posterior): - if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - return self.psicomp.psicomputations(self.variances, Z, variational_posterior)[1] - else: - return self.K(variational_posterior.mean, Z) #the variance, it does nothing + return self.psicomp.psicomputations(self.variances, Z, variational_posterior)[1] @Cache_this(limit=1) def psi2(self, Z, variational_posterior): - if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - return self.psicomp.psicomputations(self.variances, Z, variational_posterior)[2] - else: - ZA = Z * self.variances - ZAinner = self._ZAinner(variational_posterior, Z) - return np.dot(ZAinner, ZA.T) + return self.psicomp.psicomputations(self.variances, Z, variational_posterior)[2] def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): - if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - dL_dvar,_,_,_,_ = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variances, Z, variational_posterior) - if self.ARD: - self.variances.gradient = dL_dvar - else: - self.variances.gradient = dL_dvar.sum() + dL_dvar = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variances, Z, variational_posterior)[0] + if self.ARD: + self.variances.gradient = dL_dvar else: - #psi1 - self.update_gradients_full(dL_dpsi1, variational_posterior.mean, Z) - # psi0: - tmp = dL_dpsi0[:, None] * self._mu2S(variational_posterior) - if self.ARD: self.variances.gradient += tmp.sum(0) - else: self.variances.gradient += tmp.sum() - #psi2 - if self.ARD: - tmp = dL_dpsi2[:, :, :, None] * (self._ZAinner(variational_posterior, Z)[:, :, None, :] * Z[None, None, :, :]) - self.variances.gradient += 2.*tmp.sum(0).sum(0).sum(0) - else: - self.variances.gradient += 2.*np.sum(dL_dpsi2 * self.psi2(Z, variational_posterior))/self.variances + self.variances.gradient = dL_dvar.sum() def gradients_Z_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): - if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - _,dL_dZ,_,_,_ = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variances, Z, variational_posterior) - return dL_dZ - else: - #psi1 - grad = self.gradients_X(dL_dpsi1.T, Z, variational_posterior.mean) - #psi2 - self._weave_dpsi2_dZ(dL_dpsi2, Z, variational_posterior, grad) - return grad + return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variances, Z, variational_posterior)[1] def gradients_qX_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): - if isinstance(variational_posterior, variational.SpikeAndSlabPosterior): - _,_,dL_dmu, dL_dS, dL_dgamma = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variances, Z, variational_posterior) - return dL_dmu, dL_dS, dL_dgamma - else: - grad_mu, grad_S = np.zeros(variational_posterior.mean.shape), np.zeros(variational_posterior.mean.shape) - # psi0 - grad_mu += dL_dpsi0[:, None] * (2.0 * variational_posterior.mean * self.variances) - grad_S += dL_dpsi0[:, None] * self.variances - # psi1 - grad_mu += (dL_dpsi1[:, :, None] * (Z * self.variances)).sum(1) - # psi2 - self._weave_dpsi2_dmuS(dL_dpsi2, Z, variational_posterior, grad_mu, grad_S) + return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variances, Z, variational_posterior)[2:] - return grad_mu, grad_S - - #--------------------------------------------------# - # Helpers for psi statistics # - #--------------------------------------------------# - - - def _weave_dpsi2_dmuS(self, dL_dpsi2, Z, vp, target_mu, target_S): - # Think N,num_inducing,num_inducing,input_dim - ZA = Z * self.variances - AZZA = ZA.T[:, None, :, None] * ZA[None, :, None, :] - AZZA = AZZA + AZZA.swapaxes(1, 2) - AZZA_2 = AZZA/2. - if config.getboolean('parallel', 'openmp'): - pragma_string = '#pragma omp parallel for private(m,mm,q,qq,factor,tmp)' - header_string = '#include ' - weave_options = {'headers' : [''], - 'extra_compile_args': ['-fopenmp -O3'], - 'extra_link_args' : ['-lgomp'], - 'libraries': ['gomp']} - else: - pragma_string = '' - header_string = '' - weave_options = {'extra_compile_args': ['-O3']} - - #Using weave, we can exploit the symmetry of this problem: - code = """ - int n, m, mm,q,qq; - double factor,tmp; - %s - for(n=0;n - """ % header_string - mu = vp.mean - N,num_inducing,input_dim,mu = mu.shape[0],Z.shape[0],mu.shape[1],param_to_array(mu) - weave.inline(code, support_code=support_code, - arg_names=['N','num_inducing','input_dim','mu','AZZA','AZZA_2','target_mu','target_S','dL_dpsi2'], - type_converters=weave.converters.blitz,**weave_options) - - - def _weave_dpsi2_dZ(self, dL_dpsi2, Z, vp, target): - AZA = self.variances*self._ZAinner(vp, Z) - - if config.getboolean('parallel', 'openmp'): - pragma_string = '#pragma omp parallel for private(n,mm,q)' - header_string = '#include ' - weave_options = {'headers' : [''], - 'extra_compile_args': ['-fopenmp -O3'], - 'extra_link_args' : ['-lgomp'], - 'libraries': ['gomp']} - else: - pragma_string = '' - header_string = '' - weave_options = {'extra_compile_args': ['-O3']} - - code=""" - int n,m,mm,q; - %s - for(m=0;m - """ % header_string - - N,num_inducing,input_dim = vp.mean.shape[0],Z.shape[0],vp.mean.shape[1] - mu = param_to_array(vp.mean) - weave.inline(code, support_code=support_code, - arg_names=['N','num_inducing','input_dim','AZA','target','dL_dpsi2'], - type_converters=weave.converters.blitz,**weave_options) - - - @Cache_this(limit=1, ignore_args=(0,)) - def _mu2S(self, vp): - return np.square(vp.mean) + vp.variance - - @Cache_this(limit=1) - def _ZAinner(self, vp, Z): - ZA = Z*self.variances - inner = (vp.mean[:, None, :] * vp.mean[:, :, None]) - diag_indices = np.diag_indices(vp.mean.shape[1], 2) - inner[:, diag_indices[0], diag_indices[1]] += vp.variance - - return np.dot(ZA, inner).swapaxes(0, 1) # NOTE: self.ZAinner \in [num_inducing x num_data x input_dim]! - - def input_sensitivity(self): - return np.ones(self.input_dim) * self.variances class LinearFull(Kern): def __init__(self, input_dim, rank, W=None, kappa=None, active_dims=None, name='linear_full'): diff --git a/GPy/kern/_src/psi_comp/__init__.py b/GPy/kern/_src/psi_comp/__init__.py index 1395ad98..48abc68c 100644 --- a/GPy/kern/_src/psi_comp/__init__.py +++ b/GPy/kern/_src/psi_comp/__init__.py @@ -7,6 +7,7 @@ from ....core.parameterization import variational import rbf_psi_comp import ssrbf_psi_comp import sslinear_psi_comp +import linear_psi_comp class PSICOMP_RBF(Pickleable): @@ -33,7 +34,7 @@ class PSICOMP_Linear(Pickleable): @Cache_this(limit=2, ignore_args=(0,)) def psicomputations(self, variance, Z, variational_posterior): if isinstance(variational_posterior, variational.NormalPosterior): - raise NotImplementedError + return linear_psi_comp.psicomputations(variance, Z, variational_posterior) elif isinstance(variational_posterior, variational.SpikeAndSlabPosterior): return sslinear_psi_comp.psicomputations(variance, Z, variational_posterior) else: @@ -42,7 +43,7 @@ class PSICOMP_Linear(Pickleable): @Cache_this(limit=2, ignore_args=(0,1,2,3)) def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior): if isinstance(variational_posterior, variational.NormalPosterior): - raise NotImplementedError + return linear_psi_comp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior) elif isinstance(variational_posterior, variational.SpikeAndSlabPosterior): return sslinear_psi_comp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior) else: diff --git a/GPy/kern/_src/psi_comp/linear_psi_comp.py b/GPy/kern/_src/psi_comp/linear_psi_comp.py new file mode 100644 index 00000000..4f064a59 --- /dev/null +++ b/GPy/kern/_src/psi_comp/linear_psi_comp.py @@ -0,0 +1,74 @@ +# Copyright (c) 2012, GPy authors (see AUTHORS.txt). +# Licensed under the BSD 3-clause license (see LICENSE.txt) + +""" +The package for the Psi statistics computation of the linear kernel for Bayesian GPLVM +""" + +import numpy as np + +def psicomputations(variance, Z, variational_posterior): + """ + Compute psi-statistics for ss-linear kernel + """ + # here are the "statistics" for psi0, psi1 and psi2 + # Produced intermediate results: + # psi0 N + # psi1 NxM + # psi2 MxM + mu = variational_posterior.mean + S = variational_posterior.variance + + psi0 = np.einsum('q,nq->n',variance,np.square(mu)+S) + psi1 = np.einsum('q,mq,nq->nm',variance,Z,mu) + + tmp = np.einsum('q,mq,nq->nm',variance,Z,mu) + psi2 = np.einsum('q,mq,oq,nq->mo',np.square(variance),Z,Z,S) + np.einsum('nm,no->mo',tmp,tmp) + + return psi0, psi1, psi2 + +def psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior): + mu = variational_posterior.mean + S = variational_posterior.variance + + dL_dvar, dL_dmu, dL_dS, dL_dZ = _psi2computations(dL_dpsi2, variance, Z, mu, S) + + # Compute for psi0 and psi1 + mu2S = np.square(mu)+S + dL_dvar += np.einsum('n,nq->q',dL_dpsi0,mu2S) + np.einsum('nm,mq,nq->q',dL_dpsi1,Z,mu) + dL_dmu += np.einsum('n,q,nq->nq',dL_dpsi0,2.*variance,mu) + np.einsum('nm,q,mq->nq',dL_dpsi1,variance,Z) + dL_dS += np.einsum('n,q->nq',dL_dpsi0,variance) + dL_dZ += np.einsum('nm,q,nq->mq',dL_dpsi1, variance,mu) + + return dL_dvar, dL_dZ, dL_dmu, dL_dS + +def _psi2computations(dL_dpsi2, variance, Z, mu, S): + """ + Z - MxQ + mu - NxQ + S - NxQ + gamma - NxQ + """ + # here are the "statistics" for psi1 and psi2 + # Produced intermediate results: + # _psi2_dvariance Q + # _psi2_dZ MxQ + # _psi2_dmu NxQ + # _psi2_dS NxQ + + variance2 = np.square(variance) + common_sum = np.einsum('q,mq,nq->nm',variance,Z,mu) # NxM + + dL_dvar = np.einsum('mo,nq,q,mq,oq->q',dL_dpsi2,2.*S,variance,Z,Z)+\ + np.einsum('mo,mq,nq,no->q',dL_dpsi2,Z,mu,common_sum)+\ + np.einsum('mo,oq,nq,nm->q',dL_dpsi2,Z,mu,common_sum) + + dL_dmu = np.einsum('mo,q,mq,no->nq',dL_dpsi2,variance,Z,common_sum)+\ + np.einsum('mo,q,oq,nm->nq',dL_dpsi2,variance,Z,common_sum) + + dL_dS = np.empty(S.shape) + dL_dS[:] = np.einsum('mo,q,mq,oq->q',dL_dpsi2,variance2,Z,Z) + + dL_dZ = 2.*(np.einsum('om,q,mq,nq->oq',dL_dpsi2,variance2,Z,S)+np.einsum('om,q,nq,nm->oq',dL_dpsi2,variance,mu,common_sum)) + + return dL_dvar, dL_dmu, dL_dS, dL_dZ From c6d0e7f1d1bcf0e4512e3d293ee5051e97f178de Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Mon, 11 Aug 2014 15:00:57 +0100 Subject: [PATCH 53/55] some tidy up --- GPy/kern/_src/linear.py | 3 - GPy/kern/_src/ssrbf.py | 139 ---------------------------------------- GPy/kern/_src/static.py | 1 - 3 files changed, 143 deletions(-) delete mode 100644 GPy/kern/_src/ssrbf.py diff --git a/GPy/kern/_src/linear.py b/GPy/kern/_src/linear.py index 0577b5c9..41ec3cae 100644 --- a/GPy/kern/_src/linear.py +++ b/GPy/kern/_src/linear.py @@ -3,14 +3,11 @@ import numpy as np -from scipy import weave from kern import Kern from ...util.linalg import tdot -from ...util.misc import param_to_array from ...core.parameterization import Param from ...core.parameterization.transformations import Logexp from ...util.caching import Cache_this -from ...core.parameterization import variational from ...util.config import * from .psi_comp import PSICOMP_Linear diff --git a/GPy/kern/_src/ssrbf.py b/GPy/kern/_src/ssrbf.py deleted file mode 100644 index bf87bf76..00000000 --- a/GPy/kern/_src/ssrbf.py +++ /dev/null @@ -1,139 +0,0 @@ -# Copyright (c) 2012, GPy authors (see AUTHORS.txt). -# Licensed under the BSD 3-clause license (see LICENSE.txt) - - -from kern import Kern -import numpy as np -from ...util.linalg import tdot -from ...util.config import * -from stationary import Stationary -from psi_comp import ssrbf_psi_comp - -class SSRBF(Stationary): - """ - Radial Basis Function kernel, aka squared-exponential, exponentiated quadratic or Gaussian kernel - for Spike-and-Slab GPLVM - - .. math:: - - k(r) = \sigma^2 \exp \\bigg(- \\frac{1}{2} r^2 \\bigg) \ \ \ \ \ \\text{ where } r^2 = \sum_{i=1}^d \\frac{ (x_i-x^\prime_i)^2}{\ell_i^2} - - where \ell_i is the lengthscale, \sigma^2 the variance and d the dimensionality of the input. - - :param input_dim: the number of input dimensions - :type input_dim: int - :param variance: the variance of the kernel - :type variance: float - :param lengthscale: the vector of lengthscale of the kernel - :type lengthscale: array or list of the appropriate size (or float if there is only one lengthscale parameter) - :param ARD: Auto Relevance Determination. If equal to "False", the kernel is isotropic (ie. one single lengthscale parameter \ell), otherwise there is one lengthscale parameter per dimension. - :type ARD: Boolean - :rtype: kernel object - - .. Note: this object implements both the ARD and 'spherical' version of the function - """ - - def __init__(self, input_dim, variance=1., lengthscale=None, ARD=True, active_dims=None, name='SSRBF'): - assert ARD==True, "Not Implemented!" - super(SSRBF, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name) - - def K_of_r(self, r): - return self.variance * np.exp(-0.5 * r**2) - - def dK_dr(self, r): - return -r*self.K_of_r(r) - - def parameters_changed(self): - pass - - def Kdiag(self, X): - ret = np.empty(X.shape[0]) - ret[:] = self.variance - return ret - - #---------------------------------------# - # PSI statistics # - #---------------------------------------# - - def psi0(self, Z, variational_posterior): - ret = np.empty(variational_posterior.mean.shape[0]) - ret[:] = self.variance - return ret - - def psi1(self, Z, variational_posterior): - _psi1, _, _, _, _, _, _ = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) - return _psi1 - - def psi2(self, Z, variational_posterior): - _psi2, _, _, _, _, _, _ = ssrbf_psi_comp._psi2computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) - return _psi2 - - def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): - _, _dpsi1_dvariance, _, _, _, _, _dpsi1_dlengthscale = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) - _, _dpsi2_dvariance, _, _, _, _, _dpsi2_dlengthscale = ssrbf_psi_comp._psi2computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) - - #contributions from psi0: - self.variance.gradient = np.sum(dL_dpsi0) - - #from psi1 - self.variance.gradient += np.sum(dL_dpsi1 * _dpsi1_dvariance) - self.lengthscale.gradient = (dL_dpsi1[:,:,None]*_dpsi1_dlengthscale).reshape(-1,self.input_dim).sum(axis=0) - - - #from psi2 - self.variance.gradient += (dL_dpsi2 * _dpsi2_dvariance).sum() - self.lengthscale.gradient += (dL_dpsi2[:,:,:,None] * _dpsi2_dlengthscale).reshape(-1,self.input_dim).sum(axis=0) - - def gradients_Z_expectations(self, dL_dpsi1, dL_dpsi2, Z, variational_posterior): - _, _, _, _, _, _dpsi1_dZ, _ = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) - _, _, _, _, _, _dpsi2_dZ, _ = ssrbf_psi_comp._psi2computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) - - #psi1 - grad = (dL_dpsi1[:, :, None] * _dpsi1_dZ).sum(axis=0) - - #psi2 - grad += (dL_dpsi2[:, :, :, None] * _dpsi2_dZ).sum(axis=0).sum(axis=1) - - return grad - - def gradients_qX_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior): - ndata = variational_posterior.mean.shape[0] - - _, _, _dpsi1_dgamma, _dpsi1_dmu, _dpsi1_dS, _, _ = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) - _, _, _dpsi2_dgamma, _dpsi2_dmu, _dpsi2_dS, _, _ = ssrbf_psi_comp._psi2computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob) - - #psi1 - grad_mu = (dL_dpsi1[:, :, None] * _dpsi1_dmu).sum(axis=1) - grad_S = (dL_dpsi1[:, :, None] * _dpsi1_dS).sum(axis=1) - grad_gamma = (dL_dpsi1[:,:,None] * _dpsi1_dgamma).sum(axis=1) - #psi2 - grad_mu += (dL_dpsi2[:, :, :, None] * _dpsi2_dmu).reshape(ndata,-1,self.input_dim).sum(axis=1) - grad_S += (dL_dpsi2[:, :, :, None] * _dpsi2_dS).reshape(ndata,-1,self.input_dim).sum(axis=1) - grad_gamma += (dL_dpsi2[:,:,:, None] * _dpsi2_dgamma).reshape(ndata,-1,self.input_dim).sum(axis=1) - - return grad_mu, grad_S, grad_gamma - - #---------------------------------------# - # Precomputations # - #---------------------------------------# - - #@cache_this(1) - def _K_computations(self, X, X2): - """ - K(X,X2) - X is NxQ - Q -> input dimension (self.input_dim) - """ - if X2 is None: - self._X2 = None - - X = X / self.lengthscale - Xsquare = np.sum(np.square(X), axis=1) - self._K_dist2 = -2.*tdot(X) + (Xsquare[:, None] + Xsquare[None, :]) - else: - self._X2 = X2.copy() - - X = X / self.lengthscale - X2 = X2 / self.lengthscale - self._K_dist2 = -2.*np.dot(X, X2.T) + (np.sum(np.square(X), axis=1)[:, None] + np.sum(np.square(X2), axis=1)[None, :]) - self._K_dvar = np.exp(-0.5 * self._K_dist2) - diff --git a/GPy/kern/_src/static.py b/GPy/kern/_src/static.py index f8ce7fd1..e191b569 100644 --- a/GPy/kern/_src/static.py +++ b/GPy/kern/_src/static.py @@ -6,7 +6,6 @@ from kern import Kern import numpy as np from ...core.parameterization import Param from ...core.parameterization.transformations import Logexp -import numpy as np class Static(Kern): def __init__(self, input_dim, variance, active_dims, name): From 9f1bd3ef25196c7c446954ab082b10a11695f19a Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Mon, 11 Aug 2014 15:08:08 +0100 Subject: [PATCH 54/55] fix kern/__init__.py --- GPy/kern/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/GPy/kern/__init__.py b/GPy/kern/__init__.py index e28bd6ed..db983309 100644 --- a/GPy/kern/__init__.py +++ b/GPy/kern/__init__.py @@ -8,7 +8,6 @@ from _src.mlp import MLP from _src.periodic import PeriodicExponential, PeriodicMatern32, PeriodicMatern52 from _src.independent_outputs import IndependentOutputs, Hierarchical from _src.coregionalize import Coregionalize -from _src.ssrbf import SSRBF # TODO: ZD: did you remove this? from _src.ODE_UY import ODE_UY from _src.ODE_UYC import ODE_UYC from _src.ODE_st import ODE_st From 1061bf52482aa3bf6769db810c955d5fbf51ceae Mon Sep 17 00:00:00 2001 From: Zhenwen Dai Date: Tue, 12 Aug 2014 10:38:58 +0100 Subject: [PATCH 55/55] change for ssgplvm example --- GPy/examples/dimensionality_reduction.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GPy/examples/dimensionality_reduction.py b/GPy/examples/dimensionality_reduction.py index 3d73a62c..842d0bf8 100644 --- a/GPy/examples/dimensionality_reduction.py +++ b/GPy/examples/dimensionality_reduction.py @@ -317,7 +317,7 @@ def bgplvm_simulation(optimize=True, verbose=1, def ssgplvm_simulation(optimize=True, verbose=1, plot=True, plot_sim=False, - max_iters=2e4, + max_iters=2e4, useGPU=False ): from GPy import kern from GPy.models import SSGPLVM @@ -325,7 +325,7 @@ def ssgplvm_simulation(optimize=True, verbose=1, D1, D2, D3, N, num_inducing, Q = 13, 5, 8, 45, 3, 9 _, _, Ylist = _simulate_sincos(D1, D2, D3, N, num_inducing, Q, plot_sim) Y = Ylist[0] - k = kern.Linear(Q, ARD=True)# + kern.white(Q, _np.exp(-2)) # + kern.bias(Q) + k = kern.Linear(Q, ARD=True, useGPU=useGPU)# + kern.white(Q, _np.exp(-2)) # + kern.bias(Q) #k = kern.RBF(Q, ARD=True, lengthscale=10.) m = SSGPLVM(Y, Q, init="pca", num_inducing=num_inducing, kernel=k) m.X.variance[:] = _np.random.uniform(0,.01,m.X.shape)