diff --git a/GPy/inference/latent_function_inference/exact_gaussian_inference.py b/GPy/inference/latent_function_inference/exact_gaussian_inference.py index 2a0a2592..76b10f08 100644 --- a/GPy/inference/latent_function_inference/exact_gaussian_inference.py +++ b/GPy/inference/latent_function_inference/exact_gaussian_inference.py @@ -64,3 +64,17 @@ class ExactGaussianInference(LatentFunctionInference): dL_dthetaL = likelihood.exact_inference_gradients(np.diag(dL_dK),Y_metadata) return Posterior(woodbury_chol=LW, woodbury_vector=alpha, K=K), log_marginal, {'dL_dK':dL_dK, 'dL_dthetaL':dL_dthetaL, 'dL_dm':alpha} + + def LOO(self, kern, X, Y, likelihood, posterior, Y_metadata=None, K=None): + """ + Leave one out error as found in + "Bayesian leave-one-out cross-validation approximations for Gaussian latent variable models" + Vehtari et al. 2014. + """ + g = posterior.woodbury_vector + c = posterior.woodbury_inv + c_diag = np.diag(c)[:, None] + neg_log_marginal_LOO = 0.5*np.log(2*np.pi) - 0.5*np.log(c_diag) + 0.5*(g**2)/c_diag + #believe from Predictive Approaches for Choosing Hyperparameters in Gaussian Processes + #this is the negative marginal LOO + return -neg_log_marginal_LOO diff --git a/GPy/inference/latent_function_inference/laplace.py b/GPy/inference/latent_function_inference/laplace.py index c6921f57..19d53505 100644 --- a/GPy/inference/latent_function_inference/laplace.py +++ b/GPy/inference/latent_function_inference/laplace.py @@ -19,6 +19,7 @@ def warning_on_one_line(message, category, filename, lineno, file=None, line=Non warnings.formatwarning = warning_on_one_line from scipy import optimize from . import LatentFunctionInference +from scipy.integrate import quad class Laplace(LatentFunctionInference): @@ -39,6 +40,67 @@ class Laplace(LatentFunctionInference): self.first_run = True self._previous_Ki_fhat = None + def LOO(self, kern, X, Y, likelihood, posterior, Y_metadata=None, K=None): + """ + Leave one out log predictive density as found in + "Bayesian leave-one-out cross-validation approximations for Gaussian latent variable models" + Vehtari et al. 2014. + """ + Ki_f_init = np.zeros_like(Y) + + if K is None: + K = kern.K(X) + + f_hat, _ = self.rasm_mode(K, Y, likelihood, Ki_f_init, Y_metadata=Y_metadata) + W = -likelihood.d2logpdf_df2(f_hat, Y, Y_metadata=Y_metadata) + logpdf_dfhat = likelihood.dlogpdf_df(f_hat, Y, Y_metadata=Y_metadata) + + K_Wi_i, _, _, Ki_W_i = self._compute_B_statistics(K, W, likelihood.log_concave) + + #Eq 37 + posterior_cav_var = 1./(1./np.diag(Ki_W_i) - 1./np.diag(W))[:, None] + posterior_cav_mean = f_hat - posterior_cav_var*logpdf_dfhat + + flat_y = Y.flatten() + flat_mu = posterior_cav_mean.flatten() + flat_var = posterior_cav_var.flatten() + + if Y_metadata is not None: + #Need to zip individual elements of Y_metadata aswell + Y_metadata_flat = {} + if Y_metadata is not None: + for key, val in Y_metadata.items(): + Y_metadata_flat[key] = np.atleast_1d(val).reshape(-1, 1) + + zipped_values = [] + + for i in range(Y.shape[0]): + y_m = {} + for key, val in Y_metadata_flat.items(): + if np.isscalar(val) or val.shape[0] == 1: + y_m[key] = val + else: + #Won't broadcast yet + y_m[key] = val[i] + zipped_values.append((flat_y[i], flat_mu[i], flat_var[i], y_m)) + else: + #Otherwise just pass along None's + zipped_values = zip(flat_y, flat_mu, flat_var, [None]*Y.shape[0]) + + def integral_generator(yi, mi, vi, yi_m): + def f(fi_star): + #More stable in the log space + return np.exp(likelihood.logpdf(fi_star, yi, yi_m) + - 0.5*np.log(2*np.pi*vi) + - 0.5*np.square(mi-fi_star)/vi) + return f + + #Eq 25 + p_ystar, _ = zip(*[quad(integral_generator(y, m, v, yi_m), -np.inf, np.inf) + for y, m, v, yi_m in zipped_values]) + p_ystar = np.array(p_ystar).reshape(-1, 1) + return np.log(p_ystar) + def inference(self, kern, X, likelihood, Y, mean_function=None, Y_metadata=None): """ Returns a Posterior class containing essential quantities of the posterior diff --git a/GPy/likelihoods/gaussian.py b/GPy/likelihoods/gaussian.py index 9ecf7dbf..9abb8cde 100644 --- a/GPy/likelihoods/gaussian.py +++ b/GPy/likelihoods/gaussian.py @@ -132,10 +132,8 @@ class Gaussian(Likelihood): :returns: log likelihood evaluated for this point :rtype: float """ - N = y.shape[0] ln_det_cov = np.log(self.variance) - - return -0.5*((y-link_f)**2/self.variance + ln_det_cov + np.log(2.*np.pi)) + return -(1.0/(2*self.variance))*((y-link_f)**2) - 0.5*ln_det_cov - 0.5*np.log(2.*np.pi) def dlogpdf_dlink(self, link_f, y, Y_metadata=None): """ @@ -220,7 +218,6 @@ class Gaussian(Likelihood): """ e = y - link_f s_4 = 1.0/(self.variance**2) - N = y.shape[0] dlik_dsigma = -0.5/self.variance + 0.5*s_4*np.square(e) return dlik_dsigma