Merge branch 'newGP' of github.com:SheffieldML/GPy into newGP

Conflicts:
	GPy/likelihoods/EP.py
	GPy/likelihoods/likelihood_functions.py

GPy/likelihoods/EP.py

@@ -1,11 +1,9 @@
 import numpy as np
-import random
 from scipy import stats, linalg
-#from ..core import model
 from ..util.linalg import pdinv,mdot,jitchol
-from ..util.plot import gpplot
+from likelihood import likelihood
 
-class EP:
+class EP(likelihood):
     def __init__(self,data,likelihood_function,epsilon=1e-3,power_ep=[1.,1.]):
         """
         Expectation Propagation
@@ -20,11 +18,10 @@ class EP:
         self.eta, self.delta = power_ep
         self.data = data
         self.N = self.data.size
+        self.is_heteroscedastic = True
 
-        """
-        Initial values - Likelihood approximation parameters:
-        p(y|f) = t(f|tau_tilde,v_tilde)
-        """
+        #Initial values - Likelihood approximation parameters:
+        #p(y|f) = t(f|tau_tilde,v_tilde)
         self.tau_tilde = np.zeros(self.N)
         self.v_tilde = np.zeros(self.N)
 
@@ -51,9 +48,11 @@ class EP:
         mu_tilde = self.v_tilde/self.tau_tilde #When calling EP, this variable is used instead of Y in the GP model
         sigma_sum = 1./self.tau_ + 1./self.tau_tilde
         mu_diff_2 = (self.v_/self.tau_ - mu_tilde)**2
-        Z_ep = np.sum(np.log(self.Z_hat)) + 0.5*np.sum(np.log(sigma_sum)) + 0.5*np.sum(mu_diff_2/sigma_sum) #Normalization constant
-        self.Y, self.beta, self.Z =  mu_tilde[:,None],self.tau_tilde[:,None], Z_ep
-        self.variance = np.diag(1./self.beta.flatten())
+        self.Z = np.sum(np.log(self.Z_hat)) + 0.5*np.sum(np.log(sigma_sum)) + 0.5*np.sum(mu_diff_2/sigma_sum) #Normalization constant, aka Z_ep
+
+        self.Y =  mu_tilde[:,None]
+        self.precsion = self.tau_tilde[:,None]
+        self.covariance_matrix = np.diag(1./self.precision)
 
     def fit_full(self,K):
         """

GPy/likelihoods/Gaussian.py

@@ -1,7 +1,9 @@
 import numpy as np
+from likelihood import likelihood
 
-class Gaussian:
+class Gaussian(likelihood):
     def __init__(self,data,variance=1.,normalize=False):
+        self.is_heteroscedastic = False
         self.data = data
         self.N,D = data.shape
         self.Z = 0. # a correction factor which accounts for the approximation made
@@ -19,6 +21,7 @@ class Gaussian:
         self.YYT = np.dot(self.Y,self.Y.T)
         self._set_params(np.asarray(variance))
 
+
     def _get_params(self):
         return np.asarray(self._variance)
 
@@ -27,7 +30,8 @@ class Gaussian:
 
     def _set_params(self,x):
         self._variance = x
-        self.variance = np.eye(self.N)*self._variance
+        self.covariance_matrix = np.eye(self.N)*self._variance
+        self.precision = 1./self._variance
 
     def predictive_values(self,mu,var):
         """

GPy/likelihoods/likelihood.py

@@ -0,0 +1,35 @@
+import numpy as np
+
+class likelihood:
+    """
+    The atom for a likelihood class
+
+    This object interfaces the GP and the data. The most basic likelihood
+    (Gaussian) inherits directly from this, as does the EP algorithm
+
+    Some things must be defined for this to work properly:
+    self.Y : the effective Gaussian target of the GP
+    self.N, self.D : Y.shape
+    self.covariance_matrix : the effective (noise) covariance of the GP targets
+    self.Z : a factor which gets added to the likelihood (0 for a Gaussian, Z_EP for EP)
+    self.is_heteroscedastic : enables significant computational savings in GP
+    self.precision : a scalar or vector representation of the effective target precision
+    self.YYT : (optional) = np.dot(self.Y, self.Y.T) enables computational savings for D>N
+    """
+    def __init__(self,data):
+        raise ValueError, "this class is not to be instantiated"
+
+    def _get_params(self):
+        raise NotImplementedError
+
+    def _get_param_names(self):
+        raise NotImplementedError
+
+    def _set_params(self,x):
+        raise NotImplementedError
+
+    def fit(self):
+        raise NotImplementedError
+
+    def _gradients(self,partial):
+        raise NotImplementedError

GPy/likelihoods/likelihood_functions.py

@@ -9,18 +9,22 @@ import pylab as pb
 from ..util.plot import gpplot
 #from . import EP
 
-class likelihood:
+class likelihood_function:
     """
     Likelihood class for doing Expectation propagation
 
     :param Y: observed output (Nx1 numpy.darray)
-    ..Note:: Y values allowed depend on the likelihood used
+    ..Note:: Y values allowed depend on the likelihood_function used
     """
     def __init__(self,location=0,scale=1):
         self.location = location
         self.scale = scale
 
+<<<<<<< HEAD
 class Probit(likelihood):
+=======
+class probit(likelihood_function):
+>>>>>>> 346f9dd8bd3207959b87ded258e55aeb094f1ea3
     """
     Probit likelihood
     Y is expected to take values in {-1,1}
@@ -29,6 +33,11 @@ class Probit(likelihood):
     L(x) = \\Phi (Y_i*f_i)
     $$
     """
+<<<<<<< HEAD
+=======
+    def __init__(self,location=0,scale=1):
+        likelihood_function.__init__(self,Y,location,scale)
+>>>>>>> 346f9dd8bd3207959b87ded258e55aeb094f1ea3
 
     def moments_match(self,data_i,tau_i,v_i):
         """
@@ -57,7 +66,11 @@ class Probit(likelihood):
         p_95 = np.ones([mu.size])
         return mean, p_05, p_95
 
+<<<<<<< HEAD
 class Poisson(likelihood):
+=======
+class poisson(likelihood_function):
+>>>>>>> 346f9dd8bd3207959b87ded258e55aeb094f1ea3
     """
     Poisson likelihood
     Y is expected to take values in {0,1,2,...}
@@ -66,6 +79,12 @@ class Poisson(likelihood):
     L(x) = \exp(\lambda) * \lambda**Y_i / Y_i!
     $$
     """
+<<<<<<< HEAD
+=======
+    def __init__(self,Y,location=0,scale=1):
+        assert len(Y[Y<0]) == 0, "Output cannot have negative values"
+        likelihood_function.__init__(self,Y,location,scale)
+>>>>>>> 346f9dd8bd3207959b87ded258e55aeb094f1ea3
 
     def moments_match(self,i,tau_i,v_i):
         """
@@ -129,7 +148,54 @@ class Poisson(likelihood):
         Compute  mean, and conficence interval (percentiles 5 and 95) of the  prediction
         """
         mean = np.exp(mu*self.scale + self.location)
+<<<<<<< HEAD
         tmp = stats.poisson.ppf(np.array([.05,.95]),mu)
         p_05 = tmp[:,0]
         p_95 = tmp[:,1]
         return mean,p_05,p_95
+=======
+        if all:
+            tmp = stats.poisson.ppf(np.array([.05,.95]),mu)
+            p_05 = tmp[:,0]
+            p_95 = tmp[:,1]
+            return mean,mean,p_05,p_95
+        else:
+            return mean
+
+    def _log_likelihood_gradients():
+        raise NotImplementedError
+
+    def plot(self,X,mu,var,phi,X_obs,Z=None,samples=0):
+        assert X_obs.shape[1] == 1, 'Number of dimensions must be 1'
+        gpplot(X,phi,phi.flatten())
+        pb.plot(X_obs,self.Y,'kx',mew=1.5)
+        if samples:
+            phi_samples = np.vstack([np.random.poisson(phi.flatten(),phi.size) for s in range(samples)])
+            pb.plot(X,phi_samples.T, alpha = 0.4, c='#3465a4', linewidth = 0.8)
+        if Z is not None:
+            pb.plot(Z,Z*0+pb.ylim()[0],'k|',mew=1.5,markersize=12)
+
+class gaussian(likelihood_function):
+    """
+    Gaussian likelihood
+    Y is expected to take values in (-inf,inf)
+    """
+    def moments_match(self,i,tau_i,v_i):
+        """
+        Moments match of the marginal approximation in EP algorithm
+
+        :param i: number of observation (int)
+        :param tau_i: precision of the cavity distribution (float)
+        :param v_i: mean/variance of the cavity distribution (float)
+        """
+        mu = v_i/tau_i
+        sigma = np.sqrt(1./tau_i)
+        s = 1. if self.Y[i] == 0 else 1./self.Y[i]
+        sigma2_hat = 1./(1./sigma**2 + 1./s**2)
+        mu_hat = sigma2_hat*(mu/sigma**2 + self.Y[i]/s**2)
+        Z_hat = 1./np.sqrt(2*np.pi) * 1./np.sqrt(sigma**2+s**2) * np.exp(-.5*(mu-self.Y[i])**2/(sigma**2 + s**2))
+        return Z_hat, mu_hat, sigma2_hat
+
+    def _log_likelihood_gradients():
+        raise NotImplementedError
+>>>>>>> 346f9dd8bd3207959b87ded258e55aeb094f1ea3