Changes in EP/EPDTC to fix numerical issues and increase the flexibility of the inference.

Changes to avoid numerical issues and improve the performance:
    - Keep value of the EP parameters between calls
    - Enforce positivity of tau_tilde
    - Stable computation of the EP moments for the Bernoulli likelihood
    - Compute marginal in the GP model without directly inverting tau_tilde

    Changes to improve the flexibility:
    - Add parameter for maximum number of iterations
    - Distinguish between alternated/nested mode
    - Distinguish between sequential/parallel updates in EP

GPy/testing/inference_tests.py

@@ -49,6 +49,43 @@ class InferenceXTestCase(unittest.TestCase):
         m.optimize()
         x, mi = m.infer_newX(m.Y, optimize=True)
         np.testing.assert_array_almost_equal(m.X, mi.X, decimal=2)
+class InferenceGPEP(unittest.TestCase):
+
+    def genData(self):
+        np.random.seed(1)
+        k = GPy.kern.RBF(1, variance=7., lengthscale=0.2)
+        X = np.random.rand(200,1)
+        f = np.random.multivariate_normal(np.zeros(200), k.K(X) + 1e-5 * np.eye(X.shape[0]))
+        lik = GPy.likelihoods.Bernoulli()
+        p = lik.gp_link.transf(f) # squash the latent function
+        Y = lik.samples(f).reshape(-1,1)
+        return X, Y
+
+    def test_inference_EP(self):
+        from paramz import ObsAr
+        X, Y = self.genData()
+        lik = GPy.likelihoods.Bernoulli()
+        k = GPy.kern.RBF(1, variance=7., lengthscale=0.2)
+        inf = GPy.inference.latent_function_inference.expectation_propagation.EP(max_iters=30, delta=0.5)
+        self.model = GPy.core.GP(X=X,
+                        Y=Y,
+                        kernel=k,
+                        inference_method=inf,
+                        likelihood=lik)
+        K = self.model.kern.K(X)
+        mu, Sigma, mu_tilde, tau_tilde, log_Z_tilde = self.model.inference_method.expectation_propagation(K, ObsAr(Y), lik, None)
+
+        v_tilde = mu_tilde * tau_tilde
+        p, m, d = self.model.inference_method._inference(K, tau_tilde, v_tilde, lik, Y_metadata=None,  Z_tilde=log_Z_tilde.sum())
+        p0, m0, d0 = super(GPy.inference.latent_function_inference.expectation_propagation.EP, inf).inference(k, X,lik ,mu_tilde[:,None], mean_function=None, variance=1./tau_tilde, K=K, Z_tilde=log_Z_tilde.sum() + np.sum(- 0.5*np.log(tau_tilde) + 0.5*(v_tilde*v_tilde*1./tau_tilde)))
+
+        assert (np.sum(np.array([m - m0,
+                    np.sum(d['dL_dK'] - d0['dL_dK']),
+                    np.sum(d['dL_dthetaL'] - d0['dL_dthetaL']),
+                    np.sum(d['dL_dm'] - d0['dL_dm']),
+                    np.sum(p._woodbury_vector - p0._woodbury_vector),
+                    np.sum(p.woodbury_inv - p0.woodbury_inv)])) < 1e6)
+
 
 class HMCSamplerTest(unittest.TestCase):
 
@@ -64,7 +101,7 @@ class HMCSamplerTest(unittest.TestCase):
 
         hmc = GPy.inference.mcmc.HMC(m,stepsize=1e-2)
         s = hmc.sample(num_samples=3)
-        
+
 class MCMCSamplerTest(unittest.TestCase):
 
     def test_sampling(self):