From 7b2af57aee34e6e09dfdc32fd95736786c5738e1 Mon Sep 17 00:00:00 2001 From: Moreno Date: Wed, 16 May 2018 08:53:55 +0100 Subject: [PATCH] Sparse GP serialization --- GPy/core/sparse_gp.py | 39 ++++++ .../expectation_propagation.py | 14 +-- GPy/models/sparse_gp_classification.py | 27 +++- GPy/testing/serialization_tests.py | 118 ++++++++++++------ 4 files changed, 146 insertions(+), 52 deletions(-) diff --git a/GPy/core/sparse_gp.py b/GPy/core/sparse_gp.py index 4c7e98c4..820b16e7 100644 --- a/GPy/core/sparse_gp.py +++ b/GPy/core/sparse_gp.py @@ -117,3 +117,42 @@ class SparseGP(GP): self.Z.gradient = self.kern.gradients_X(self.grad_dict['dL_dKmm'], self.Z) self.Z.gradient += self.kern.gradients_X(self.grad_dict['dL_dKnm'].T, self.Z, self.X) self._Zgrad = self.Z.gradient.copy() + + def to_dict(self, save_data=True): + input_dict = super(SparseGP, self).to_dict(save_data) + input_dict["class"] = "GPy.core.SparseGP" + input_dict["Z"] = self.Z.tolist() + return input_dict + + @staticmethod + def _from_dict(input_dict, data=None): + import GPy + if (input_dict['X'] is None) or (input_dict['Y'] is None): + assert(data is not None) + input_dict["X"], input_dict["Y"] = np.array(data[0]), np.array(data[1]) + elif data is not None: + print("WARNING: The model has been saved with X,Y! The original values are being overriden!") + input_dict["X"], input_dict["Y"] = np.array(data[0]), np.array(data[1]) + else: + input_dict["X"], input_dict["Y"] = np.array(input_dict['X']), np.array(input_dict['Y']) + + input_dict["Z"] = np.array(input_dict['Z']) + input_dict["kernel"] = GPy.kern.Kern.from_dict(input_dict["kernel"]) + input_dict["likelihood"] = GPy.likelihoods.likelihood.Likelihood.from_dict(input_dict["likelihood"]) + mean_function = input_dict.get("mean_function") + if mean_function is not None: + input_dict["mean_function"] = GPy.core.mapping.Mapping.from_dict(mean_function) + else: + input_dict["mean_function"] = mean_function + input_dict["inference_method"] = GPy.inference.latent_function_inference.LatentFunctionInference.from_dict(input_dict["inference_method"]) + + #FIXME: Assumes the Y_metadata is serializable. We should create a Metadata class + Y_metadata = input_dict.get("Y_metadata") + input_dict["Y_metadata"] = Y_metadata + + normalizer = input_dict.get("normalizer") + if normalizer is not None: + input_dict["normalizer"] = GPy.util.normalizer._Norm.from_dict(normalizer) + else: + input_dict["normalizer"] = normalizer + return SparseGP(**input_dict) diff --git a/GPy/inference/latent_function_inference/expectation_propagation.py b/GPy/inference/latent_function_inference/expectation_propagation.py index 61d3feff..635a1bbd 100644 --- a/GPy/inference/latent_function_inference/expectation_propagation.py +++ b/GPy/inference/latent_function_inference/expectation_propagation.py @@ -402,7 +402,7 @@ class EP(EPBase, ExactGaussianInference): class EPDTC(EPBase, VarDTC): def inference(self, kern, X, Z, likelihood, Y, mean_function=None, Y_metadata=None, Lm=None, dL_dKmm=None, psi0=None, psi1=None, psi2=None): - if self.always_reset: + if self.always_reset and not self.loading: self.reset() num_data, output_dim = Y.shape @@ -420,11 +420,11 @@ class EPDTC(EPBase, VarDTC): else: Kmn = psi1.T - if self.ep_mode=="nested": + if self.ep_mode=="nested" and not self.loading: #Force EP at each step of the optimization self._ep_approximation = None post_params, ga_approx, log_Z_tilde = self._ep_approximation = self.expectation_propagation(Kmm, Kmn, Y, likelihood, Y_metadata) - elif self.ep_mode=="alternated": + elif self.ep_mode=="alternated" or self.loading: if getattr(self, '_ep_approximation', None) is None: #if we don't yet have the results of runnign EP, run EP and store the computed factors in self._ep_approximation post_params, ga_approx, log_Z_tilde = self._ep_approximation = self.expectation_propagation(Kmm, Kmn, Y, likelihood, Y_metadata) @@ -434,6 +434,8 @@ class EPDTC(EPBase, VarDTC): else: raise ValueError("ep_mode value not valid") + self.loading = False + mu_tilde = ga_approx.v / ga_approx.tau.astype(float) return super(EPDTC, self).inference(kern, X, Z, likelihood, ObsAr(mu_tilde[:,None]), @@ -551,8 +553,7 @@ class EPDTC(EPBase, VarDTC): input_dict["_ep_approximation"] = {} input_dict["_ep_approximation"]["post_params"] = self._ep_approximation[0].to_dict() input_dict["_ep_approximation"]["ga_approx"] = self._ep_approximation[1].to_dict() - input_dict["_ep_approximation"]["cav_params"] = self._ep_approximation[2].to_dict() - input_dict["_ep_approximation"]["log_Z_tilde"] = self._ep_approximation[3].tolist() + input_dict["_ep_approximation"]["log_Z_tilde"] = self._ep_approximation[2] return input_dict @@ -566,8 +567,7 @@ class EPDTC(EPBase, VarDTC): if _ep_approximation is not None: _ep_approximation.append(posteriorParamsDTC.from_dict(_ep_approximation_dict["post_params"])) _ep_approximation.append(gaussianApproximation.from_dict(_ep_approximation_dict["ga_approx"])) - _ep_approximation.append(cavityParams.from_dict(_ep_approximation_dict["cav_params"])) - _ep_approximation.append(np.array(_ep_approximation_dict["log_Z_tilde"])) + _ep_approximation.append(_ep_approximation_dict["log_Z_tilde"]) ee = EPDTC(**input_dict) ee.ga_approx_old = ga_approx_old ee._ep_approximation = _ep_approximation diff --git a/GPy/models/sparse_gp_classification.py b/GPy/models/sparse_gp_classification.py index a996732a..7a86e1d5 100644 --- a/GPy/models/sparse_gp_classification.py +++ b/GPy/models/sparse_gp_classification.py @@ -7,6 +7,7 @@ from ..core import SparseGP from .. import likelihoods from .. import kern from ..inference.latent_function_inference import EPDTC +from copy import deepcopy class SparseGPClassification(SparseGP): """ @@ -40,6 +41,27 @@ class SparseGPClassification(SparseGP): SparseGP.__init__(self, X, Y, Z, kernel, likelihood, inference_method=EPDTC(), name='SparseGPClassification',Y_metadata=Y_metadata) + @staticmethod + def from_sparse_gp(sparse_gp): + from copy import deepcopy + sparse_gp = deepcopy(sparse_gp) + SparseGPClassification(sparse_gp.X, sparse_gp.Y, sparse_gp.Z, sparse_gp.kern, sparse_gp.likelihood, sparse_gp.inference_method, sparse_gp.mean_function, name='sparse_gp_classification') + + def to_dict(self, save_data=True): + model_dict = super(SparseGPClassification,self).to_dict(save_data) + model_dict["class"] = "GPy.models.SparseGPClassification" + return model_dict + + @staticmethod + def from_dict(input_dict, data=None): + import GPy + m = GPy.core.model.Model.from_dict(input_dict, data) + return GPClassification.from_sparse_gp(m) + + def save_model(self, output_filename, compress=True, save_data=True): + self._save_model(output_filename, compress=True, save_data=True) + + class SparseGPClassificationUncertainInput(SparseGP): """ Sparse Gaussian Process model for classification with uncertain inputs. @@ -87,8 +109,3 @@ class SparseGPClassificationUncertainInput(SparseGP): self.psi2 = self.kern.psi2n(self.Z, self.X) self.posterior, self._log_marginal_likelihood, self.grad_dict = self.inference_method.inference(self.kern, self.X, self.Z, self.likelihood, self.Y, self.Y_metadata, psi0=self.psi0, psi1=self.psi1, psi2=self.psi2) self._update_gradients() - - - - - diff --git a/GPy/testing/serialization_tests.py b/GPy/testing/serialization_tests.py index 7eb3fe5c..57781bf3 100644 --- a/GPy/testing/serialization_tests.py +++ b/GPy/testing/serialization_tests.py @@ -11,6 +11,7 @@ import tempfile import GPy from nose import SkipTest import numpy as np +import os fixed_seed = 11 @@ -116,46 +117,38 @@ class Test(unittest.TestCase): np.testing.assert_array_equal(e1._ep_approximation[2].v[:], e1_r._ep_approximation[2].v[:]) np.testing.assert_array_equal(e1._ep_approximation[3][:], e1_r._ep_approximation[3][:]) - - e1 = GPy.inference.latent_function_inference.expectation_propagation.EPDTC(ep_mode="nested") - e1.ga_approx_old = GPy.inference.latent_function_inference.expectation_propagation.gaussianApproximation(np.random.rand(10),np.random.rand(10)) - e1._ep_approximation = [] - e1._ep_approximation.append(GPy.inference.latent_function_inference.expectation_propagation.posteriorParamsDTC(np.random.rand(10),np.random.rand(10))) - e1._ep_approximation.append(GPy.inference.latent_function_inference.expectation_propagation.gaussianApproximation(np.random.rand(10),np.random.rand(10))) - e1._ep_approximation.append(GPy.inference.latent_function_inference.expectation_propagation.cavityParams(10)) - e1._ep_approximation[-1].v = np.random.rand(10) - e1._ep_approximation[-1].tau = np.random.rand(10) - e1._ep_approximation.append(np.random.rand(10)) - e1_r = GPy.inference.latent_function_inference.LatentFunctionInference.from_dict(e1.to_dict()) - - - assert type(e1) == type(e1_r) - assert e1.epsilon==e1_r.epsilon - assert e1.eta==e1_r.eta - assert e1.delta==e1_r.delta - assert e1.always_reset==e1_r.always_reset - assert e1.max_iters==e1_r.max_iters - assert e1.ep_mode==e1_r.ep_mode - assert e1.parallel_updates==e1_r.parallel_updates - - np.testing.assert_array_equal(e1.ga_approx_old.tau[:], e1_r.ga_approx_old.tau[:]) - np.testing.assert_array_equal(e1.ga_approx_old.v[:], e1_r.ga_approx_old.v[:]) - np.testing.assert_array_equal(e1._ep_approximation[0].mu[:], e1_r._ep_approximation[0].mu[:]) - np.testing.assert_array_equal(e1._ep_approximation[0].Sigma_diag[:], e1_r._ep_approximation[0].Sigma_diag[:]) - np.testing.assert_array_equal(e1._ep_approximation[1].tau[:], e1_r._ep_approximation[1].tau[:]) - np.testing.assert_array_equal(e1._ep_approximation[1].v[:], e1_r._ep_approximation[1].v[:]) - np.testing.assert_array_equal(e1._ep_approximation[2].tau[:], e1_r._ep_approximation[2].tau[:]) - np.testing.assert_array_equal(e1._ep_approximation[2].v[:], e1_r._ep_approximation[2].v[:]) - np.testing.assert_array_equal(e1._ep_approximation[3][:], e1_r._ep_approximation[3][:]) - - - e2 = GPy.inference.latent_function_inference.exact_gaussian_inference.ExactGaussianInference() + e2 = GPy.inference.latent_function_inference.expectation_propagation.EPDTC(ep_mode="nested") + e2.ga_approx_old = GPy.inference.latent_function_inference.expectation_propagation.gaussianApproximation(np.random.rand(10),np.random.rand(10)) + e2._ep_approximation = [] + e2._ep_approximation.append(GPy.inference.latent_function_inference.expectation_propagation.posteriorParamsDTC(np.random.rand(10),np.random.rand(10))) + e2._ep_approximation.append(GPy.inference.latent_function_inference.expectation_propagation.gaussianApproximation(np.random.rand(10),np.random.rand(10))) + e2._ep_approximation.append(100.0) e2_r = GPy.inference.latent_function_inference.LatentFunctionInference.from_dict(e2.to_dict()) assert type(e2) == type(e2_r) + assert e2.epsilon==e2_r.epsilon + assert e2.eta==e2_r.eta + assert e2.delta==e2_r.delta + assert e2.always_reset==e2_r.always_reset + assert e2.max_iters==e2_r.max_iters + assert e2.ep_mode==e2_r.ep_mode + assert e2.parallel_updates==e2_r.parallel_updates + + np.testing.assert_array_equal(e2.ga_approx_old.tau[:], e2_r.ga_approx_old.tau[:]) + np.testing.assert_array_equal(e2.ga_approx_old.v[:], e2_r.ga_approx_old.v[:]) + np.testing.assert_array_equal(e2._ep_approximation[0].mu[:], e2_r._ep_approximation[0].mu[:]) + np.testing.assert_array_equal(e2._ep_approximation[0].Sigma_diag[:], e2_r._ep_approximation[0].Sigma_diag[:]) + np.testing.assert_array_equal(e2._ep_approximation[1].tau[:], e2_r._ep_approximation[1].tau[:]) + np.testing.assert_array_equal(e2._ep_approximation[1].v[:], e2_r._ep_approximation[1].v[:]) + assert(e2._ep_approximation[2] == e2_r._ep_approximation[2]) + + e3 = GPy.inference.latent_function_inference.exact_gaussian_inference.ExactGaussianInference() + e3_r = GPy.inference.latent_function_inference.LatentFunctionInference.from_dict(e3.to_dict()) + + assert type(e3) == type(e3_r) - def test_serialize_deserialize_model(self): + def test_serialize_deserialize_GP(self): np.random.seed(fixed_seed) N = 20 Nhalf = int(N/2) @@ -165,13 +158,13 @@ class Test(unittest.TestCase): likelihood = GPy.likelihoods.Bernoulli() inference_method=GPy.inference.latent_function_inference.expectation_propagation.EP(ep_mode="nested") mean_function=None + m = GPy.core.GP(X=X, Y=Y, kernel=kernel, likelihood=likelihood, inference_method=inference_method, mean_function=mean_function, normalizer=True, name='gp_classification') m.optimize() m.save_model("temp_test_gp_with_data.json", compress=True, save_data=True) m.save_model("temp_test_gp_without_data.json", compress=True, save_data=False) m1_r = GPy.core.GP.load_model("temp_test_gp_with_data.json.zip") m2_r = GPy.core.GP.load_model("temp_test_gp_without_data.json.zip", (X,Y)) - import os os.remove("temp_test_gp_with_data.json.zip") os.remove("temp_test_gp_without_data.json.zip") var = m.predict(X)[0] @@ -180,7 +173,32 @@ class Test(unittest.TestCase): np.testing.assert_array_equal(np.array(var).flatten(), np.array(var1_r).flatten()) np.testing.assert_array_equal(np.array(var).flatten(), np.array(var2_r).flatten()) - def test_serialize_deserialize_inference_GPRegressor(self): + def test_serialize_deserialize_SparseGP(self): + np.random.seed(fixed_seed) + N = 20 + Nhalf = int(N/2) + X = np.hstack([np.random.normal(5, 2, Nhalf), np.random.normal(10, 2, Nhalf)])[:, None] + Y = np.hstack([np.ones(Nhalf), np.zeros(Nhalf)])[:, None] + kernel = GPy.kern.RBF(1) + likelihood = GPy.likelihoods.Bernoulli() + inference_method=GPy.inference.latent_function_inference.expectation_propagation.EPDTC(ep_mode="nested") + mean_function=None + + sm = GPy.core.SparseGP(X=X, Y=Y, Z=X[0:20,:], kernel=kernel, likelihood=likelihood, inference_method=inference_method, mean_function=mean_function, normalizer=True, name='sparse_gp_classification') + sm.optimize() + sm.save_model("temp_test_gp_with_data.json", compress=True, save_data=True) + sm.save_model("temp_test_gp_without_data.json", compress=True, save_data=False) + sm1_r = GPy.core.GP.load_model("temp_test_gp_with_data.json.zip") + sm2_r = GPy.core.GP.load_model("temp_test_gp_without_data.json.zip", (X,Y)) + os.remove("temp_test_gp_with_data.json.zip") + os.remove("temp_test_gp_without_data.json.zip") + var = sm.predict(X)[0] + var1_r = sm1_r.predict(X)[0] + var2_r = sm2_r.predict(X)[0] + np.testing.assert_array_equal(np.array(var).flatten(), np.array(var1_r).flatten()) + np.testing.assert_array_equal(np.array(var).flatten(), np.array(var2_r).flatten()) + + def test_serialize_deserialize_GPRegressor(self): np.random.seed(fixed_seed) N = 50 N_new = 50 @@ -195,7 +213,6 @@ class Test(unittest.TestCase): m.save_model("temp_test_gp_regressor_without_data.json", compress=True, save_data=False) m1_r = GPy.models.GPRegression.load_model("temp_test_gp_regressor_with_data.json.zip") m2_r = GPy.models.GPRegression.load_model("temp_test_gp_regressor_without_data.json.zip", (X,Y)) - import os os.remove("temp_test_gp_regressor_with_data.json.zip") os.remove("temp_test_gp_regressor_without_data.json.zip") @@ -208,7 +225,7 @@ class Test(unittest.TestCase): np.testing.assert_array_equal(var.flatten(), var1_r.flatten()) np.testing.assert_array_equal(var.flatten(), var2_r.flatten()) - def test_serialize_deserialize_inference_GPClassifier(self): + def test_serialize_deserialize_GPClassification(self): np.random.seed(fixed_seed) N = 50 Nhalf = int(N/2) @@ -221,7 +238,6 @@ class Test(unittest.TestCase): m.save_model("temp_test_gp_classifier_without_data.json", compress=True, save_data=False) m1_r = GPy.models.GPClassification.load_model("temp_test_gp_classifier_with_data.json.zip") m2_r = GPy.models.GPClassification.load_model("temp_test_gp_classifier_without_data.json.zip", (X,Y)) - import os os.remove("temp_test_gp_classifier_with_data.json.zip") os.remove("temp_test_gp_classifier_without_data.json.zip") @@ -231,6 +247,28 @@ class Test(unittest.TestCase): np.testing.assert_array_equal(np.array(var).flatten(), np.array(var1_r).flatten()) np.testing.assert_array_equal(np.array(var).flatten(), np.array(var1_r).flatten()) + def test_serialize_deserialize_SparseGPClassification(self): + np.random.seed(fixed_seed) + N = 50 + Nhalf = int(N/2) + X = np.hstack([np.random.normal(5, 2, Nhalf), np.random.normal(10, 2, Nhalf)])[:, None] + Y = np.hstack([np.ones(Nhalf), np.zeros(Nhalf)])[:, None] + kernel = GPy.kern.RBF(1) + m = GPy.models.SparseGPClassification(X, Y, num_inducing=3, kernel=kernel) + m.optimize() + m.save_model("temp_test_sparse_gp_classifier_with_data.json", compress=True, save_data=True) + m.save_model("temp_test_sparse_gp_classifier_without_data.json", compress=True, save_data=False) + m1_r = GPy.models.SparseGPClassification.load_model("temp_test_sparse_gp_classifier_with_data.json.zip") + m2_r = GPy.models.SparseGPClassification.load_model("temp_test_sparse_gp_classifier_without_data.json.zip", (X,Y)) + os.remove("temp_test_sparse_gp_classifier_with_data.json.zip") + os.remove("temp_test_sparse_gp_classifier_without_data.json.zip") + + var = m.predict(X)[0] + var1_r = m1_r.predict(X)[0] + var2_r = m2_r.predict(X)[0] + np.testing.assert_array_equal(np.array(var).flatten(), np.array(var1_r).flatten()) + np.testing.assert_array_equal(np.array(var).flatten(), np.array(var1_r).flatten()) + if __name__ == "__main__": #import sys;sys.argv = ['', 'Test.test_parameter_index_operations'] unittest.main()