From e4a4344334709abcc648acf368d9b1301c7042be Mon Sep 17 00:00:00 2001 From: mzwiessele Date: Tue, 8 Mar 2016 09:47:24 +0000 Subject: [PATCH] [stochastics] update for new stochastic iptimizers in gpy --- GPy/inference/optimization/__init__.py | 5 +- GPy/inference/optimization/stochastics.py | 119 ++++++++++++++++++++++ GPy/models/sparse_gp_minibatch.py | 12 ++- GPy/testing/minibatch_tests.py | 18 ++++ 4 files changed, 152 insertions(+), 2 deletions(-) create mode 100644 GPy/inference/optimization/stochastics.py diff --git a/GPy/inference/optimization/__init__.py b/GPy/inference/optimization/__init__.py index a6247d96..2fa96960 100644 --- a/GPy/inference/optimization/__init__.py +++ b/GPy/inference/optimization/__init__.py @@ -1,5 +1,8 @@ -from paramz.optimization import stochastics, Optimizer +from paramz.optimization import Optimizer +from . import stochastics + from paramz.optimization import * import sys + sys.modules['GPy.inference.optimization.stochastics'] = stochastics sys.modules['GPy.inference.optimization.Optimizer'] = Optimizer diff --git a/GPy/inference/optimization/stochastics.py b/GPy/inference/optimization/stochastics.py new file mode 100644 index 00000000..41f5320b --- /dev/null +++ b/GPy/inference/optimization/stochastics.py @@ -0,0 +1,119 @@ +#=============================================================================== +# Copyright (c) 2015, Max Zwiessele +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of paramax nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#=============================================================================== + +class StochasticStorage(object): + ''' + This is a container for holding the stochastic parameters, + such as subset indices or step length and so on. + + self.d has to be a list of lists: + [dimension indices, nan indices for those dimensions] + so that the minibatches can be used as efficiently as possible. + ''' + def __init__(self, model): + """ + Initialize this stochastic container using the given model + """ + + def do_stochastics(self): + """ + Update the internal state to the next batch of the stochastic + descent algorithm. + """ + pass + + def reset(self): + """ + Reset the state of this stochastics generator. + """ + +class SparseGPMissing(StochasticStorage): + def __init__(self, model, batchsize=1): + """ + Here we want to loop over all dimensions everytime. + Thus, we can just make sure the loop goes over self.d every + time. We will try to get batches which look the same together + which speeds up calculations significantly. + """ + import numpy as np + self.Y = model.Y_normalized + bdict = {} + #For N > 1000 array2string default crops + opt = np.get_printoptions() + np.set_printoptions(threshold=np.inf) + for d in range(self.Y.shape[1]): + inan = np.isnan(self.Y)[:, d] + arr_str = np.array2string(inan, np.inf, 0, True, '', formatter={'bool':lambda x: '1' if x else '0'}) + try: + bdict[arr_str][0].append(d) + except: + bdict[arr_str] = [[d], ~inan] + np.set_printoptions(**opt) + self.d = bdict.values() + +class SparseGPStochastics(StochasticStorage): + """ + For the sparse gp we need to store the dimension we are in, + and the indices corresponding to those + """ + def __init__(self, model, batchsize=1, missing_data=True): + self.batchsize = batchsize + self.output_dim = model.Y.shape[1] + self.Y = model.Y_normalized + self.missing_data = missing_data + self.reset() + self.do_stochastics() + + def do_stochastics(self): + import numpy as np + if self.batchsize == 1: + self.current_dim = (self.current_dim+1)%self.output_dim + self.d = [[[self.current_dim], np.isnan(self.Y[:, self.current_dim]) if self.missing_data else None]] + else: + self.d = np.random.choice(self.output_dim, size=self.batchsize, replace=False) + bdict = {} + if self.missing_data: + opt = np.get_printoptions() + np.set_printoptions(threshold=np.inf) + for d in self.d: + inan = np.isnan(self.Y[:, d]) + arr_str = np.array2string(inan,np.inf, 0,True, '',formatter={'bool':lambda x: '1' if x else '0'}) + try: + bdict[arr_str][0].append(d) + except: + bdict[arr_str] = [[d], ~inan] + np.set_printoptions(**opt) + self.d = bdict.values() + else: + self.d = [[self.d, None]] + + def reset(self): + self.current_dim = -1 + self.d = None diff --git a/GPy/models/sparse_gp_minibatch.py b/GPy/models/sparse_gp_minibatch.py index 6afb19e9..92a340f5 100644 --- a/GPy/models/sparse_gp_minibatch.py +++ b/GPy/models/sparse_gp_minibatch.py @@ -41,6 +41,7 @@ class SparseGPMiniBatch(SparseGP): def __init__(self, X, Y, Z, kernel, likelihood, inference_method=None, name='sparse gp', Y_metadata=None, normalizer=False, missing_data=False, stochastic=False, batchsize=1): + self._update_stochastics = False # pick a sensible inference method if inference_method is None: @@ -73,7 +74,14 @@ class SparseGPMiniBatch(SparseGP): logger.info("Adding Z as parameter") self.link_parameter(self.Z, index=0) self.posterior = None - + + def optimize(self, optimizer=None, start=None, **kwargs): + try: + self._update_stochastics = True + SparseGP.optimize(self, optimizer=optimizer, start=start, **kwargs) + finally: + self._update_stochastics = False + def has_uncertain_inputs(self): return isinstance(self.X, VariationalPosterior) @@ -314,6 +322,8 @@ class SparseGPMiniBatch(SparseGP): if self.missing_data: self._outer_loop_for_missing_data() elif self.stochastics: + if self._update_stochastics: + self.stochastics.do_stochastics() self._outer_loop_without_missing_data() else: self.posterior, self._log_marginal_likelihood, self.grad_dict = self._inner_parameters_changed(self.kern, self.X, self.Z, self.likelihood, self.Y_normalized, self.Y_metadata) diff --git a/GPy/testing/minibatch_tests.py b/GPy/testing/minibatch_tests.py index 6dd1db22..a5e9a884 100644 --- a/GPy/testing/minibatch_tests.py +++ b/GPy/testing/minibatch_tests.py @@ -124,6 +124,24 @@ class SparseGPMinibatchTest(unittest.TestCase): np.testing.assert_allclose(m.gradient, self.m_full.gradient) assert(m.checkgrad()) + def test_sparsegp_init(self): + # Test if the different implementations give the exact same likelihood as the full model. + # All of the following settings should give the same likelihood and gradients as the full model: + np.random.seed(1234) + Z = self.X[np.random.choice(self.X.shape[0], replace=False, size=10)].copy() + Q = Z.shape[1] + m = GPy.models.sparse_gp_minibatch.SparseGPMiniBatch(self.X, self.Y, Z, GPy.kern.RBF(Q)+GPy.kern.Matern32(Q)+GPy.kern.Bias(Q), GPy.likelihoods.Gaussian(), missing_data=True, stochastic=False) + assert(m.checkgrad()) + + m = GPy.models.sparse_gp_minibatch.SparseGPMiniBatch(self.X, self.Y, Z, GPy.kern.RBF(Q)+GPy.kern.Matern32(Q)+GPy.kern.Bias(Q), GPy.likelihoods.Gaussian(), missing_data=True, stochastic=True) + assert(m.checkgrad()) + + m = GPy.models.sparse_gp_minibatch.SparseGPMiniBatch(self.X, self.Y, Z, GPy.kern.RBF(Q)+GPy.kern.Matern32(Q)+GPy.kern.Bias(Q), GPy.likelihoods.Gaussian(), missing_data=False, stochastic=False) + assert(m.checkgrad()) + + m = GPy.models.sparse_gp_minibatch.SparseGPMiniBatch(self.X, self.Y, Z, GPy.kern.RBF(Q)+GPy.kern.Matern32(Q)+GPy.kern.Bias(Q), GPy.likelihoods.Gaussian(), missing_data=False, stochastic=True) + assert(m.checkgrad()) + def test_predict_missing_data(self): m = GPy.models.bayesian_gplvm_minibatch.BayesianGPLVMMiniBatch(self.Y, self.Q, X_variance=False, missing_data=True, stochastic=True, batchsize=self.Y.shape[1]) m[:] = self.m_full[:]