diff --git a/GPy/core/model.py b/GPy/core/model.py index 908b70f6..d61b9b43 100644 --- a/GPy/core/model.py +++ b/GPy/core/model.py @@ -10,6 +10,8 @@ import multiprocessing as mp import numpy as np from numpy.linalg.linalg import LinAlgError import itertools +import sys +from .verbose_optimization import VerboseOptimization # import numdifftools as ndt class Model(Parameterized): @@ -24,6 +26,7 @@ class Model(Parameterized): from .parameterization.ties_and_remappings import Tie self.tie = Tie() self.link_parameter(self.tie, -1) + self.obj_grads = None self.add_observer(self.tie, self.tie._parameters_changed_notification, priority=-500) def log_likelihood(self): @@ -165,14 +168,14 @@ class Model(Parameterized): try: # self._set_params_transformed(x) self.optimizer_array = x - obj_grads = self._transform_gradients(self.objective_function_gradients()) + self.obj_grads = self._transform_gradients(self.objective_function_gradients()) self._fail_count = 0 except (LinAlgError, ZeroDivisionError, ValueError): if self._fail_count >= self._allowed_failures: raise self._fail_count += 1 - obj_grads = np.clip(self._transform_gradients(self.objective_function_gradients()), -1e100, 1e100) - return obj_grads + self.obj_grads = np.clip(self._transform_gradients(self.objective_function_gradients()), -1e100, 1e100) + return self.obj_grads def _objective(self, x): """ @@ -200,17 +203,17 @@ class Model(Parameterized): def _objective_and_grads(self, x): try: self.optimizer_array = x - obj_f, obj_grads = self.objective_function(), self._transform_gradients(self.objective_function_gradients()) + obj_f, self.obj_grads = self.objective_function(), self._transform_gradients(self.objective_function_gradients()) self._fail_count = 0 except (LinAlgError, ZeroDivisionError, ValueError): if self._fail_count >= self._allowed_failures: raise self._fail_count += 1 obj_f = np.inf - obj_grads = np.clip(self._transform_gradients(self.objective_function_gradients()), -1e10, 1e10) - return obj_f, obj_grads + self.obj_grads = np.clip(self._transform_gradients(self.objective_function_gradients()), -1e10, 1e10) + return obj_f, self.obj_grads - def optimize(self, optimizer=None, start=None, **kwargs): + def optimize(self, optimizer=None, start=None, messages=False, max_iters=1000, ipython_notebook=False, **kwargs): """ Optimize the model using self.log_likelihood and self.log_likelihood_gradient, as well as self.priors. @@ -218,8 +221,8 @@ class Model(Parameterized): :param max_f_eval: maximum number of function evaluations :type max_f_eval: int - :messages: whether to display during optimisation - :type messages: bool + :messages: True: Display messages during optimisation, "ipython_notebook": + :type messages: bool"string :param optimizer: which optimizer to use (defaults to self.preferred optimizer) :type optimizer: string @@ -251,9 +254,10 @@ class Model(Parameterized): opt.model = self else: optimizer = optimization.get_optimizer(optimizer) - opt = optimizer(start, model=self, **kwargs) + opt = optimizer(start, model=self, max_iters=max_iters, **kwargs) - opt.run(f_fp=self._objective_and_grads, f=self._objective, fp=self._objective_grads) + with VerboseOptimization(self, maxiters=max_iters, verbose=messages, ipython_notebook=ipython_notebook): + opt.run(f_fp=self._objective_and_grads, f=self._objective, fp=self._objective_grads) self.optimization_runs.append(opt) diff --git a/GPy/core/parameterization/parameter_core.py b/GPy/core/parameterization/parameter_core.py index a15d8d53..656bd1c5 100644 --- a/GPy/core/parameterization/parameter_core.py +++ b/GPy/core/parameterization/parameter_core.py @@ -683,7 +683,7 @@ class OptimizationHandlable(Indexable): [np.put(g, i, c.gradfactor(self.param_array[i], g[i])) for c, i in self.constraints.iteritems() if c != __fixed__] if self._has_fixes(): return g[self._fixes_] return g - + def _transform_gradients_non_natural(self, g): """ Transform the gradients by multiplying the gradient factor for each @@ -809,7 +809,7 @@ class Parameterizable(OptimizationHandlable): A parameterisable class. This class provides the parameters list (ArrayList) and standard parameter handling, - such as {add|remove}_parameter(), traverse hierarchy and param_array, gradient_array + such as {link|unlink}_parameter(), traverse hierarchy and param_array, gradient_array and the empty parameters_changed(). This class is abstract and should not be instantiated. diff --git a/GPy/core/verbose_optimization.py b/GPy/core/verbose_optimization.py new file mode 100644 index 00000000..352621d7 --- /dev/null +++ b/GPy/core/verbose_optimization.py @@ -0,0 +1,96 @@ +# Copyright (c) 2012-2014, Max Zwiessele. +# Licensed under the BSD 3-clause license (see LICENSE.txt) + + +import numpy as np +import sys + +def exponents(fnow, current_grad): + exps = [np.abs(np.float(fnow)), current_grad] + return np.sign(exps) * np.log10(exps).astype(int) + +class VerboseOptimization(object): + def __init__(self, model, maxiters, verbose=True, current_iteration=0, ipython_notebook=False): + self.verbose = verbose + if self.verbose: + self.model = model + self.iteration = current_iteration + self.ipython_notebook = ipython_notebook + self.p_iter = self.iteration + self.maxiters = maxiters + self.len_maxiters = len(str(maxiters)) + self.model.add_observer(self, self.print_status) + + self.update() + + if self.ipython_notebook: + from IPython.display import display + from IPython.html.widgets import IntProgressWidget, HTMLWidget + self.text = HTMLWidget() + display(self.text) + self.progress = IntProgressWidget() + display(self.progress) + else: + self.exps = exponents(self.fnow, self.current_gradient) + print ' {0:{mi}s} {1:11s} {2:11s}'.format("I", "F", "|g|", mi=self.len_maxiters) + + def __enter__(self): + return self + + def print_out(self): + if self.ipython_notebook: + names_vals = [['Iteration', "{:>0{l}}".format(self.iteration, l=self.len_maxiters)], + ['f', "{: > 05.3E}".format(self.fnow)], + ['||Gradient||', "{: >+05.3E}".format(float(self.current_gradient))], + ] + #message = "Lik:{:5.3E} Grad:{:5.3E} Lik:{:5.3E} Len:{!s}".format(float(m.log_likelihood()), np.einsum('i,i->', grads, grads), float(m.likelihood.variance), " ".join(["{:3.2E}".format(l) for l in m.kern.lengthscale.values])) + html_begin = """ + """ + html_end = "
" + html_body = "" + for name, val in names_vals: + html_body += "" + html_body += "{}".format(name) + html_body += "{}".format(val) + html_body += "" + self.text.value = html_begin + html_body + html_end + self.progress.value = 100*(self.iteration+1)/self.maxiters + else: + n_exps = exponents(self.fnow, self.current_gradient) + if self.iteration - self.p_iter >= 20 * np.random.rand(): + a = self.iteration >= self.p_iter * 2.78 + b = np.any(n_exps < self.exps) + if a or b: + self.p_iter = self.iteration + print '' + if b: + self.exps = n_exps + print '\r', + print '{0:>0{mi}g} {1:> 12e} {2:> 12e}'.format(self.iteration, float(self.fnow), float(self.current_gradient), mi=self.len_maxiters), # print 'Iteration:', iteration, ' Objective:', fnow, ' Scale:', beta, '\r', + sys.stdout.flush() + + def print_status(self, me, which=None): + self.update() + + #sys.stdout.write(" "*len(self.message)) + self.print_out() + + self.iteration += 1 + + def update(self): + self.fnow = self.model.objective_function() + if self.model.obj_grads is not None: + grad = self.model.obj_grads + self.current_gradient = np.dot(grad, grad) + else: + self.current_gradient = np.nan + + def __exit__(self, type, value, traceback): + if self.verbose: + self.model.remove_observer(self) \ No newline at end of file diff --git a/GPy/inference/optimization/optimization.py b/GPy/inference/optimization/optimization.py index 8f673198..d5089c4e 100644 --- a/GPy/inference/optimization/optimization.py +++ b/GPy/inference/optimization/optimization.py @@ -31,7 +31,8 @@ class Optimizer(): ftol=None, gtol=None, xtol=None, bfgs_factor=None): self.opt_name = None self.x_init = x_init - self.messages = messages + # Turning messages off and using internal structure for print outs: + self.messages = False #messages self.f_opt = None self.x_opt = None self.funct_eval = None