[optimization prints] unified printouts for optimizers, added ipython_notebook flag for use in ipython notebooks

GPy/core/model.py

@@ -10,6 +10,8 @@ import multiprocessing as mp
 import numpy as np
 from numpy.linalg.linalg import LinAlgError
 import itertools
+import sys
+from .verbose_optimization import VerboseOptimization
 # import numdifftools as ndt
 
 class Model(Parameterized):
@@ -24,6 +26,7 @@ class Model(Parameterized):
         from .parameterization.ties_and_remappings import Tie
         self.tie = Tie()
         self.link_parameter(self.tie, -1)
+        self.obj_grads = None
         self.add_observer(self.tie, self.tie._parameters_changed_notification, priority=-500)
 
     def log_likelihood(self):
@@ -165,14 +168,14 @@ class Model(Parameterized):
         try:
             # self._set_params_transformed(x)
             self.optimizer_array = x
-            obj_grads = self._transform_gradients(self.objective_function_gradients())
+            self.obj_grads = self._transform_gradients(self.objective_function_gradients())
             self._fail_count = 0
         except (LinAlgError, ZeroDivisionError, ValueError):
             if self._fail_count >= self._allowed_failures:
                 raise
             self._fail_count += 1
-            obj_grads = np.clip(self._transform_gradients(self.objective_function_gradients()), -1e100, 1e100)
+            self.obj_grads = np.clip(self._transform_gradients(self.objective_function_gradients()), -1e100, 1e100)
-        return obj_grads
+        return self.obj_grads
 
     def _objective(self, x):
         """
@@ -200,17 +203,17 @@ class Model(Parameterized):
     def _objective_and_grads(self, x):
         try:
             self.optimizer_array = x
-            obj_f, obj_grads = self.objective_function(), self._transform_gradients(self.objective_function_gradients())
+            obj_f, self.obj_grads = self.objective_function(), self._transform_gradients(self.objective_function_gradients())
             self._fail_count = 0
         except (LinAlgError, ZeroDivisionError, ValueError):
             if self._fail_count >= self._allowed_failures:
                 raise
             self._fail_count += 1
             obj_f = np.inf
-            obj_grads = np.clip(self._transform_gradients(self.objective_function_gradients()), -1e10, 1e10)
+            self.obj_grads = np.clip(self._transform_gradients(self.objective_function_gradients()), -1e10, 1e10)
-        return obj_f, obj_grads
+        return obj_f, self.obj_grads
 
-    def optimize(self, optimizer=None, start=None, **kwargs):
+    def optimize(self, optimizer=None, start=None, messages=False, max_iters=1000, ipython_notebook=False, **kwargs):
         """
         Optimize the model using self.log_likelihood and self.log_likelihood_gradient, as well as self.priors.
 
@@ -218,8 +221,8 @@ class Model(Parameterized):
 
         :param max_f_eval: maximum number of function evaluations
         :type max_f_eval: int
-        :messages: whether to display during optimisation
+        :messages: True: Display messages during optimisation, "ipython_notebook":
-        :type messages: bool
+        :type messages: bool"string
         :param optimizer: which optimizer to use (defaults to self.preferred optimizer)
         :type optimizer: string
 
@@ -251,9 +254,10 @@ class Model(Parameterized):
             opt.model = self
         else:
             optimizer = optimization.get_optimizer(optimizer)
-            opt = optimizer(start, model=self, **kwargs)
+            opt = optimizer(start, model=self, max_iters=max_iters, **kwargs)
 
-        opt.run(f_fp=self._objective_and_grads, f=self._objective, fp=self._objective_grads)
+        with VerboseOptimization(self, maxiters=max_iters, verbose=messages, ipython_notebook=ipython_notebook):
+            opt.run(f_fp=self._objective_and_grads, f=self._objective, fp=self._objective_grads)
 
         self.optimization_runs.append(opt)
 

GPy/core/parameterization/parameter_core.py

@@ -683,7 +683,7 @@ class OptimizationHandlable(Indexable):
         [np.put(g, i, c.gradfactor(self.param_array[i], g[i])) for c, i in self.constraints.iteritems() if c != __fixed__]
         if self._has_fixes(): return g[self._fixes_]
         return g
-    
+
     def _transform_gradients_non_natural(self, g):
         """
         Transform the gradients by multiplying the gradient factor for each
@@ -809,7 +809,7 @@ class Parameterizable(OptimizationHandlable):
     A parameterisable class.
 
     This class provides the parameters list (ArrayList) and standard parameter handling,
-    such as {add|remove}_parameter(), traverse hierarchy and param_array, gradient_array
+    such as {link|unlink}_parameter(), traverse hierarchy and param_array, gradient_array
     and the empty parameters_changed().
 
     This class is abstract and should not be instantiated.

GPy/core/verbose_optimization.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2012-2014, Max Zwiessele.
+# Licensed under the BSD 3-clause license (see LICENSE.txt)
+
+
+import numpy as np
+import sys
+
+def exponents(fnow, current_grad):
+    exps = [np.abs(np.float(fnow)), current_grad]
+    return np.sign(exps) * np.log10(exps).astype(int)
+
+class VerboseOptimization(object):
+    def __init__(self, model, maxiters, verbose=True, current_iteration=0, ipython_notebook=False):
+        self.verbose = verbose
+        if self.verbose:
+            self.model = model
+            self.iteration = current_iteration
+            self.ipython_notebook = ipython_notebook
+            self.p_iter = self.iteration
+            self.maxiters = maxiters
+            self.len_maxiters = len(str(maxiters))
+            self.model.add_observer(self, self.print_status)
+
+            self.update()
+
+            if self.ipython_notebook:
+                from IPython.display import display
+                from IPython.html.widgets import IntProgressWidget, HTMLWidget
+                self.text = HTMLWidget()
+                display(self.text)
+                self.progress = IntProgressWidget()
+                display(self.progress)
+            else:
+                self.exps = exponents(self.fnow, self.current_gradient)
+                print ' {0:{mi}s}   {1:11s}    {2:11s}'.format("I", "F", "|g|", mi=self.len_maxiters)
+
+    def __enter__(self):
+        return self
+
+    def print_out(self):
+        if self.ipython_notebook:
+            names_vals = [['Iteration', "{:>0{l}}".format(self.iteration, l=self.len_maxiters)],
+                          ['f', "{: > 05.3E}".format(self.fnow)],
+                          ['||Gradient||', "{: >+05.3E}".format(float(self.current_gradient))],
+                      ]
+            #message = "Lik:{:5.3E} Grad:{:5.3E} Lik:{:5.3E} Len:{!s}".format(float(m.log_likelihood()), np.einsum('i,i->', grads, grads), float(m.likelihood.variance), " ".join(["{:3.2E}".format(l) for l in m.kern.lengthscale.values]))
+            html_begin = """<style type="text/css">
+    .tg-opt  {font-family:"Courier New", Courier, monospace !important;padding:2px 3px;word-break:normal;border-collapse:collapse;border-spacing:0;border-color:#DCDCDC;margin:0px auto;}
+    .tg-opt td{font-family:"Courier New", Courier, monospace !important;font-weight:bold;color:#444;background-color:#F7FDFA;border-style:solid;border-width:1px;overflow:hidden;word-break:normal;border-color:#DCDCDC;}
+    .tg-opt th{font-family:"Courier New", Courier, monospace !important;font-weight:normal;color:#fff;background-color:#26ADE4;border-style:solid;border-width:1px;overflow:hidden;word-break:normal;border-color:#DCDCDC;}
+    .tg-opt .tg-left{font-family:"Courier New", Courier, monospace !important;font-weight:normal;text-align:left;}
+    .tg-opt .tg-right{font-family:"Courier New", Courier, monospace !important;font-weight:normal;text-align:right;}
+    </style>
+    <table class="tg-opt">"""
+            html_end = "</table>"
+            html_body = ""
+            for name, val in names_vals:
+                html_body += "<tr>"
+                html_body += "<td class='tg-left'>{}</td>".format(name)
+                html_body += "<td class='tg-right'>{}</td>".format(val)
+                html_body += "</tr>"
+            self.text.value = html_begin + html_body + html_end
+            self.progress.value = 100*(self.iteration+1)/self.maxiters
+        else:
+            n_exps = exponents(self.fnow, self.current_gradient)
+            if self.iteration - self.p_iter >= 20 * np.random.rand():
+                a = self.iteration >= self.p_iter * 2.78
+                b = np.any(n_exps < self.exps)
+                if a or b:
+                    self.p_iter = self.iteration
+                    print ''
+                if b:
+                    self.exps = n_exps
+            print '\r',
+            print '{0:>0{mi}g}  {1:> 12e}  {2:> 12e}'.format(self.iteration, float(self.fnow), float(self.current_gradient), mi=self.len_maxiters), # print 'Iteration:', iteration, ' Objective:', fnow, '  Scale:', beta, '\r',
+            sys.stdout.flush()
+
+    def print_status(self, me, which=None):
+        self.update()
+
+        #sys.stdout.write(" "*len(self.message))
+        self.print_out()
+
+        self.iteration += 1
+
+    def update(self):
+        self.fnow = self.model.objective_function()
+        if self.model.obj_grads is not None:
+            grad = self.model.obj_grads
+            self.current_gradient = np.dot(grad, grad)
+        else:
+            self.current_gradient = np.nan
+
+    def __exit__(self, type, value, traceback):
+        if self.verbose:
+            self.model.remove_observer(self)

GPy/inference/optimization/optimization.py

@@ -31,7 +31,8 @@ class Optimizer():
                  ftol=None, gtol=None, xtol=None, bfgs_factor=None):
         self.opt_name = None
         self.x_init = x_init
-        self.messages = messages
+        # Turning messages off and using internal structure for print outs:
+        self.messages = False #messages
         self.f_opt = None
         self.x_opt = None
         self.funct_eval = None