1 ## Copyright (C) 2002 Etienne Grossmann <etienne@egdn.net>
3 ## This program is free software; you can redistribute it and/or modify it under
4 ## the terms of the GNU General Public License as published by the Free Software
5 ## Foundation; either version 3 of the License, or (at your option) any later
8 ## This program is distributed in the hope that it will be useful, but WITHOUT
9 ## ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 ## FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
13 ## You should have received a copy of the GNU General Public License along with
14 ## this program; if not, see <http://www.gnu.org/licenses/>.
16 ## [x,v,nev,h,args] = d2_min(f,d2f,args,ctl,code) - Newton-like minimization
18 ## Minimize f(x) using 1st and 2nd derivatives. Any function w/ second
19 ## derivatives can be minimized, as in Newton. f(x) decreases at each
20 ## iteration, as in Levenberg-Marquardt. This function is inspired from the
21 ## Levenberg-Marquardt algorithm found in the book "Numerical Recipes".
24 ## f : string : Cost function's name
26 ## d2f : string : Name of function returning the cost (1x1), its
27 ## differential (1xN) and its second differential or it's
28 ## pseudo-inverse (NxN) (see ctl(5) below) :
30 ## [v,dv,d2v] = d2f (x).
32 ## args : list : f and d2f's arguments. By default, minimize the 1st
33 ## or matrix : argument.
35 ## ctl : vector : Control arguments (see below)
38 ## code : string : code will be evaluated after each outer loop that
39 ## produced some (any) improvement. Variables visible from
40 ## "code" include "x", the best parameter found, "v" the
41 ## best value and "args", the list of all arguments. All can
42 ## be modified. This option can be used to re-parameterize
43 ## the argument space during optimization
45 ## CONTROL VARIABLE ctl : (optional). May be a struct or a vector of length
46 ## ---------------------- 5 or less where NaNs are ignored. Default values
47 ## are written <value>.
51 ## ftol, f N/A : Stop search when value doesn't improve, as tested by
53 ## f > Deltaf/max(|f(x)|,1)
55 ## where Deltaf is the decrease in f observed in the last
56 ## iteration. <10*sqrt(eps)>
58 ## utol, u N/A : Stop search when updates are small, as tested by
60 ## u > max { dx(i)/max(|x(i)|,1) | i in 1..N }
62 ## where dx is the change in the x that occured in the last
65 ## dtol, d N/A : Stop search when derivative is small, as tested by
67 ## d > norm (dv) <eps>
69 ## crit, c ctl(1) : Set one stopping criterion, 'ftol' (c=1), 'utol' (c=2)
70 ## or 'dtol' (c=3) to the value of by the 'tol' option. <1>
72 ## tol, t ctl(2) : Threshold in termination test chosen by 'crit' <10*eps>
74 ## narg, n ctl(3) : Position of the minimized argument in args <1>
75 ## maxev,m ctl(4) : Maximum number of function evaluations <inf>
76 ## maxout,m : Maximum number of outer loops <inf>
77 ## id2f, i ctl(5) : 0 if d2f returns the 2nd derivatives, 1 if <0>
78 ## it returns its pseudo-inverse.
80 ## verbose, v N/A : Be more or less verbose (quiet=0) <0>
82 function [xbest,vbest,nev,hbest,args] = d2_min (f,d2f,args,ctl,code)
87 tcoeff = 0.5 ; # Discount on total weight
88 ncoeff = 0.5 ; # Discount on weight of newton
89 ocoeff = 1.5 ; # Factor for outwards searching
91 report = 0 ; # Never report
92 verbose = 0 ; # Be quiet
93 prudent = 1 ; # Check coherence of d2f and f?
97 crit = 0; # Default control variables
98 ftol = 10 * sqrt (eps);
105 if nargin >= 4 # Read arguments
107 if length (ctl)>=1 && !isnan (ctl(1)), crit = ctl(1); end
108 if length (ctl)>=2 && !isnan (ctl(2)), tol = ctl(2); end
109 if length (ctl)>=3 && !isnan (ctl(3)), narg = ctl(3); end
110 if length (ctl)>=4 && !isnan (ctl(4)), maxev = ctl(4); end
111 if length (ctl)>=5 && !isnan (ctl(5)), id2f = ctl(5); end
112 elseif isstruct (ctl)
113 if isfield (ctl, "crit") , crit = ctl.crit ; end
114 if isfield (ctl, "tol") , tol = ctl.tol ; end
115 if isfield (ctl, "narg") , narg = ctl.narg ; end
116 if isfield (ctl, "maxev") , maxev = ctl.maxev ; end
117 if isfield (ctl, "maxout") , maxout = ctl.maxout ; end
118 if isfield (ctl, "id2f") , id2f = ctl.id2f ; end
119 if isfield (ctl, "verbose"), verbose = ctl.verbose; end
120 if isfield (ctl, "code") , code = ctl.code ; end
122 error ("The 'ctl' argument should be either a vector or a struct");
126 if crit == 1, ftol = tol;
127 elseif crit == 2, utol = tol;
128 elseif crit == 3, dtol = tol;
129 elseif crit, error ("crit is %i. Should be 1,2 or 3.\n");
133 if nargin < 5, code = "" ; end
135 if iscell (args) # List of arguments
137 else # Single argument
142 ############################## Checking ##############################
143 if narg > length (args)
144 error ("d2_min : narg==%i, length (args)==%i\n",
145 narg, length (args));
149 printf ("d2_min : tol=%8.3g <= 0\n",tol) ;
152 if !ischar (d2f) || !ischar (f)
153 printf ("d2_min : f and d2f should be strings!\n");
156 sz = size (x); N = prod (sz);
158 v = feval (f, args{:});
161 if prudent && (! isnumeric (v) || isnan (v) || any (size (v)>1))
162 error ("Function '%s' returns inadequate output", f);
166 vold = vbest = nan ; # Values of f
167 hbest = nan ; # Inv. Hessian
170 printf ( "d2_min : Initially, v=%8.3g\n",v);
173 while niter <= maxout
175 if nev(1) < maxev, break; end;
177 [v,d,h] = feval (d2f, args{1:narg-1},reshape(x,sz),args{narg+1:end});
180 if prudent && niter <= 1 && \
181 (! isnumeric (v) || isnan (v) || any (size (v)>1) || \
182 ! isnumeric (d) || length (d(:)) != N || \
183 ! isnumeric (h) || any (size (h) != N))
184 error ("Function '%s' returns inadequate output", d2f);
187 if ! id2f, h = pinv (h); end
191 v2 = feval (f, args{1:narg-1},reshape(x,sz),args{narg+1:end});
193 if abs(v2-v) > 0.001 * sqrt(eps) * max (abs(v2), 1)
194 printf ("d2_min : f and d2f disagree %8.3g\n",abs(v2-v));
199 if ! isnan (vbest) # Check that v ==vbest
200 if abs (vbest - v) > 1000*eps * max (vbest, 1)
201 printf ("d2_min : vbest changed at beginning of outer loop\n");
207 if length (code), abest = args; end # Eventually stash all args
209 if verbose || (report && rem(niter,max(report,1)) == 1)
210 printf ("d2_min : niter=%d, v=%8.3g\n",niter,v );
213 if norm (d) < dtol # Check for small derivative
215 printf ("d2_min : Exiting because of low gradient\n");
217 break; # Exit outer loop
220 dnewton = -h*d ; # Newton step
221 # Heuristic for negative hessian
222 if dnewton'*d > 0, dnewton = -100*d; end
223 wn = 1 ; # Weight of Newton step
224 wt = 1 ; # Total weight
227 done_inner = 0; # 0=not found. 1=Ready to quit inner.
229 # ##########################################
230 while ninner < maxinner, # Inner loop ###############################
233 dx = wt*(wn*dnewton - (1-wn)*d) ;
237 printf (["Weight : total=%8.3g, newtons's=%8.3g vbest=%8.3g ",...
238 "Norm:Newton=%8.3g, deriv=%8.3g\n"],...
239 wt,wn,vbest,norm(wt*wn*dnewton),norm(wt*(1-wn)*d));
242 printf ("d2_min : Whoa!! any(isnan(xnew)) (1)\n");
245 vnew = feval (f, args{1:narg-1},reshape(xnew,sz),args{narg+1:end});
248 if vnew<vbest # Stash best values
253 done_inner = 1 ; # Will go out at next increase
255 printf ( "d2_min : Found better value\n");
258 elseif done_inner == 1 # Time to go out
260 printf ( "d2_min : Quitting %d th inner loop\n",niter);
262 break; # out of inner loop
264 wt = wt*tcoeff ; # Reduce norm of proposed step
265 wn = wn*ncoeff ; # And bring it closer to derivative
267 end # End of inner loop ########################
268 # ##########################################
270 wbest = 0; # Best coeff for dbest
272 if ninner >= maxinner # There was a problem
274 printf ( "d2_min : Too many inner loops (vnew=%8.3g)\n",vnew);
277 # ##########################################
278 else # Look for improvement along dbest
282 printf ("d2_min : Whoa!! any(isnan(xnew)) (2)\n");
284 vnew = feval (f, args{1:narg-1},reshape(xnew,sz),args{narg+1:end});
288 vbest = vnew; # Stash best values
293 vnew = feval (f, args{1:narg-1},reshape(xnew,sz),args{narg+1:length(args)});
295 printf ( "Looking farther : v = %8.3g\n",vnew);
299 end # End of improving along dbest
300 # ##########################################
302 if verbose || rem(niter,max(report,1)) == 1
305 printf ("d2_min : Inner loop : vbest=%8.5g, vbest/vold=%8.5g\n",\
310 printf ( "d2_min : Inner loop : vbest=%8.5g, vold=0\n", vbest);
316 ## "improvement found"
318 tmpv = feval (f, args{1:narg-1},reshape(xbest,sz),args{2:end});
321 if abs (tmpv-vbest) > eps
322 printf ("d2_min : Whoa! Value at xbest changed by %g\n",\
326 v = vbest; x = xbest;
329 printf ("d2_min : Going to eval (\"%s\")\n",code);
334 args = abest; # Here : added 2001/11/07. Is that right?
336 eval (code, "printf (\"code fails\\n\");");
339 # Check whether eval (code) changes value
341 tmpv = feval (f, args{1:narg-1},reshape(x,sz),args{2:end});
343 if abs (tmpv-vbest) > max (min (100*eps,0.00001*abs(vbest)), eps) ,
344 printf ("d2_min : Whoa! Value changes by %g after eval (code)\n",\
351 if ! isnan (ftol) && ftol > (vold-vbest)/max(vold,1),
353 printf ("d2_min : Quitting, niter=%-3d v=%8.3g, ",niter,v);
354 if vold, printf ("v/vold=%8.3g \n",v/vold);
355 else printf ("vold =0 \n",v);
358 break ; # out of outer loop
360 if ! isnan (utol) && utol > max (abs (wbest*dbest)) / max(abs (xbest),1)
362 printf ("d2_min : Quitting, niter=%-3d v=%8.3g, ",niter,v);
363 if vold, printf ("v/vold=%8.3g \n",v/vold);
364 else printf ("vold =0 \n",v);
367 break ; # out of outer loop
369 end # End of outer loop ##################
371 xbest = reshape (xbest, sz);
379 printf ( "d2_min : Outer loop lasts forever\n");
385 err = feval (f, args{1:narg-1},reshape(xbest,sz),args{2:end});
388 if abs (err-vbest) > eps,
389 printf ("d2_min : Whoa!! xbest does not eval to vbest\n");
390 printf (" : %8.3e - %8.3e = %8.3e != 0\n",err,vbest,err-vbest);