diff --git a/docs/docs/algorithms.xml b/docs/docs/algorithms.xml index 8cbc0df98..0408620e4 100644 --- a/docs/docs/algorithms.xml +++ b/docs/docs/algorithms.xml @@ -13,7 +13,7 @@ This page documents library components that are all basically just implementations of mathematical functions or algorithms that don't fit in any of the other pages of the dlib documentation. So this includes things like checksums, cryptographic hashes, - optimization, sorting, etc. + sorting, etc.

@@ -30,32 +30,6 @@ running_stats running_covariance random_subset_selector - - Optimization - - derivative - negate_function - make_line_search_function - poly_min_extrap - lagrange_poly_min_extrap - line_search - find_min - find_min_single_variable - find_min_using_approximate_derivatives - find_min_bobyqa - solve_qp_using_smo - find_max - find_max_single_variable - find_max_using_approximate_derivatives - find_max_bobyqa - - cg_search_strategy - bfgs_search_strategy - lbfgs_search_strategy - objective_delta_stop_strategy - gradient_norm_stop_strategy - - Quantum Computing @@ -129,374 +103,7 @@ - - - derivative - dlib/optimization.h - dlib/optimization/optimization_abstract.h - - This is a function that takes another function as input and returns - a function object that numerically computes the derivative of the input function. - - - - - - - - negate_function - dlib/optimization.h - dlib/optimization/optimization_abstract.h - - This is a function that takes another function as input and returns - a function object that computes the negation of the input function. - - - - - - - - - make_line_search_function - dlib/optimization.h - dlib/optimization/optimization_line_search_abstract.h - - This is a function that takes another function f(x) as input and returns - a function object l(z) = f(start + z*direction). It is useful for - turning multi-variable functions into single-variable functions for - use with the line_search routine. - - - - - - - - - poly_min_extrap - dlib/optimization.h - dlib/optimization/optimization_line_search_abstract.h - - This function finds the 3rd degree polynomial that interpolates a - set of points and returns you the minimum of that polynomial. - - - - - - - - lagrange_poly_min_extrap - dlib/optimization.h - dlib/optimization/optimization_line_search_abstract.h - - This function finds the second order polynomial that interpolates a - set of points and returns you the minimum of that polynomial. - - - - - - - - line_search - dlib/optimization.h - dlib/optimization/optimization_line_search_abstract.h - - Performs a gradient based line search on a given function and returns the input - that makes the function significantly smaller. - - - - - - - - - cg_search_strategy - dlib/optimization.h - dlib/optimization/optimization_search_strategies_abstract.h - - This object represents a strategy for determining which direction - a line search should be carried out along. This particular object - is an implementation of the Polak-Ribiere conjugate gradient method - for determining this direction. - -

- This method uses an amount of memory that is linear in the number - of variables to be optimized. So it is capable of handling problems - with a very large number of variables. However, it is generally - not as good as the L-BFGS algorithm (see the - lbfgs_search_strategy class). -

- - - optimization_ex.cpp.html - - - - - - - - bfgs_search_strategy - dlib/optimization.h - dlib/optimization/optimization_search_strategies_abstract.h - - This object represents a strategy for determining which direction - a line search should be carried out along. This particular object - is an implementation of the BFGS quasi-newton method for determining - this direction. - -

- This method uses an amount of memory that is quadratic in the number - of variables to be optimized. It is generally very effective but - if your problem has a very large number of variables then it isn't - appropriate. Instead You should try the lbfgs_search_strategy. -

- - - optimization_ex.cpp.html - - - - - - - - lbfgs_search_strategy - dlib/optimization.h - dlib/optimization/optimization_search_strategies_abstract.h - - This object represents a strategy for determining which direction - a line search should be carried out along. This particular object - is an implementation of the L-BFGS quasi-newton method for determining - this direction. - -

- This method uses an amount of memory that is linear in the number - of variables to be optimized. This makes it an excellent method - to use when an optimization problem has a large number of variables. -

- - - optimization_ex.cpp.html - - - - - - - - objective_delta_stop_strategy - dlib/optimization.h - dlib/optimization/optimization_stop_strategies_abstract.h - - This object represents a strategy for deciding if an optimization - algorithm should terminate. This particular object looks at the - change in the objective function from one iteration to the next and - bases its decision on how large this change is. If the change - is below a user given threshold then the search stops. - - - optimization_ex.cpp.html - - - - - - - - gradient_norm_stop_strategy - dlib/optimization.h - dlib/optimization/optimization_stop_strategies_abstract.h - - This object represents a strategy for deciding if an optimization - algorithm should terminate. This particular object looks at the - norm (i.e. the length) of the current gradient vector and - stops if it is smaller than a user given threshold. - - - - - - - - find_min - dlib/optimization.h - dlib/optimization/optimization_abstract.h - - Performs an unconstrained minimization of a nonlinear function using - some search strategy (e.g. bfgs_search_strategy). - - - optimization_ex.cpp.html - - - - - - - - find_min_single_variable - dlib/optimization.h - dlib/optimization/optimization_line_search_abstract.h - - Performs a bound constrained minimization of a nonlinear function. The - function must be of a single variable. Derivatives are not required. - - - - - - - - solve_qp_using_smo - dlib/optimization.h - dlib/optimization/optimization_solve_qp_using_smo_abstract.h - - This function solves the following quadratic program: -
-   Minimize: f(alpha) == 0.5*trans(alpha)*Q*alpha - trans(alpha)*b
-   subject to the following constraints:
-      sum(alpha) == C 
-      min(alpha) >= 0 
-
- - - - - - - - - - find_min_bobyqa - dlib/optimization.h - dlib/optimization/optimization_bobyqa_abstract.h - - This function defines the dlib interface to the BOBYQA software developed by M.J.D Powell. - BOBYQA is a method for optimizing a function in the absence of derivative information. - Powell described it as a method that seeks the least value of a function of many - variables, by applying a trust region method that forms quadratic models by - interpolation. There is usually some freedom in the interpolation conditions, - which is taken up by minimizing the Frobenius norm of the change to the second - derivative of the model, beginning with the zero matrix. The values of the variables - are constrained by upper and lower bounds. - -

- The following paper, published in 2009 by Powell, describes the - detailed working of the BOBYQA algorithm. - -

- The BOBYQA algorithm for bound constrained optimization - without derivatives by M.J.D. Powell -
-

- -

- Note that BOBYQA only works on functions of two or more variables. So if you need to perform - derivative-free optimization on a function of a single variable - then you should use the find_min_single_variable - function. -

- - - - optimization_ex.cpp.html - model_selection_ex.cpp.html - - - - - - - - find_max_bobyqa - dlib/optimization.h - dlib/optimization/optimization_bobyqa_abstract.h - - This function is identical to the find_min_bobyqa routine - except that it negates the objective function before performing optimization. - Thus this function will attempt to find the maximizer of the objective rather than - the minimizer. -

- Note that BOBYQA only works on functions of two or more variables. So if you need to perform - derivative-free optimization on a function of a single variable - then you should use the find_max_single_variable - function. -

- - - optimization_ex.cpp.html - model_selection_ex.cpp.html - - - - - - - - find_min_using_approximate_derivatives - dlib/optimization.h - dlib/optimization/optimization_abstract.h - - Performs an unconstrained minimization of a nonlinear function using - some search strategy (e.g. bfgs_search_strategy). - This version doesn't take a gradient function but instead numerically approximates - the gradient. - - - optimization_ex.cpp.html - - - - - - - - find_max - dlib/optimization.h - dlib/optimization/optimization_abstract.h - - Performs an unconstrained maximization of a nonlinear function using - some search strategy (e.g. bfgs_search_strategy). - - - - - - - - find_max_single_variable - dlib/optimization.h - dlib/optimization/optimization_line_search_abstract.h - - Performs a bound constrained maximization of a nonlinear function. The - function must be of a single variable. Derivatives are not required. - - - - - - - - find_max_using_approximate_derivatives - dlib/optimization.h - dlib/optimization/optimization_abstract.h - - Performs an unconstrained maximization of a nonlinear function using - some search strategy (e.g. bfgs_search_strategy). - This version doesn't take a gradient function but instead numerically approximates - the gradient. - - - - - - bigint dlib/bigint.h diff --git a/docs/docs/books.xml b/docs/docs/books.xml index 781ddc41b..6d2f96633 100644 --- a/docs/docs/books.xml +++ b/docs/docs/books.xml @@ -222,7 +222,7 @@
  • Introduction to Derivative-Free Optimization by Conn, Scheinberg, and Vicente -
      If you want to understand algorithms like BOBYQA +
        If you want to understand algorithms like BOBYQA then this is a good recent book on the subject. Note that a book like Practical Methods of Optimization is almost certainly a prerequisite for reading this book. As an aside, BOBYQA is not discussed in this book but its predecessor, NEWUOA is. diff --git a/docs/docs/index.xml b/docs/docs/index.xml index cf5185b98..104cda865 100644 --- a/docs/docs/index.xml +++ b/docs/docs/index.xml @@ -109,11 +109,11 @@ transpose, trig functions, etc.
      • Unconstrained non-linear optimization algorithms using the - conjugate gradient, - BFGS, and - L-BFGS techniques
        • + conjugate gradient, + BFGS, and + L-BFGS techniques
      • Box-constrained derivative-free optimization via the - BOBYQA algorithm
        • + BOBYQA algorithm
      • A big integer object
      • A random number object
      diff --git a/docs/docs/main_menu.xml b/docs/docs/main_menu.xml index b53399383..aefddad0d 100644 --- a/docs/docs/main_menu.xml +++ b/docs/docs/main_menu.xml @@ -10,6 +10,11 @@ algorithms.html algorithms.xml + + Optimization + optimization.html + optimization.xml + Machine Learning ml.html @@ -31,7 +36,7 @@ api.xml - Network + Networking network.html network.xml diff --git a/docs/docs/optimization.xml b/docs/docs/optimization.xml new file mode 100644 index 000000000..7c2666a26 --- /dev/null +++ b/docs/docs/optimization.xml @@ -0,0 +1,439 @@ + + + + + Optimization + + + + +

      + +

      + This page documents library components that attempt to find the + minimum or maximum of a user supplied function. +

      + + + + + + + +
      + Main Routines + find_min + find_min_single_variable + find_min_using_approximate_derivatives + find_min_bobyqa + solve_qp_using_smo + find_max + find_max_single_variable + find_max_using_approximate_derivatives + find_max_bobyqa +
      + +
      + Strategies + cg_search_strategy + bfgs_search_strategy + lbfgs_search_strategy + objective_delta_stop_strategy + gradient_norm_stop_strategy +
      + +
      + Helper Routines + derivative + negate_function + make_line_search_function + poly_min_extrap + lagrange_poly_min_extrap + line_search +
      + +       +       + + + + + + + + + + + derivative + dlib/optimization.h + dlib/optimization/optimization_abstract.h + + This is a function that takes another function as input and returns + a function object that numerically computes the derivative of the input function. + + + + + + + + + negate_function + dlib/optimization.h + dlib/optimization/optimization_abstract.h + + This is a function that takes another function as input and returns + a function object that computes the negation of the input function. + + + + + + + + + make_line_search_function + dlib/optimization.h + dlib/optimization/optimization_line_search_abstract.h + + This is a function that takes another function f(x) as input and returns + a function object l(z) = f(start + z*direction). It is useful for + turning multi-variable functions into single-variable functions for + use with the line_search routine. + + + + + + + + + poly_min_extrap + dlib/optimization.h + dlib/optimization/optimization_line_search_abstract.h + + This function finds the 3rd degree polynomial that interpolates a + set of points and returns you the minimum of that polynomial. + + + + + + + + lagrange_poly_min_extrap + dlib/optimization.h + dlib/optimization/optimization_line_search_abstract.h + + This function finds the second order polynomial that interpolates a + set of points and returns you the minimum of that polynomial. + + + + + + + + line_search + dlib/optimization.h + dlib/optimization/optimization_line_search_abstract.h + + Performs a gradient based line search on a given function and returns the input + that makes the function significantly smaller. + + + + + + + + + cg_search_strategy + dlib/optimization.h + dlib/optimization/optimization_search_strategies_abstract.h + + This object represents a strategy for determining which direction + a line search should be carried out along. This particular object + is an implementation of the Polak-Ribiere conjugate gradient method + for determining this direction. + +

      + This method uses an amount of memory that is linear in the number + of variables to be optimized. So it is capable of handling problems + with a very large number of variables. However, it is generally + not as good as the L-BFGS algorithm (see the + lbfgs_search_strategy class). +

      +       + + optimization_ex.cpp.html + + +       + + + + + bfgs_search_strategy + dlib/optimization.h + dlib/optimization/optimization_search_strategies_abstract.h + + This object represents a strategy for determining which direction + a line search should be carried out along. This particular object + is an implementation of the BFGS quasi-newton method for determining + this direction. + +

      + This method uses an amount of memory that is quadratic in the number + of variables to be optimized. It is generally very effective but + if your problem has a very large number of variables then it isn't + appropriate. Instead You should try the lbfgs_search_strategy. +

      +       + + optimization_ex.cpp.html + + +       + + + + + lbfgs_search_strategy + dlib/optimization.h + dlib/optimization/optimization_search_strategies_abstract.h + + This object represents a strategy for determining which direction + a line search should be carried out along. This particular object + is an implementation of the L-BFGS quasi-newton method for determining + this direction. + +

      + This method uses an amount of memory that is linear in the number + of variables to be optimized. This makes it an excellent method + to use when an optimization problem has a large number of variables. +

      +       + + optimization_ex.cpp.html + + +       + + + + + objective_delta_stop_strategy + dlib/optimization.h + dlib/optimization/optimization_stop_strategies_abstract.h + + This object represents a strategy for deciding if an optimization + algorithm should terminate. This particular object looks at the + change in the objective function from one iteration to the next and + bases its decision on how large this change is. If the change + is below a user given threshold then the search stops. + + + optimization_ex.cpp.html + + + + + + + + gradient_norm_stop_strategy + dlib/optimization.h + dlib/optimization/optimization_stop_strategies_abstract.h + + This object represents a strategy for deciding if an optimization + algorithm should terminate. This particular object looks at the + norm (i.e. the length) of the current gradient vector and + stops if it is smaller than a user given threshold. + + + + + + + + find_min + dlib/optimization.h + dlib/optimization/optimization_abstract.h + + Performs an unconstrained minimization of a nonlinear function using + some search strategy (e.g. bfgs_search_strategy). + + + optimization_ex.cpp.html + + + + + + + + find_min_single_variable + dlib/optimization.h + dlib/optimization/optimization_line_search_abstract.h + + Performs a bound constrained minimization of a nonlinear function. The + function must be of a single variable. Derivatives are not required. + + + + + + + + solve_qp_using_smo + dlib/optimization.h + dlib/optimization/optimization_solve_qp_using_smo_abstract.h + + This function solves the following quadratic program: +
      +   Minimize: f(alpha) == 0.5*trans(alpha)*Q*alpha - trans(alpha)*b
+   subject to the following constraints:
+      sum(alpha) == C 
+      min(alpha) >= 0 
+
      + +       + +       + + + + + + find_min_bobyqa + dlib/optimization.h + dlib/optimization/optimization_bobyqa_abstract.h + + This function defines the dlib interface to the BOBYQA software developed by M.J.D Powell. + BOBYQA is a method for optimizing a function in the absence of derivative information. + Powell described it as a method that seeks the least value of a function of many + variables, by applying a trust region method that forms quadratic models by + interpolation. There is usually some freedom in the interpolation conditions, + which is taken up by minimizing the Frobenius norm of the change to the second + derivative of the model, beginning with the zero matrix. The values of the variables + are constrained by upper and lower bounds. + +

      + The following paper, published in 2009 by Powell, describes the + detailed working of the BOBYQA algorithm. + +

      + The BOBYQA algorithm for bound constrained optimization + without derivatives by M.J.D. Powell +
      +

      + +

      + Note that BOBYQA only works on functions of two or more variables. So if you need to perform + derivative-free optimization on a function of a single variable + then you should use the find_min_single_variable + function. +

      + +       + + optimization_ex.cpp.html + model_selection_ex.cpp.html + + +       + + + + + find_max_bobyqa + dlib/optimization.h + dlib/optimization/optimization_bobyqa_abstract.h + + This function is identical to the find_min_bobyqa routine + except that it negates the objective function before performing optimization. + Thus this function will attempt to find the maximizer of the objective rather than + the minimizer. +

      + Note that BOBYQA only works on functions of two or more variables. So if you need to perform + derivative-free optimization on a function of a single variable + then you should use the find_max_single_variable + function. +

      +       + + optimization_ex.cpp.html + model_selection_ex.cpp.html + + +       + + + + + find_min_using_approximate_derivatives + dlib/optimization.h + dlib/optimization/optimization_abstract.h + + Performs an unconstrained minimization of a nonlinear function using + some search strategy (e.g. bfgs_search_strategy). + This version doesn't take a gradient function but instead numerically approximates + the gradient. + + + optimization_ex.cpp.html + + + + + + + + find_max + dlib/optimization.h + dlib/optimization/optimization_abstract.h + + Performs an unconstrained maximization of a nonlinear function using + some search strategy (e.g. bfgs_search_strategy). + + + + + + + + find_max_single_variable + dlib/optimization.h + dlib/optimization/optimization_line_search_abstract.h + + Performs a bound constrained maximization of a nonlinear function. The + function must be of a single variable. Derivatives are not required. + + + + + + + + find_max_using_approximate_derivatives + dlib/optimization.h + dlib/optimization/optimization_abstract.h + + Performs an unconstrained maximization of a nonlinear function using + some search strategy (e.g. bfgs_search_strategy). + This version doesn't take a gradient function but instead numerically approximates + the gradient. + + + + + + +       + + + + +       + diff --git a/docs/docs/term_index.xml b/docs/docs/term_index.xml index 2aac8e951..df5d4a885 100644 --- a/docs/docs/term_index.xml +++ b/docs/docs/term_index.xml @@ -32,27 +32,27 @@ - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + +