diff --git a/dlib/test/CMakeLists.txt b/dlib/test/CMakeLists.txt
index 30cda27f1..9d91891ef 100644
--- a/dlib/test/CMakeLists.txt
+++ b/dlib/test/CMakeLists.txt
@@ -45,6 +45,7 @@ set (tests
    is_same_object.cpp
    kcentroid.cpp
    kernel_matrix.cpp
+   least_squares.cpp
    linear_manifold_regularizer.cpp
    lz77_buffer.cpp
    map.cpp
diff --git a/dlib/test/least_squares.cpp b/dlib/test/least_squares.cpp
new file mode 100644
index 000000000..3f22b6f81
--- /dev/null
+++ b/dlib/test/least_squares.cpp
@@ -0,0 +1,433 @@
+// Copyright (C) 2010  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+
+
+#include <dlib/optimization.h>
+#include "optimization_test_functions.h"
+#include <sstream>
+#include <string>
+#include <cstdlib>
+#include <ctime>
+#include <vector>
+#include "../rand.h"
+
+#include "tester.h"
+
+
+namespace  
+{
+
+    using namespace test;
+    using namespace dlib;
+    using namespace std;
+    using namespace dlib::test_functions;
+
+    logger dlog("test.least_squares");
+
+// ----------------------------------------------------------------------------------------
+
+    void test_with_chebyquad()
+    {
+        print_spinner();
+        {
+            matrix<double,0,1> ch;
+
+            ch = chebyquad_start(2);
+
+            solve_least_squares(objective_delta_stop_strategy(1e-13, 80),
+                                &chebyquad_residual,
+                                derivative(&chebyquad_residual),
+                                range(0,ch.size()-1),
+                                ch);
+
+            dlog << LINFO << "chebyquad 2 obj: " << chebyquad(ch);
+            dlog << LINFO << "chebyquad 2 der: " << length(chebyquad_derivative(ch));
+            dlog << LINFO << "chebyquad 2 error: " << length(ch - chebyquad_solution(2));
+
+            DLIB_TEST(length(ch - chebyquad_solution(2)) < 1e-5);
+
+        }
+        {
+            matrix<double,0,1> ch;
+
+            ch = chebyquad_start(2);
+
+            solve_least_squares_lm(objective_delta_stop_strategy(1e-13, 80),
+                                &chebyquad_residual,
+                                derivative(&chebyquad_residual),
+                                range(0,ch.size()-1),
+                                ch);
+
+            dlog << LINFO << "LM chebyquad 2 obj: " << chebyquad(ch);
+            dlog << LINFO << "LM chebyquad 2 der: " << length(chebyquad_derivative(ch));
+            dlog << LINFO << "LM chebyquad 2 error: " << length(ch - chebyquad_solution(2));
+
+            DLIB_TEST(length(ch - chebyquad_solution(2)) < 1e-5);
+
+        }
+
+        print_spinner();
+        {
+            matrix<double,2,1> ch;
+
+            ch = chebyquad_start(2);
+
+            solve_least_squares(objective_delta_stop_strategy(1e-13, 80),
+                                &chebyquad_residual,
+                                derivative(&chebyquad_residual),
+                                range(0,ch.size()-1),
+                                ch);
+
+            dlog << LINFO << "chebyquad 2 obj: " << chebyquad(ch);
+            dlog << LINFO << "chebyquad 2 der: " << length(chebyquad_derivative(ch));
+            dlog << LINFO << "chebyquad 2 error: " << length(ch - chebyquad_solution(2));
+
+            DLIB_TEST(length(ch - chebyquad_solution(2)) < 1e-5);
+
+        }
+        print_spinner();
+        {
+            matrix<double,2,1> ch;
+
+            ch = chebyquad_start(2);
+
+            solve_least_squares_lm(objective_delta_stop_strategy(1e-13, 80),
+                                &chebyquad_residual,
+                                derivative(&chebyquad_residual),
+                                range(0,ch.size()-1),
+                                ch);
+
+            dlog << LINFO << "LM chebyquad 2 obj: " << chebyquad(ch);
+            dlog << LINFO << "LM chebyquad 2 der: " << length(chebyquad_derivative(ch));
+            dlog << LINFO << "LM chebyquad 2 error: " << length(ch - chebyquad_solution(2));
+
+            DLIB_TEST(length(ch - chebyquad_solution(2)) < 1e-5);
+
+        }
+
+        print_spinner();
+        {
+            matrix<double,0,1> ch;
+
+            ch = chebyquad_start(4);
+
+            solve_least_squares(objective_delta_stop_strategy(1e-13, 80),
+                                &chebyquad_residual,
+                                derivative(&chebyquad_residual),
+                                range(0,ch.size()-1),
+                                ch);
+
+            dlog << LINFO << "chebyquad 4 obj: " << chebyquad(ch);
+            dlog << LINFO << "chebyquad 4 der: " << length(chebyquad_derivative(ch));
+            dlog << LINFO << "chebyquad 4 error: " << length(ch - chebyquad_solution(4));
+
+            DLIB_TEST(length(ch - chebyquad_solution(4)) < 1e-5);
+
+        }
+        print_spinner();
+        {
+            matrix<double,0,1> ch;
+
+            ch = chebyquad_start(4);
+
+            solve_least_squares_lm(objective_delta_stop_strategy(1e-13, 80),
+                                &chebyquad_residual,
+                                derivative(&chebyquad_residual),
+                                range(0,ch.size()-1),
+                                ch);
+
+            dlog << LINFO << "LM chebyquad 4 obj: " << chebyquad(ch);
+            dlog << LINFO << "LM chebyquad 4 der: " << length(chebyquad_derivative(ch));
+            dlog << LINFO << "LM chebyquad 4 error: " << length(ch - chebyquad_solution(4));
+
+            DLIB_TEST(length(ch - chebyquad_solution(4)) < 1e-5);
+
+        }
+
+
+        print_spinner();
+        {
+            matrix<double,0,1> ch;
+
+            ch = chebyquad_start(6);
+
+            solve_least_squares(objective_delta_stop_strategy(1e-13, 80),
+                                &chebyquad_residual,
+                                derivative(&chebyquad_residual),
+                                range(0,ch.size()-1),
+                                ch);
+
+            dlog << LINFO << "chebyquad 6 obj: " << chebyquad(ch);
+            dlog << LINFO << "chebyquad 6 der: " << length(chebyquad_derivative(ch));
+            dlog << LINFO << "chebyquad 6 error: " << length(ch - chebyquad_solution(6));
+
+            // the ch variable contains a permutation of what is in chebyquad_solution(6).
+            // Apparently there is more than one minimum?.  Just check that the objective 
+            // goes to zero.
+            DLIB_TEST(chebyquad(ch) < 1e-10);
+
+        }
+        print_spinner();
+        {
+            matrix<double,0,1> ch;
+
+            ch = chebyquad_start(6);
+
+            solve_least_squares_lm(objective_delta_stop_strategy(1e-13, 80),
+                                &chebyquad_residual,
+                                derivative(&chebyquad_residual),
+                                range(0,ch.size()-1),
+                                ch);
+
+            dlog << LINFO << "LM chebyquad 6 obj: " << chebyquad(ch);
+            dlog << LINFO << "LM chebyquad 6 der: " << length(chebyquad_derivative(ch));
+            dlog << LINFO << "LM chebyquad 6 error: " << length(ch - chebyquad_solution(6));
+
+            DLIB_TEST(chebyquad(ch) < 1e-10);
+
+        }
+
+
+        print_spinner();
+        {
+            matrix<double,0,1> ch;
+
+            ch = chebyquad_start(8);
+
+            solve_least_squares(objective_delta_stop_strategy(1e-13, 80),
+                                &chebyquad_residual,
+                                derivative(&chebyquad_residual),
+                                range(0,ch.size()-1),
+                                ch);
+
+            dlog << LINFO << "chebyquad 8 obj: " << chebyquad(ch);
+            dlog << LINFO << "chebyquad 8 der: " << length(chebyquad_derivative(ch));
+            dlog << LINFO << "chebyquad 8 error: " << length(ch - chebyquad_solution(8));
+
+            DLIB_TEST(length(ch - chebyquad_solution(8)) < 1e-5);
+
+        }
+        print_spinner();
+        {
+            matrix<double,0,1> ch;
+
+            ch = chebyquad_start(8);
+
+            solve_least_squares_lm(objective_delta_stop_strategy(1e-13, 80),
+                                &chebyquad_residual,
+                                derivative(&chebyquad_residual),
+                                range(0,ch.size()-1),
+                                ch);
+
+            dlog << LINFO << "LM chebyquad 8 obj: " << chebyquad(ch);
+            dlog << LINFO << "LM chebyquad 8 der: " << length(chebyquad_derivative(ch));
+            dlog << LINFO << "LM chebyquad 8 error: " << length(ch - chebyquad_solution(8));
+
+            DLIB_TEST(length(ch - chebyquad_solution(8)) < 1e-5);
+
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    void test_with_brown()
+    {
+        print_spinner();
+        {
+            matrix<double,4,1> ch;
+
+            ch = brown_start();
+
+            solve_least_squares(objective_delta_stop_strategy(1e-13, 80),
+                                &brown_residual,
+                                derivative(&brown_residual),
+                                range(1,20),
+                                ch);
+
+            dlog << LINFO << "brown obj: " << brown(ch);
+            dlog << LINFO << "brown der: " << length(brown_derivative(ch));
+            dlog << LINFO << "brown error: " << length(ch - brown_solution());
+
+            DLIB_TEST(length(ch - brown_solution()) < 1e-5);
+
+        }
+        print_spinner();
+        {
+            matrix<double,4,1> ch;
+
+            ch = brown_start();
+
+            solve_least_squares_lm(objective_delta_stop_strategy(1e-13, 80),
+                                &brown_residual,
+                                derivative(&brown_residual),
+                                range(1,20),
+                                ch);
+
+            dlog << LINFO << "LM brown obj: " << brown(ch);
+            dlog << LINFO << "LM brown der: " << length(brown_derivative(ch));
+            dlog << LINFO << "LM brown error: " << length(ch - brown_solution());
+
+            DLIB_TEST(length(ch - brown_solution()) < 1e-5);
+
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    void test_with_rosen()
+    {
+
+        print_spinner();
+        {
+            matrix<double,2,1> ch;
+
+            ch = rosen_start<double>();
+
+            solve_least_squares(objective_delta_stop_strategy(1e-13, 80),
+                                &rosen_residual<double>,
+                                &rosen_residual_derivative<double>,
+                                range(1,20),
+                                ch);
+
+            dlog << LINFO << "rosen obj: " << rosen(ch);
+            dlog << LINFO << "rosen error: " << length(ch - rosen_solution<double>());
+
+            DLIB_TEST(length(ch - rosen_solution<double>()) < 1e-5);
+
+        }
+        print_spinner();
+        {
+            matrix<double,2,1> ch;
+
+            ch = rosen_start<double>();
+
+            solve_least_squares_lm(objective_delta_stop_strategy(1e-13, 80),
+                                &rosen_residual<double>,
+                                &rosen_residual_derivative<double>,
+                                range(1,20),
+                                ch);
+
+            dlog << LINFO << "lm rosen obj: " << rosen(ch);
+            dlog << LINFO << "lm rosen error: " << length(ch - rosen_solution<double>());
+
+            DLIB_TEST(length(ch - rosen_solution<double>()) < 1e-5);
+
+        }
+
+
+
+        print_spinner();
+        {
+            matrix<double,2,1> ch;
+
+            ch = rosen_start<double>();
+
+            solve_least_squares(objective_delta_stop_strategy(1e-13, 80),
+                                &rosen_residual<double>,
+                                derivative(&rosen_residual<double>),
+                                range(1,20),
+                                ch);
+
+            dlog << LINFO << "rosen obj: " << rosen(ch);
+            dlog << LINFO << "rosen error: " << length(ch - rosen_solution<double>());
+
+            DLIB_TEST(length(ch - rosen_solution<double>()) < 1e-5);
+
+        }
+        print_spinner();
+        {
+            matrix<float,2,1> ch;
+
+            ch = rosen_start<float>();
+
+            solve_least_squares(objective_delta_stop_strategy(1e-13, 80),
+                                &rosen_residual<float>,
+                                derivative(&rosen_residual<float>),
+                                range(1,20),
+                                ch);
+
+            dlog << LINFO << "float rosen obj: " << rosen(ch);
+            dlog << LINFO << "float rosen error: " << length(ch - rosen_solution<float>());
+
+            DLIB_TEST(length(ch - rosen_solution<float>()) < 1e-5);
+
+        }
+        print_spinner();
+        {
+            matrix<float,2,1> ch;
+
+            ch = rosen_start<float>();
+
+            solve_least_squares_lm(objective_delta_stop_strategy(1e-13, 80),
+                                &rosen_residual<float>,
+                                derivative(&rosen_residual<float>),
+                                range(1,20),
+                                ch);
+
+            dlog << LINFO << "LM float rosen obj: " << rosen(ch);
+            dlog << LINFO << "LM float rosen error: " << length(ch - rosen_solution<float>());
+
+            DLIB_TEST(length(ch - rosen_solution<float>()) < 1e-5);
+
+        }
+        print_spinner();
+        {
+            matrix<double,2,1> ch;
+
+            ch = rosen_start<double>();
+
+            solve_least_squares_lm(objective_delta_stop_strategy(1e-13, 80),
+                                &rosen_residual<double>,
+                                derivative(&rosen_residual<double>),
+                                range(1,20),
+                                ch);
+
+            dlog << LINFO << "LM rosen obj: " << rosen(ch);
+            dlog << LINFO << "LM rosen error: " << length(ch - rosen_solution<double>());
+
+            DLIB_TEST(length(ch - rosen_solution<double>()) < 1e-5);
+
+        }
+        print_spinner();
+        {
+            matrix<double,2,1> ch;
+
+            ch = rosen_big_start<double>();
+
+            solve_least_squares(objective_delta_stop_strategy(1e-13, 80),
+                                &rosen_big_residual<double>,
+                                derivative(&rosen_big_residual<double>),
+                                range(1,2),
+                                ch);
+
+            dlog << LINFO << "rosen big obj: " << rosen_big(ch);
+            dlog << LINFO << "rosen big error: " << length(ch - rosen_big_solution<double>());
+
+            DLIB_TEST(length(ch - rosen_big_solution<double>()) < 1e-5);
+
+        }
+    }
+
+// ----------------------------------------------------------------------------------------
+
+    class optimization_tester : public tester
+    {
+    public:
+        optimization_tester (
+        ) :
+            tester ("test_least_squares",
+                    "Runs tests on the least squares optimization component.")
+        {}
+
+        void perform_test (
+        )
+        {
+            test_with_chebyquad();
+            test_with_brown();
+            test_with_rosen();
+        }
+    } a;
+
+}
+
+
diff --git a/dlib/test/makefile b/dlib/test/makefile
index 63f40254c..29f17b947 100644
--- a/dlib/test/makefile
+++ b/dlib/test/makefile
@@ -55,6 +55,7 @@ SRC += image.cpp
 SRC += is_same_object.cpp
 SRC += kcentroid.cpp
 SRC += kernel_matrix.cpp
+SRC += least_squares.cpp
 SRC += linear_manifold_regularizer.cpp
 SRC += lz77_buffer.cpp
 SRC += map.cpp
diff --git a/dlib/test/optimization_test_functions.h b/dlib/test/optimization_test_functions.h
index 02fb98fd0..a71fe8bee 100644
--- a/dlib/test/optimization_test_functions.h
+++ b/dlib/test/optimization_test_functions.h
@@ -155,7 +155,7 @@ namespace dlib
         T rosen_big ( const matrix<T,2,1>& m)
         {
             using std::pow;
-            return 0.5*(pow(rosen_big_residual(1,m),2) + pow(rosen_big_residual(2,m)));
+            return 0.5*(pow(rosen_big_residual(1,m),2) + pow(rosen_big_residual(2,m),2));
         }
 
         template <typename T>
@@ -208,6 +208,25 @@ namespace dlib
             }
         }
 
+        template <typename T>
+        matrix<T,2,1> rosen_residual_derivative (int i, const matrix<T,2,1>& m)
+        {
+            const T x = m(0); 
+            const T y = m(1);
+
+            matrix<T,2,1> d;
+
+            if (i == 1)
+            {
+                d = -20*x, 10;
+            }
+            else
+            {
+                d = -1, 0;
+            }
+            return d;
+        }
+
         template <typename T>
         const matrix<T,2,1> rosen_derivative ( const matrix<T,2,1>& m)
         {