diff --git a/dlib/optimization/max_sum_submatrix.h b/dlib/optimization/max_sum_submatrix.h
index a91b73472..b305a93d6 100644
--- a/dlib/optimization/max_sum_submatrix.h
+++ b/dlib/optimization/max_sum_submatrix.h
@@ -212,8 +212,10 @@ namespace dlib
                 rectangle rect(left, q.top().top_min, 
                                right, q.top().bottom_min);
 
-                if (weight > thresh)
-                    results.push_back(rect);
+                if (weight <= thresh)
+                    break;
+
+                results.push_back(rect);
 
                 if (results.size() >= max_rects)
                     break;