diff --git a/src/classifier.c b/src/classifier.c index 27aafad2..5f5ab1bf 100644 --- a/src/classifier.c +++ b/src/classifier.c @@ -132,6 +132,10 @@ void train_classifier(char *datacfg, char *cfgfile, char *weightfile, int *gpus, int iter_topk = get_current_batch(net); float topk = 0; + int count = 0; + double start, end, tmp, time_remaining[1000] = {0}; + start = what_time_is_it_now(); + while(get_current_batch(net) < net.max_batches || net.max_batches == 0){ time=clock(); @@ -182,7 +186,18 @@ void train_classifier(char *datacfg, char *cfgfile, char *weightfile, int *gpus, printf("%d, %.3f: %f, %f avg, %f rate, %lf seconds, %ld images\n", get_current_batch(net), (float)(*net.seen)/ train_images_num, loss, avg_loss, get_current_rate(net), sec(clock()-time), *net.seen); #ifdef OPENCV - if (!dontuse_opencv) draw_train_loss(windows_name, img, img_size, avg_loss, max_img_loss, i, net.max_batches, topk, draw_precision, topk_buff, dont_show, mjpeg_port); + end = what_time_is_it_now(); + time_remaining[i%1000] = (net.max_batches - i)*(end - start) / 60 / 60; + tmp = 0.0; + int j, count = 0; + for (j = 0; j < 1000; ++j){ + if (time_remaining[j] != 0){ + tmp += time_remaining[j]; + count++; + } + } + if (!dontuse_opencv) draw_train_loss(windows_name, img, img_size, avg_loss, max_img_loss, i, net.max_batches, topk, draw_precision, topk_buff, dont_show, mjpeg_port, tmp/count); + start = what_time_is_it_now(); #endif // OPENCV if (i >= (iter_save + 1000)) { diff --git a/src/detector.c b/src/detector.c index 2a147724..8052c36e 100644 --- a/src/detector.c +++ b/src/detector.c @@ -163,7 +163,11 @@ void train_detector(char *datacfg, char *cfgfile, char *weightfile, int *gpus, i //printf(" imgs = %d \n", imgs); pthread_t load_thread = load_data(args); + int count = 0; + double start, end, tmp, time_remaining[1000] = {0}; + start = what_time_is_it_now(); + //while(i*imgs < N*120){ while (get_current_batch(net) < net.max_batches) { if (l.random && count++ % 10 == 0) { @@ -300,7 +304,18 @@ void train_detector(char *datacfg, char *cfgfile, char *weightfile, int *gpus, i draw_precision = 1; } #ifdef OPENCV - draw_train_loss(windows_name, img, img_size, avg_loss, max_img_loss, i, net.max_batches, mean_average_precision, draw_precision, "mAP%", dont_show, mjpeg_port); + end = what_time_is_it_now(); + time_remaining[i%1000] = (net.max_batches - i)*(end - start) / 60 / 60; + tmp = 0.0; + int j, count = 0; + for (j = 0; j < 1000; ++j){ + if (time_remaining[j] != 0){ + tmp += time_remaining[j]; + count++; + } + } + draw_train_loss(windows_name, img, img_size, avg_loss, max_img_loss, i, net.max_batches, mean_average_precision, draw_precision, "mAP%", dont_show, mjpeg_port, tmp/count); + start = what_time_is_it_now(); #endif // OPENCV //if (i % 1000 == 0 || (i < 1000 && i % 100 == 0)) { diff --git a/src/image_opencv.cpp b/src/image_opencv.cpp index 73808477..3854084f 100644 --- a/src/image_opencv.cpp +++ b/src/image_opencv.cpp @@ -1063,7 +1063,7 @@ extern "C" mat_cv* draw_train_chart(char *windows_name, float max_img_loss, int // ---------------------------------------- extern "C" void draw_train_loss(char *windows_name, mat_cv* img_src, int img_size, float avg_loss, float max_img_loss, int current_batch, int max_batches, - float precision, int draw_precision, char *accuracy_name, int dont_show, int mjpeg_port) + float precision, int draw_precision, char *accuracy_name, int dont_show, int mjpeg_port, double time_remaining) { try { cv::Mat &img = *(cv::Mat*)img_src; @@ -1104,10 +1104,9 @@ extern "C" void draw_train_loss(char *windows_name, mat_cv* img_src, int img_siz old_precision = precision; iteration_old = current_batch; } - - sprintf(char_buff, "current avg loss = %2.4f iteration = %d", avg_loss, current_batch); + sprintf(char_buff, "current avg loss = %2.4f iteration = %d approx. time remaining = %2.2f hours", avg_loss, current_batch, time_remaining); pt1.x = 15, pt1.y = draw_size + 18; - pt2.x = pt1.x + 460, pt2.y = pt1.y + 20; + pt2.x = pt1.x + 800, pt2.y = pt1.y + 20; cv::rectangle(img, pt1, pt2, CV_RGB(255, 255, 255), CV_FILLED, 8, 0); pt1.y += 15; cv::putText(img, char_buff, pt1, cv::FONT_HERSHEY_COMPLEX_SMALL, 0.7, CV_RGB(0, 0, 100), 1, CV_AA); diff --git a/src/image_opencv.h b/src/image_opencv.h index 136b7388..a87593db 100644 --- a/src/image_opencv.h +++ b/src/image_opencv.h @@ -94,7 +94,7 @@ void draw_detections_cv_v3(mat_cv* show_img, detection *dets, int num, float thr // Draw Loss & Accuracy chart mat_cv* draw_train_chart(char *windows_name, float max_img_loss, int max_batches, int number_of_lines, int img_size, int dont_show); void draw_train_loss(char *windows_name, mat_cv* img, int img_size, float avg_loss, float max_img_loss, int current_batch, int max_batches, - float precision, int draw_precision, char *accuracy_name, int dont_show, int mjpeg_port); + float precision, int draw_precision, char *accuracy_name, int dont_show, int mjpeg_port, double time_remaining); // Data augmentation image image_data_augmentation(mat_cv* mat, int w, int h,