From e7072b8489da7347561a47be849f401c8a0a2abd Mon Sep 17 00:00:00 2001
From: Joseph Redmon <pjreddie@gmail.com>
Date: Mon, 20 Jun 2016 13:18:59 -0700
Subject: [PATCH] checkpoint

---
 Makefile                  |  6 +++---
 src/activation_kernels.cu | 16 ++++++++++++++++
 src/activations.c         |  7 +++++++
 src/activations.h         | 14 +++++++++++++-
 src/network.c             |  1 +
 src/network.h             |  1 +
 src/parser.c              |  1 +
 7 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 28a0d17a..f3e4b792 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
-GPU=0
-CUDNN=0
-OPENCV=0
+GPU=1
+CUDNN=1
+OPENCV=1
 DEBUG=0
 
 ARCH= --gpu-architecture=compute_52 --gpu-code=compute_52 
diff --git a/src/activation_kernels.cu b/src/activation_kernels.cu
index 362d5d72..5d61529b 100644
--- a/src/activation_kernels.cu
+++ b/src/activation_kernels.cu
@@ -8,6 +8,18 @@ extern "C" {
 }
 
 
+__device__ float lhtan_activate_kernel(float x)
+{
+    if(x < 0) return .001*x;
+    if(x > 1) return .001*(x-1) + 1;
+    return x;
+}
+__device__ float lhtan_gradient_kernel(float x)
+{
+    if(x > 0 && x < 1) return 1;
+    return .001;
+}
+
 __device__ float hardtan_activate_kernel(float x)
 {
     if (x < -1) return -1;
@@ -89,6 +101,8 @@ __device__ float activate_kernel(float x, ACTIVATION a)
             return stair_activate_kernel(x);
         case HARDTAN:
             return hardtan_activate_kernel(x);
+        case LHTAN:
+            return lhtan_activate_kernel(x);
     }
     return 0;
 }
@@ -120,6 +134,8 @@ __device__ float gradient_kernel(float x, ACTIVATION a)
             return stair_gradient_kernel(x);
         case HARDTAN:
             return hardtan_gradient_kernel(x);
+        case LHTAN:
+            return lhtan_gradient_kernel(x);
     }
     return 0;
 }
diff --git a/src/activations.c b/src/activations.c
index 6ab4963f..0cbb2f55 100644
--- a/src/activations.c
+++ b/src/activations.c
@@ -32,6 +32,8 @@ char *get_activation_string(ACTIVATION a)
             return "stair";
         case HARDTAN:
             return "hardtan";
+        case LHTAN:
+            return "lhtan";
         default:
             break;
     }
@@ -47,6 +49,7 @@ ACTIVATION get_activation(char *s)
     if (strcmp(s, "relie")==0) return RELIE;
     if (strcmp(s, "plse")==0) return PLSE;
     if (strcmp(s, "hardtan")==0) return HARDTAN;
+    if (strcmp(s, "lhtan")==0) return LHTAN;
     if (strcmp(s, "linear")==0) return LINEAR;
     if (strcmp(s, "ramp")==0) return RAMP;
     if (strcmp(s, "leaky")==0) return LEAKY;
@@ -83,6 +86,8 @@ float activate(float x, ACTIVATION a)
             return stair_activate(x);
         case HARDTAN:
             return hardtan_activate(x);
+        case LHTAN:
+            return lhtan_activate(x);
     }
     return 0;
 }
@@ -122,6 +127,8 @@ float gradient(float x, ACTIVATION a)
             return stair_gradient(x);
         case HARDTAN:
             return hardtan_gradient(x);
+        case LHTAN:
+            return lhtan_gradient(x);
     }
     return 0;
 }
diff --git a/src/activations.h b/src/activations.h
index fed2908a..d1b8c37e 100644
--- a/src/activations.h
+++ b/src/activations.h
@@ -4,7 +4,7 @@
 #include "math.h"
 
 typedef enum{
-    LOGISTIC, RELU, RELIE, LINEAR, RAMP, TANH, PLSE, LEAKY, ELU, LOGGY, STAIR, HARDTAN
+    LOGISTIC, RELU, RELIE, LINEAR, RAMP, TANH, PLSE, LEAKY, ELU, LOGGY, STAIR, HARDTAN, LHTAN
 }ACTIVATION;
 
 ACTIVATION get_activation(char *s);
@@ -47,6 +47,18 @@ static inline float plse_activate(float x)
     return .125*x + .5;
 }
 
+static inline float lhtan_activate(float x)
+{
+    if(x < 0) return .001*x;
+    if(x > 1) return .001*(x-1) + 1;
+    return x;
+}
+static inline float lhtan_gradient(float x)
+{
+    if(x > 0 && x < 1) return 1;
+    return .001;
+}
+
 static inline float hardtan_gradient(float x)
 {
     if (x > -1 && x < 1) return 1;
diff --git a/src/network.c b/src/network.c
index 51f74d96..a9e50277 100644
--- a/src/network.c
+++ b/src/network.c
@@ -64,6 +64,7 @@ float get_current_rate(network net)
         case EXP:
             return net.learning_rate * pow(net.gamma, batch_num);
         case POLY:
+            if (batch_num < net.burn_in) return net.learning_rate * pow((float)batch_num / net.burn_in, net.power);
             return net.learning_rate * pow(1 - (float)batch_num / net.max_batches, net.power);
         case RANDOM:
             return net.learning_rate * pow(rand_uniform(0,1), net.power);
diff --git a/src/network.h b/src/network.h
index 15b58b88..af64e06f 100644
--- a/src/network.h
+++ b/src/network.h
@@ -34,6 +34,7 @@ typedef struct network{
     float *scales;
     int   *steps;
     int num_steps;
+    int burn_in;
 
     int inputs;
     int h, w, c;
diff --git a/src/parser.c b/src/parser.c
index 71f54cc0..00de059a 100644
--- a/src/parser.c
+++ b/src/parser.c
@@ -467,6 +467,7 @@ void parse_net_options(list *options, network *net)
 
     char *policy_s = option_find_str(options, "policy", "constant");
     net->policy = get_policy(policy_s);
+    net->burn_in = option_find_int_quiet(options, "burn_in", 0);
     if(net->policy == STEP){
         net->step = option_find_int(options, "step", 1);
         net->scale = option_find_float(options, "scale", 1);