From 10a586140c99a905a504e6813b7cc629ff50ea42 Mon Sep 17 00:00:00 2001
From: AlexeyAB <alexeyab84@gmail.com>
Date: Thu, 27 Feb 2020 18:03:58 +0300
Subject: [PATCH] Minor fix. Fixed forward MISH on CPU

---
 src/activations.h      |  3 ++-
 src/network.c          |  7 +++++++
 src/network_kernels.cu | 10 ++++++++++
 3 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/activations.h b/src/activations.h
index fdbc28ef..16285c04 100644
--- a/src/activations.h
+++ b/src/activations.h
@@ -61,7 +61,8 @@ static inline float selu_activate(float x) { return (x >= 0)*1.0507f*x + (x < 0)
 static inline float relie_activate(float x){return (x>0) ? x : .01f*x;}
 static inline float ramp_activate(float x){return x*(x>0)+.1f*x;}
 static inline float leaky_activate(float x){return (x>0) ? x : .1f*x;}
-static inline float tanh_activate(float x){return (expf(2*x)-1)/(expf(2*x)+1);}
+//static inline float tanh_activate(float x){return (expf(2*x)-1)/(expf(2*x)+1);}
+static inline float tanh_activate(float x) { return (2 / (1 + expf(-2 * x)) - 1); }
 static inline float softplus_activate(float x, float threshold) {
     if (x > threshold) return x;                // too large
     else if (x < -threshold) return expf(x);    // too small
diff --git a/src/network.c b/src/network.c
index 239b3d82..771df85c 100644
--- a/src/network.c
+++ b/src/network.c
@@ -266,6 +266,13 @@ void forward_network(network net, network_state state)
         l.forward(l, state);
         //printf("%d - Predicted in %lf milli-seconds.\n", i, ((double)get_time_point() - time) / 1000);
         state.input = l.output;
+
+        /*
+        float avg_val = 0;
+        int k;
+        for (k = 0; k < l.outputs; ++k) avg_val += l.output[k];
+        printf(" i: %d - avg_val = %f \n", i, avg_val / l.outputs);
+        */
     }
 }
 
diff --git a/src/network_kernels.cu b/src/network_kernels.cu
index 14ade8ed..34d9ec6a 100644
--- a/src/network_kernels.cu
+++ b/src/network_kernels.cu
@@ -106,6 +106,16 @@ void forward_network_gpu(network net, network_state state)
             cudaStreamSynchronize(get_cuda_stream());
         state.input = l.output_gpu;
         //cudaDeviceSynchronize();
+
+        /*
+        cuda_pull_array(l.output_gpu, l.output, l.outputs);
+        cudaStreamSynchronize(get_cuda_stream());
+        float avg_val = 0;
+        int k;
+        for (k = 0; k < l.outputs; ++k) avg_val += l.output[k];
+        printf(" i: %d - avg_val = %f \n", i, avg_val / l.outputs);
+        */
+
 /*
         cuda_pull_array(l.output_gpu, l.output, l.batch*l.outputs);
         if (l.out_w >= 0 && l.out_h >= 1 && l.c >= 3) {