Make gradient on edges smaller

coreylowman · Sep 28, 2023 · 30bd707 · 30bd707
1 parent b5dccc0
commit 30bd707
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/src/tensor_ops/sigmoidr/cpu_kernel.rs b/src/tensor_ops/sigmoidr/cpu_kernel.rs
@@ -10,6 +10,6 @@ impl<F: num_traits::Float> UnaryDerivative<F> for super::SigmoidrKernelOp {
     #[inline(always)]
     fn df(&self, &fx: &F) -> F {
         let d = fx * (F::one() - fx);
-        F::max(d, F::from(0.0001).unwrap())
+        F::max(d, F::from(0.0000001).unwrap())
     }
 }
diff --git a/src/tensor_ops/sigmoidr/mod.rs b/src/tensor_ops/sigmoidr/mod.rs
@@ -53,7 +53,7 @@ mod tests {
         let g = r.mean().backward();
         assert_close_to_literal!(
             g.get(&x),
-            [0.020998716, 0.039322387, 0.05, 0.039322387, 0.00002]
+            [0.020998716, 0.039322387, 0.05, 0.039322387, 0.00000002]
         );
     }
 }
diff --git a/src/tensor_ops/sigmoidr/sigmoidr.cu b/src/tensor_ops/sigmoidr/sigmoidr.cu
@@ -12,7 +12,7 @@ template<typename T>
 __device__ __forceinline__ T sigmoidr_bwd(T y) {
     T one = 1.0;
   T d = y * (one - y);
-    return max(d, 0.0001);
+    return max(d, 0.0000001);
 }
 
 UNARY_OP(__half, sigmoidr_fwd_f16, sigmoidr_bwd_f16, SigmoidrKernelOp,