fc_layer: scale initial weights and bias by 1/sqrt(inputsize)

seldon-code · Oct 28, 2023 · 0912f2e · 0912f2e
1 parent 201a8c1
commit 0912f2e
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/include/fc_layer.hpp b/include/fc_layer.hpp
@@ -27,8 +27,8 @@ class FCLayer : public Layer<scalar>
 
         const auto random_lambda = [&]( [[maybe_unused]] scalar x ) { return dist( gen ); };
 
-        weights = weights.array().unaryExpr( random_lambda );
-        bias    = bias.array().unaryExpr( random_lambda );
+        weights = weights.array().unaryExpr( random_lambda ) / std::sqrt( input_size );
+        bias    = bias.array().unaryExpr( random_lambda ) / std::sqrt( input_size );
     }
 
     std::string name() override