First commit

1 files changed, 115 insertions, 0 deletions

diff --git a/src/CuTNetLib/cuSparseLinearity.h b/src/CuTNetLib/cuSparseLinearity.h
new file mode 100644
index 0000000..3cdf078
--- /dev/null
+++ b/src/CuTNetLib/cuSparseLinearity.h
@@ -0,0 +1,115 @@
+#ifndef _CUSPARSE_LINEARITY_H_
+#define _CUSPARSE_LINEARITY_H_
+
+
+#include "cuComponent.h"
+#include "cumatrix.h"
+
+
+#include "Matrix.h"
+#include "Vector.h"
+
+
+namespace TNet {
+
+  /**
+   * \brief CuSparseLinearity summation function
+   *
+   * \ingroup CuNNUpdatable
+   * Using weight masks to avoid fluctuation in the output
+   *  -Weights are masked when it is lower than certain threshold - mSparsifyWeightThreshold
+   *  -Weights are activated when the accumulated change is larger than certan value - mUnsparsifyAccu
+   *  -L1 lasso function zeroing weights
+   *  .
+   * \sa CuBiasedLinearity
+   */
+  class CuSparseLinearity : public CuUpdatableComponent
+  {
+    public:
+
+      CuSparseLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred); 
+      ~CuSparseLinearity();  
+      
+      ComponentType GetType() const;
+      const char* GetName() const;
+
+      void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y);
+      void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y);
+
+      void Update();
+      void UpdateMask();
+
+      void ReadFromStream(std::istream& rIn);
+      void WriteToStream(std::ostream& rOut);
+
+      void L1(BaseFloat l1) {
+        mL1Const = l1;
+      }
+
+    protected:
+      CuMatrix<BaseFloat> mLinearity;  ///< Matrix with neuron weights
+      CuVector<BaseFloat> mBias;       ///< Vector with biases
+      CuMatrix<BaseFloat> mSparsityMask; ///< Mask which selects active weights
+
+      CuMatrix<BaseFloat> mLinearityCorrection; ///< Matrix for linearity updates
+      CuVector<BaseFloat> mBiasCorrection;      ///< Vector for bias updates
+
+      CuMatrix<BaseFloat> mLinearityCorrectionAccu; ///< Accumulator for linearity updates
+
+      BaseFloat mL1Const; ///< L1 regularization constant
+
+      size_t mNFrames; ///< Number of accumulated frames 
+      BaseFloat mSparsifyWeightThreshold; ///< Cutoff
+      BaseFloat mUnsparsifyAccu; ///< Threshold to unsparsify the Cutoff
+
+      
+  };
+
+
+
+
+  ////////////////////////////////////////////////////////////////////////////
+  // INLINE FUNCTIONS 
+  // CuSparseLinearity::
+  inline 
+  CuSparseLinearity::
+  CuSparseLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred)
+    : CuUpdatableComponent(nInputs, nOutputs, pPred), 
+      mLinearity(nInputs,nOutputs), mBias(nOutputs), mSparsityMask(nInputs,nOutputs),
+      mLinearityCorrection(nInputs,nOutputs), mBiasCorrection(nOutputs),
+      mLinearityCorrectionAccu(nInputs,nOutputs),
+      mNFrames(0), mSparsifyWeightThreshold(1.0e-3),
+      mUnsparsifyAccu(1e20f)
+  { 
+    mLinearityCorrection.SetConst(0.0f);
+    mBiasCorrection.SetConst(0.0f);
+    mLinearityCorrectionAccu.SetConst(0.0f);
+  }
+
+
+  inline
+  CuSparseLinearity::
+  ~CuSparseLinearity()
+  { }
+
+  inline CuComponent::ComponentType
+  CuSparseLinearity::
+  GetType() const
+  {
+    return CuComponent::SPARSE_LINEARITY;
+  }
+
+  inline const char*
+  CuSparseLinearity::
+  GetName() const
+  {
+    return "<sparselinearity>";
+  }
+
+
+
+} //namespace
+
+
+
+#endif