1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
|
#ifndef _CUACT_FUN_I_
#define _CUACT_FUN_I_
#include "cuComponent.h"
#include "cumatrix.h"
namespace TNet
{
/**
* \brief Common interface for activation functions
*/
class CuActivation : public CuComponent
{
public:
CuActivation(size_t nInputs, size_t nOutputs, CuComponent *pPred);
protected:
};
/**
* \brief CuSigmoid activation function
*
* \ingroup CuNNActivation
* Implements forward pass: \f[ Y_i=\frac{1}{1+e^{-X_i}} \f]
* Error propagation: \f[ E_i=Y_i(1-Y_i)e_i \f]
*/
class CuSigmoid : public CuActivation
{
public:
CuSigmoid(size_t nInputs, size_t nOutputs, CuComponent *pPred);
ComponentType GetType() const;
const char* GetName() const;
protected:
void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y);
void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y);
};
/**
* \brief CuSoftmax activation function
*
* \ingroup CuNNActivation
* Implements forward pass: \f[ Y_i=\frac{1}{Z} e^{X_i} \f]
* where \f$ Z=\Sigma_{i=0}^{i=N-1} e^{X_i} \f$
* Error Propagation: \f[ E_i=Y_i - \Sigma_{j=0}^{j=N-1} Y_i Y_j e_j \f]
*/
class CuSoftmax : public CuActivation
{
public:
CuSoftmax(size_t nInputs, size_t nOutputs, CuComponent *pPred);
ComponentType GetType() const;
const char* GetName() const;
protected:
void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y);
void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y);
};
//////////////////////////////////////////////////////////////////////////
// Inline functions
// Activation::
inline
CuActivation::
CuActivation(size_t nInputs, size_t nOutputs, CuComponent *pPred)
: CuComponent(nInputs,nOutputs, pPred)
{
assert(nInputs == nOutputs);
}
//////////////////////////////////////////////////////////////////////////
// Inline functions
// Sigmoid::
inline
CuSigmoid::
CuSigmoid(size_t nInputs, size_t nOutputs, CuComponent *pPred)
: CuActivation(nInputs,nOutputs, pPred)
{ }
inline CuComponent::ComponentType
CuSigmoid::
GetType() const
{
return CuComponent::SIGMOID;
}
inline const char*
CuSigmoid::
GetName() const
{
return "<sigmoid>";
}
//////////////////////////////////////////////////////////////////////////
// Inline functions
// Softmax::
inline
CuSoftmax::
CuSoftmax(size_t nInputs, size_t nOutputs, CuComponent *pPred)
: CuActivation(nInputs,nOutputs, pPred)
{ }
inline CuComponent::ComponentType
CuSoftmax::
GetType() const
{
return CuComponent::SOFTMAX;
}
inline const char*
CuSoftmax::
GetName() const
{
return "<softmax>";
}
} //namespace
#endif
|
