From cccccbf6cca94a3eaf813b4468453160e91c332b Mon Sep 17 00:00:00 2001 From: Joe Zhao Date: Mon, 14 Apr 2014 08:14:45 +0800 Subject: First commit --- src/CuTNetLib/.svn/entries | 946 +++++++++++++++++++++ src/CuTNetLib/.svn/prop-base/Makefile.svn-base | 5 + .../.svn/prop-base/cuActivation.cc.svn-base | 5 + .../.svn/prop-base/cuActivation.h.svn-base | 5 + .../.svn/prop-base/cuBiasedLinearity.cc.svn-base | 5 + .../.svn/prop-base/cuBiasedLinearity.h.svn-base | 5 + .../.svn/prop-base/cuCRBEDctFeat.h.svn-base | 5 + src/CuTNetLib/.svn/prop-base/cuCache.cc.svn-base | 5 + src/CuTNetLib/.svn/prop-base/cuCache.h.svn-base | 5 + .../.svn/prop-base/cuComponent.h.svn-base | 5 + src/CuTNetLib/.svn/prop-base/cuNetwork.cc.svn-base | 5 + src/CuTNetLib/.svn/prop-base/cuNetwork.h.svn-base | 5 + .../.svn/prop-base/cuObjectiveFunction.cc.svn-base | 5 + .../.svn/prop-base/cuObjectiveFunction.h.svn-base | 5 + src/CuTNetLib/.svn/prop-base/cuRbm.cc.svn-base | 5 + src/CuTNetLib/.svn/prop-base/cuRbm.h.svn-base | 5 + .../.svn/prop-base/cuRecurrent.cc.svn-base | 5 + .../.svn/prop-base/cuRecurrent.h.svn-base | 5 + .../.svn/prop-base/cuSharedLinearity.cc.svn-base | 5 + .../.svn/prop-base/cuSharedLinearity.h.svn-base | 5 + src/CuTNetLib/.svn/text-base/Makefile.svn-base | 30 + .../.svn/text-base/cuActivation.cc.svn-base | 46 + .../.svn/text-base/cuActivation.h.svn-base | 123 +++ .../.svn/text-base/cuBiasedLinearity.cc.svn-base | 123 +++ .../.svn/text-base/cuBiasedLinearity.h.svn-base | 85 ++ .../.svn/text-base/cuBlockArray.cc.svn-base | 139 +++ .../.svn/text-base/cuBlockArray.h.svn-base | 83 ++ .../.svn/text-base/cuCRBEDctFeat.h.svn-base | 310 +++++++ src/CuTNetLib/.svn/text-base/cuCache.cc.svn-base | 203 +++++ src/CuTNetLib/.svn/text-base/cuCache.h.svn-base | 74 ++ .../.svn/text-base/cuComponent.h.svn-base | 384 +++++++++ .../.svn/text-base/cuDiscreteLinearity.cc.svn-base | 160 ++++ .../.svn/text-base/cuDiscreteLinearity.h.svn-base | 90 ++ src/CuTNetLib/.svn/text-base/cuNetwork.cc.svn-base | 380 +++++++++ src/CuTNetLib/.svn/text-base/cuNetwork.h.svn-base | 220 +++++ .../.svn/text-base/cuObjectiveFunction.cc.svn-base | 87 ++ .../.svn/text-base/cuObjectiveFunction.h.svn-base | 166 ++++ src/CuTNetLib/.svn/text-base/cuRbm.cc.svn-base | 244 ++++++ src/CuTNetLib/.svn/text-base/cuRbm.h.svn-base | 146 ++++ .../.svn/text-base/cuRbmSparse.cc.svn-base | 269 ++++++ .../.svn/text-base/cuRbmSparse.h.svn-base | 134 +++ .../.svn/text-base/cuRecurrent.cc.svn-base | 191 +++++ .../.svn/text-base/cuRecurrent.h.svn-base | 101 +++ .../.svn/text-base/cuSharedLinearity.cc.svn-base | 179 ++++ .../.svn/text-base/cuSharedLinearity.h.svn-base | 85 ++ .../.svn/text-base/cuSparseLinearity.cc.svn-base | 190 +++++ .../.svn/text-base/cuSparseLinearity.h.svn-base | 104 +++ 47 files changed, 5387 insertions(+) create mode 100644 src/CuTNetLib/.svn/entries create mode 100644 src/CuTNetLib/.svn/prop-base/Makefile.svn-base create mode 100644 src/CuTNetLib/.svn/prop-base/cuActivation.cc.svn-base create mode 100644 src/CuTNetLib/.svn/prop-base/cuActivation.h.svn-base create mode 100644 src/CuTNetLib/.svn/prop-base/cuBiasedLinearity.cc.svn-base create mode 100644 src/CuTNetLib/.svn/prop-base/cuBiasedLinearity.h.svn-base create mode 100644 src/CuTNetLib/.svn/prop-base/cuCRBEDctFeat.h.svn-base create mode 100644 src/CuTNetLib/.svn/prop-base/cuCache.cc.svn-base create mode 100644 src/CuTNetLib/.svn/prop-base/cuCache.h.svn-base create mode 100644 src/CuTNetLib/.svn/prop-base/cuComponent.h.svn-base create mode 100644 src/CuTNetLib/.svn/prop-base/cuNetwork.cc.svn-base create mode 100644 src/CuTNetLib/.svn/prop-base/cuNetwork.h.svn-base create mode 100644 src/CuTNetLib/.svn/prop-base/cuObjectiveFunction.cc.svn-base create mode 100644 src/CuTNetLib/.svn/prop-base/cuObjectiveFunction.h.svn-base create mode 100644 src/CuTNetLib/.svn/prop-base/cuRbm.cc.svn-base create mode 100644 src/CuTNetLib/.svn/prop-base/cuRbm.h.svn-base create mode 100644 src/CuTNetLib/.svn/prop-base/cuRecurrent.cc.svn-base create mode 100644 src/CuTNetLib/.svn/prop-base/cuRecurrent.h.svn-base create mode 100644 src/CuTNetLib/.svn/prop-base/cuSharedLinearity.cc.svn-base create mode 100644 src/CuTNetLib/.svn/prop-base/cuSharedLinearity.h.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/Makefile.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuActivation.cc.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuActivation.h.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuBiasedLinearity.cc.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuBiasedLinearity.h.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuBlockArray.cc.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuBlockArray.h.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuCRBEDctFeat.h.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuCache.cc.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuCache.h.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuComponent.h.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuDiscreteLinearity.cc.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuDiscreteLinearity.h.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuNetwork.cc.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuNetwork.h.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuObjectiveFunction.cc.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuObjectiveFunction.h.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuRbm.cc.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuRbm.h.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuRbmSparse.cc.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuRbmSparse.h.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuRecurrent.cc.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuRecurrent.h.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuSharedLinearity.cc.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuSharedLinearity.h.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuSparseLinearity.cc.svn-base create mode 100644 src/CuTNetLib/.svn/text-base/cuSparseLinearity.h.svn-base (limited to 'src/CuTNetLib/.svn') diff --git a/src/CuTNetLib/.svn/entries b/src/CuTNetLib/.svn/entries new file mode 100644 index 0000000..8846e9b --- /dev/null +++ b/src/CuTNetLib/.svn/entries @@ -0,0 +1,946 @@ +10 + +dir +117 +svn+ssh://merlin.fit.vutbr.cz/svn/TNet/trunk/src/CuTNetLib +svn+ssh://merlin.fit.vutbr.cz/svn/TNet + + + +2012-02-07T17:50:53.635354Z +103 +iveselyk + + + + + + + + + + + + + + +bda6da93-004a-4ae9-8e07-715c10848801 + +cuCache.cc +file + + + + +2012-04-02T13:49:14.000000Z +ba7400672c1166f91f51b1d02b0124ab +2011-12-22T15:49:51.623339Z +96 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +5190 + +cuActivation.h +file + + + + +2012-04-02T13:49:14.000000Z +a6e665ef8082542cddeb9cfb344f01d2 +2011-09-26T14:48:24.142069Z +73 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +2417 + +cuComponent.h +file + + + + +2012-04-02T13:49:14.000000Z +410e86f76bb21318f8e348f7287de915 +2012-02-07T17:50:53.635354Z +103 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +8634 + +cuCache.h +file + + + + +2012-04-02T13:49:14.000000Z +2d53b9558d6ce6627a31f4c76399905e +2011-04-04T17:14:16.666438Z +46 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +2021 + +cuCRBEDctFeat.h +file + + + + +2012-04-02T13:49:14.000000Z +02cfc50560d325a7b919e321c854a888 +2011-09-26T14:48:24.142069Z +73 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +7633 + +cuBiasedLinearity.cc +file + + + + +2012-04-02T13:49:14.000000Z +293eda5bda355fdea6417f94c3657a08 +2011-04-04T17:14:16.666438Z +46 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +3082 + +cuRecurrent.cc +file + + + + +2012-04-02T13:49:14.000000Z +45994d12f52bfb36ecc3b40d041073d0 +2011-03-22T21:01:03.678832Z +41 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +5721 + +cuBiasedLinearity.h +file + + + + +2012-04-02T13:49:14.000000Z +df1f7b17849004a0574744a43befabc8 +2011-09-26T14:48:24.142069Z +73 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +1846 + +cuRecurrent.h +file + + + + +2012-04-02T13:49:14.000000Z +f0005036c7deeb217115a673e6fff070 +2011-09-26T14:48:24.142069Z +73 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +2002 + +cuSparseLinearity.cc +file + + + + +2012-04-02T13:49:14.000000Z +e7121409664c29ea00a3abc0b3bd6caa +2011-09-26T13:47:57.076756Z +70 +iveselyk + + + + + + + + + + + + + + + + + + + + + +4896 + +cuObjectiveFunction.cc +file + + + + +2012-04-02T13:49:14.000000Z +fbedbd40c62459261f9f694f8bf62c29 +2011-03-07T10:43:43.160610Z +40 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +2128 + +cuRbm.cc +file + + + + +2012-04-02T13:49:14.000000Z +e849f983f4533fd34b87295e4d93602f +2011-10-11T11:00:50.704096Z +81 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +6654 + +cuSparseLinearity.h +file + + + + +2012-04-02T13:49:14.000000Z +0d38089478e85f485e737a1f5a5e24ad +2011-09-26T14:48:24.142069Z +73 +iveselyk + + + + + + + + + + + + + + + + + + + + + +2570 + +cuObjectiveFunction.h +file + + + + +2012-04-02T13:49:14.000000Z +ba18c448975a869560b72311cefdca58 +2011-04-04T17:14:16.666438Z +46 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +3792 + +cuRbm.h +file + + + + +2012-04-02T13:49:14.000000Z +065ff585dcb58651e1084e90a447290a +2011-12-08T10:59:03.566125Z +94 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +3522 + +cuDiscreteLinearity.cc +file + + + + +2012-04-02T13:49:14.000000Z +d5580b34ae83cdadd40902f705edb990 +2011-12-08T10:59:03.566125Z +94 +iveselyk + + + + + + + + + + + + + + + + + + + + + +4235 + +cuDiscreteLinearity.h +file + + + + +2012-04-02T13:49:14.000000Z +1bc0c4c0ab790c9585a272fb63319041 +2011-12-08T10:59:03.566125Z +94 +iveselyk + + + + + + + + + + + + + + + + + + + + + +1971 + +cuSharedLinearity.cc +file + + + + +2012-04-02T13:49:14.000000Z +ee6608a11be7ded2ba5c60b009e4846a +2011-04-04T17:14:16.666438Z +46 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +5282 + +cuNetwork.cc +file + + + + +2012-04-02T13:49:14.000000Z +6bd052ddb4d464abf3dc8ef5ef17a4b3 +2012-02-07T17:50:53.635354Z +103 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +9320 + +cuBlockArray.cc +file + + + + +2012-04-02T13:49:14.000000Z +641f990dfe9ed2666cf076a43982556e +2012-02-07T17:50:53.635354Z +103 +iveselyk + + + + + + + + + + + + + + + + + + + + + +3240 + +cuSharedLinearity.h +file + + + + +2012-04-02T13:49:14.000000Z +fa4ddac3fab6f14d678184aef599c269 +2011-09-26T14:48:24.142069Z +73 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +1776 + +cuNetwork.h +file + + + + +2012-04-02T13:49:14.000000Z +af7fdb7f8e37a6159ed4802eba57e25b +2011-09-19T11:12:27.685840Z +69 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +5499 + +cuBlockArray.h +file + + + + +2012-04-02T13:49:14.000000Z +0edf09ffe051998c7f2dd04fec19914c +2012-02-07T17:50:53.635354Z +103 +iveselyk + + + + + + + + + + + + + + + + + + + + + +1529 + +cuRbmSparse.cc +file + + + + +2012-04-02T13:49:14.000000Z +626fd92e997ff3c18dbecef8150c5e80 +2011-12-08T10:59:03.566125Z +94 +iveselyk + + + + + + + + + + + + + + + + + + + + + +7681 + +cuRbmSparse.h +file + + + + +2012-04-02T13:49:14.000000Z +a8c1d1d2a7dcf461c4180a1eb34c7991 +2011-12-08T10:59:03.566125Z +94 +iveselyk + + + + + + + + + + + + + + + + + + + + + +3500 + +cuActivation.cc +file + + + + +2012-04-02T13:49:14.000000Z +4c43c28a4c87e7fddad7958a3cfe0e1f +2011-02-24T12:12:08.754106Z +34 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +694 + +Makefile +file + + + + +2012-04-02T13:49:14.000000Z +57c55cf8cb29bdb2590cfb9d6ba02413 +2011-03-24T17:03:17.103393Z +43 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +383 + diff --git a/src/CuTNetLib/.svn/prop-base/Makefile.svn-base b/src/CuTNetLib/.svn/prop-base/Makefile.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/Makefile.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuActivation.cc.svn-base b/src/CuTNetLib/.svn/prop-base/cuActivation.cc.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuActivation.cc.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuActivation.h.svn-base b/src/CuTNetLib/.svn/prop-base/cuActivation.h.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuActivation.h.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuBiasedLinearity.cc.svn-base b/src/CuTNetLib/.svn/prop-base/cuBiasedLinearity.cc.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuBiasedLinearity.cc.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuBiasedLinearity.h.svn-base b/src/CuTNetLib/.svn/prop-base/cuBiasedLinearity.h.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuBiasedLinearity.h.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuCRBEDctFeat.h.svn-base b/src/CuTNetLib/.svn/prop-base/cuCRBEDctFeat.h.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuCRBEDctFeat.h.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuCache.cc.svn-base b/src/CuTNetLib/.svn/prop-base/cuCache.cc.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuCache.cc.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuCache.h.svn-base b/src/CuTNetLib/.svn/prop-base/cuCache.h.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuCache.h.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuComponent.h.svn-base b/src/CuTNetLib/.svn/prop-base/cuComponent.h.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuComponent.h.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuNetwork.cc.svn-base b/src/CuTNetLib/.svn/prop-base/cuNetwork.cc.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuNetwork.cc.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuNetwork.h.svn-base b/src/CuTNetLib/.svn/prop-base/cuNetwork.h.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuNetwork.h.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuObjectiveFunction.cc.svn-base b/src/CuTNetLib/.svn/prop-base/cuObjectiveFunction.cc.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuObjectiveFunction.cc.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuObjectiveFunction.h.svn-base b/src/CuTNetLib/.svn/prop-base/cuObjectiveFunction.h.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuObjectiveFunction.h.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuRbm.cc.svn-base b/src/CuTNetLib/.svn/prop-base/cuRbm.cc.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuRbm.cc.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuRbm.h.svn-base b/src/CuTNetLib/.svn/prop-base/cuRbm.h.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuRbm.h.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuRecurrent.cc.svn-base b/src/CuTNetLib/.svn/prop-base/cuRecurrent.cc.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuRecurrent.cc.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuRecurrent.h.svn-base b/src/CuTNetLib/.svn/prop-base/cuRecurrent.h.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuRecurrent.h.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuSharedLinearity.cc.svn-base b/src/CuTNetLib/.svn/prop-base/cuSharedLinearity.cc.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuSharedLinearity.cc.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuSharedLinearity.h.svn-base b/src/CuTNetLib/.svn/prop-base/cuSharedLinearity.h.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuSharedLinearity.h.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/text-base/Makefile.svn-base b/src/CuTNetLib/.svn/text-base/Makefile.svn-base new file mode 100644 index 0000000..c7678da --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/Makefile.svn-base @@ -0,0 +1,30 @@ + +include ../tnet.mk + +INCLUDE= -I. -I../ -I../KaldiLib -I../CuBaseLib -I../TNetLib -I$(CUDA_TK_BASE)/include/ + + +all : libCuTNet.a + +libCuTNet.a : $(OBJ) + $(AR) ruv $@ $? + $(RANLIB) $@ + + +%.o : %.cc + $(CXX) -c $< -o $@ $(CFLAGS) $(CXXFLAGS) $(INCLUDE) + + + + +.PHONY: clean depend + +clean : + rm -f *.o *.a + + +depend: + $(CXX) -M $(CXXFLAGS) *.cc $(INCLUDE) > .depend.mk + +-include .depend.mk + diff --git a/src/CuTNetLib/.svn/text-base/cuActivation.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuActivation.cc.svn-base new file mode 100644 index 0000000..bd57ae5 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuActivation.cc.svn-base @@ -0,0 +1,46 @@ + +#include "cuActivation.h" +#include "cumath.h" + + +namespace TNet { + + + void + CuSigmoid:: + PropagateFnc(const CuMatrix& X, CuMatrix& Y) + { + CuMath::Sigmoid(Y, X); + } + + + void + CuSigmoid:: + BackpropagateFnc(const CuMatrix& X, CuMatrix& Y) + { + CuMath::DiffSigmoid(Y, X, mOutput); + } + + + + void + CuSoftmax:: + PropagateFnc(const CuMatrix& X, CuMatrix& Y) + { + CuMath::Softmax(Y,X); + } + + + + void + CuSoftmax:: + BackpropagateFnc(const CuMatrix& X, CuMatrix& Y) + { + //we assume X is already dE/dSoftmax_input + Y.CopyFrom(X); + } + + + +} //namespace + diff --git a/src/CuTNetLib/.svn/text-base/cuActivation.h.svn-base b/src/CuTNetLib/.svn/text-base/cuActivation.h.svn-base new file mode 100644 index 0000000..9fb2862 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuActivation.h.svn-base @@ -0,0 +1,123 @@ + +#ifndef _CUACT_FUN_I_ +#define _CUACT_FUN_I_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +namespace TNet +{ + + /** + * Common interface for activation functions + */ + class CuActivation : public CuComponent + { + public: + CuActivation(size_t nInputs, size_t nOutputs, CuComponent *pPred); + + protected: + }; + + + /** + * Sigmoid activation function + */ + class CuSigmoid : public CuActivation + { + public: + CuSigmoid(size_t nInputs, size_t nOutputs, CuComponent *pPred); + + ComponentType GetType() const; + const char* GetName() const; + + protected: + void PropagateFnc(const CuMatrix& X, CuMatrix& Y); + void BackpropagateFnc(const CuMatrix& X, CuMatrix& Y); + }; + + + /** + * Softmax activation function + */ + class CuSoftmax : public CuActivation + { + public: + CuSoftmax(size_t nInputs, size_t nOutputs, CuComponent *pPred); + + ComponentType GetType() const; + const char* GetName() const; + + protected: + void PropagateFnc(const CuMatrix& X, CuMatrix& Y); + void BackpropagateFnc(const CuMatrix& X, CuMatrix& Y); + }; + + + ////////////////////////////////////////////////////////////////////////// + // Inline functions + // Activation:: + inline + CuActivation:: + CuActivation(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuComponent(nInputs,nOutputs, pPred) + { + assert(nInputs == nOutputs); + } + + + ////////////////////////////////////////////////////////////////////////// + // Inline functions + // Sigmoid:: + inline + CuSigmoid:: + CuSigmoid(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuActivation(nInputs,nOutputs, pPred) + { } + + inline CuComponent::ComponentType + CuSigmoid:: + GetType() const + { + return CuComponent::SIGMOID; + } + + inline const char* + CuSigmoid:: + GetName() const + { + return ""; + } + + + + ////////////////////////////////////////////////////////////////////////// + // Inline functions + // Softmax:: + inline + CuSoftmax:: + CuSoftmax(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuActivation(nInputs,nOutputs, pPred) + { } + + inline CuComponent::ComponentType + CuSoftmax:: + GetType() const + { + return CuComponent::SOFTMAX; + } + + inline const char* + CuSoftmax:: + GetName() const + { + return ""; + } + + +} //namespace + + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuBiasedLinearity.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuBiasedLinearity.cc.svn-base new file mode 100644 index 0000000..b9ac137 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuBiasedLinearity.cc.svn-base @@ -0,0 +1,123 @@ + + +#include "cuBiasedLinearity.h" + + +namespace TNet +{ + + void + CuBiasedLinearity:: + PropagateFnc(const CuMatrix& X, CuMatrix& Y) + { + //Y.SetConst(0.0); + Y.AddScaledRow(1.0,mBias,0.0); + Y.Gemm('N','N', 1.0, X, mLinearity, 1.0); + } + + + void + CuBiasedLinearity:: + BackpropagateFnc(const CuMatrix& X, CuMatrix& Y) + { + //Y.SetConst(0.0); + Y.Gemm('N', 'T', 1.0, X, mLinearity, 0.0); + } + + + void + CuBiasedLinearity:: + Update() + { +#if 0 + //former implementation + BaseFloat N = static_cast(GetInput().Rows()); + + mLinearityCorrection.Gemm('T','N',-mLearningRate/N,GetInput(),GetErrorInput(),mMomentum); + mBiasCorrection.AddColSum(-mLearningRate/N,GetErrorInput(),mMomentum); + + //regularization weight decay + mLinearityCorrection.AddScaled(-mLearningRate*mWeightcost,mLinearity,1.0); + + mLinearity.AddScaled(1.0,mLinearityCorrection,1.0); + mBias.AddScaled(1.0,mBiasCorrection,1.0); +#endif + +#if 1 + //new implementation + BaseFloat N = 1; + if(mGradDivFrm) { + N = static_cast(GetInput().Rows()); + } + BaseFloat mmt_gain = static_cast(1.0/(1.0-mMomentum)); + N *= mmt_gain; + + mLinearityCorrection.Gemm('T','N',1.0,GetInput(),GetErrorInput(),mMomentum); + mBiasCorrection.AddColSum(1.0,GetErrorInput(),mMomentum); + + mLinearity.AddScaled(-mLearningRate/N,mLinearityCorrection,1.0); + mBias.AddScaled(-mLearningRate/N,mBiasCorrection,1.0); + + //regularization weight decay (from actual weights only) + BaseFloat L2_decay = -mLearningRate*mWeightcost*(mGradDivFrm?1.0:GetInput().Rows()); + mLinearity.AddScaled(L2_decay, mLinearity,1.0); +#endif + } + + + void + CuBiasedLinearity:: + ReadFromStream(std::istream& rIn) + { + //matrix is stored transposed as SNet does + BfMatrix transpose; + rIn >> transpose; + mLinearity.CopyFrom(BfMatrix(transpose, TRANS)); + //biases stored normally + BfVector bias; + rIn >> bias; + mBias.CopyFrom(bias); + + if(transpose.Cols()*transpose.Rows() == 0) { + Error("Missing linearity matrix in network file"); + } + if(bias.Dim() == 0) { + Error("Missing bias vector in network file"); + } + if(mLinearity.Cols() != GetNOutputs() || + mLinearity.Rows() != GetNInputs() || + mBias.Dim() != GetNOutputs() + ){ + std::ostringstream os; + os << "Wrong dimensionalities of matrix/vector in network file\n" + << "Inputs:" << GetNInputs() + << "Outputs:" << GetNOutputs() + << "\n" + << "linearityCols:" << mLinearity.Cols() + << "linearityRows:" << mLinearity.Rows() + << "biasDims:" << mBias.Dim() + << "\n"; + Error(os.str()); + } + } + + + void + CuBiasedLinearity:: + WriteToStream(std::ostream& rOut) + { + //matrix is stored transposed as SNet does + BfMatrix tmp; + mLinearity.CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + //biases stored normally + BfVector vec; + mBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + } + + +} //namespace + diff --git a/src/CuTNetLib/.svn/text-base/cuBiasedLinearity.h.svn-base b/src/CuTNetLib/.svn/text-base/cuBiasedLinearity.h.svn-base new file mode 100644 index 0000000..18a75ac --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuBiasedLinearity.h.svn-base @@ -0,0 +1,85 @@ +#ifndef _CUBIASED_LINEARITY_H_ +#define _CUBIASED_LINEARITY_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + + class CuBiasedLinearity : public CuUpdatableComponent + { + public: + + CuBiasedLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuBiasedLinearity(); + + ComponentType GetType() const; + const char* GetName() const; + + void PropagateFnc(const CuMatrix& X, CuMatrix& Y); + void BackpropagateFnc(const CuMatrix& X, CuMatrix& Y); + + void Update(); + + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + CuMatrix mLinearity; ///< Matrix with neuron weights + CuVector mBias; ///< Vector with biases + + CuMatrix mLinearityCorrection; ///< Matrix for linearity updates + CuVector mBiasCorrection; ///< Vector for bias updates + + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuBiasedLinearity:: + inline + CuBiasedLinearity:: + CuBiasedLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred), + mLinearity(nInputs,nOutputs), mBias(nOutputs), + mLinearityCorrection(nInputs,nOutputs), mBiasCorrection(nOutputs) + { + mLinearityCorrection.SetConst(0.0); + mBiasCorrection.SetConst(0.0); + } + + + inline + CuBiasedLinearity:: + ~CuBiasedLinearity() + { } + + inline CuComponent::ComponentType + CuBiasedLinearity:: + GetType() const + { + return CuComponent::BIASED_LINEARITY; + } + + inline const char* + CuBiasedLinearity:: + GetName() const + { + return ""; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuBlockArray.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuBlockArray.cc.svn-base new file mode 100644 index 0000000..461bd37 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuBlockArray.cc.svn-base @@ -0,0 +1,139 @@ + + +#include "cuBlockArray.h" +#include "cuNetwork.h" + + +namespace TNet +{ + + void + CuBlockArray:: + PropagateFnc(const CuMatrix& X, CuMatrix& Y) + { + CuMatrix colsX; + CuMatrix colsY; + + int X_src_ori=0, Y_tgt_ori=0; + for(int i=0; iGetNInputs(); + colsX.Init(X.Rows(),colsX_cnt); + colsX.CopyCols(colsX_cnt,X_src_ori,X,0); + + //propagate through the block(network) + mBlocks[i]->Propagate(colsX,colsY); + + //copy column stripe to the output Y + int colsY_cnt=mBlocks[i]->GetNOutputs(); + Y.CopyCols(colsY_cnt,0,colsY,Y_tgt_ori); + + //shift the origin coordinates + X_src_ori += colsX_cnt; + Y_tgt_ori += colsY_cnt; + } + + assert(X_src_ori == X.Cols()); + assert(Y_tgt_ori == Y.Cols()); + } + + + void + CuBlockArray:: + BackpropagateFnc(const CuMatrix& X, CuMatrix& Y) + { + KALDI_ERR << "Unimplemented"; + } + + + void + CuBlockArray:: + Update() + { + KALDI_ERR << "Unimplemented"; + } + + + void + CuBlockArray:: + ReadFromStream(std::istream& rIn) + { + if(mBlocks.size() > 0) { + KALDI_ERR << "Cannot read block vector, " + << "aleady filled bt " + << mBlocks.size() + << "elements"; + } + + rIn >> std::ws >> mNBlocks; + if(mNBlocks < 1) { + KALDI_ERR << "Bad number of blocks:" << mNBlocks; + } + + //read all the blocks + std::string tag; + int block_id; + for(int i=0; i + rIn >> std::ws >> tag; + //make it lowercase + std::transform(tag.begin(), tag.end(), tag.begin(), tolower); + //check + if(tag!="") { + KALDI_ERR << " keywotd expected"; + } + + //read block number + rIn >> std::ws >> block_id; + if(block_id != i+1) { + KALDI_ERR << "Expected block number:" << i+1 + << " read block number: " << block_id; + } + + //read the nnet + CuNetwork* p_nnet = new CuNetwork; + p_nnet->ReadNetwork(rIn); + if(p_nnet->Layers() == 0) { + KALDI_ERR << "Cannot read empty network to a block"; + } + + //add it to the vector + mBlocks.push_back(p_nnet); + } + + //check the declared dimensionality + int sum_inputs=0, sum_outputs=0; + for(int i=0; iGetNInputs(); + sum_outputs += mBlocks[i]->GetNOutputs(); + } + if(sum_inputs != GetNInputs()) { + KALDI_ERR << "Non-matching number of INPUTS! Declared:" + << GetNInputs() + << " summed from blocks" + << sum_inputs; + } + if(sum_outputs != GetNOutputs()) { + KALDI_ERR << "Non-matching number of OUTPUTS! Declared:" + << GetNOutputs() + << " summed from blocks" + << sum_outputs; + } + } + + + void + CuBlockArray:: + WriteToStream(std::ostream& rOut) + { + rOut << " " << mBlocks.size() << " "; + for(int i=0; i " << i+1 << "\n"; + mBlocks[i]->WriteNetwork(rOut); + rOut << "\n"; + } + } + + +} //namespace + diff --git a/src/CuTNetLib/.svn/text-base/cuBlockArray.h.svn-base b/src/CuTNetLib/.svn/text-base/cuBlockArray.h.svn-base new file mode 100644 index 0000000..aea7922 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuBlockArray.h.svn-base @@ -0,0 +1,83 @@ +#ifndef _CUBLOCK_ARRAY_H_ +#define _CUBLOCK_ARRAY_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + + class CuNetwork; + + class CuBlockArray : public CuUpdatableComponent + { + public: + + CuBlockArray(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuBlockArray(); + + ComponentType GetType() const; + const char* GetName() const; + + void PropagateFnc(const CuMatrix& X, CuMatrix& Y); + void BackpropagateFnc(const CuMatrix& X, CuMatrix& Y); + + void Update(); + + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + std::vector mBlocks; ///< vector with networks, one network is one block + size_t mNBlocks; + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuBlockArray:: + inline + CuBlockArray:: + CuBlockArray(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred), + mNBlocks(0) + { } + + + inline + CuBlockArray:: + ~CuBlockArray() + { + for(int i=0; i"; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuCRBEDctFeat.h.svn-base b/src/CuTNetLib/.svn/text-base/cuCRBEDctFeat.h.svn-base new file mode 100644 index 0000000..62c62f2 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuCRBEDctFeat.h.svn-base @@ -0,0 +1,310 @@ +#ifndef _CUCRBEDCTFEATURES_H_ +#define _CUCRBEDCTFEATURES_H_ + + +#include "cuComponent.h" +#include "cumath.h" + + +namespace TNet { + + /** + * Expands the time context of the input features + * in N, out k*N, FrameOffset o_1,o_2,...,o_k + * FrameOffset example 11frames: -5 -4 -3 -2 -1 0 1 2 3 4 5 + */ + class CuExpand : public CuComponent + { + public: + CuExpand(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred) + { } + + ~CuExpand() + { } + + ComponentType GetType() const + { return EXPAND; } + + const char* GetName() const + { return ""; } + + void ReadFromStream(std::istream& rIn) + { Vector vec; rIn >> vec; mFrameOffset.CopyFrom(vec); } + + void WriteToStream(std::ostream& rOut) + { Vector vec; mFrameOffset.CopyTo(vec); rOut << vec; } + + protected: + void PropagateFnc(const CuMatrix& X, CuMatrix& Y) + { CuMath::Expand(Y,X,mFrameOffset); } + + void BackpropagateFnc(const CuMatrix& X, CuMatrix& Y) + { Error("__func__ Nonsense"); } + + protected: + CuVector mFrameOffset; + }; + + + + /** + * Rearrange the matrix columns according to the indices in mCopyFromIndices + */ + class CuCopy : public CuComponent + { + public: + CuCopy(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred) + { } + + ~CuCopy() + { } + + ComponentType GetType() const + { return COPY; } + + const char* GetName() const + { return ""; } + + void ReadFromStream(std::istream& rIn) + { Vector vec; rIn >> vec; vec.Add(-1); mCopyFromIndices.CopyFrom(vec); } + + void WriteToStream(std::ostream& rOut) + { Vector vec; mCopyFromIndices.CopyTo(vec); vec.Add(1); rOut << vec; } + + protected: + void PropagateFnc(const CuMatrix& X, CuMatrix& Y) + { CuMath::Rearrange(Y,X,mCopyFromIndices); } + + void BackpropagateFnc(const CuMatrix& X, CuMatrix& Y) + { Error("__func__ Nonsense"); } + + protected: + CuVector mCopyFromIndices; + }; + + class CuTranspose : public CuComponent + { + public: + CuTranspose(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred), mContext(0) + { } + + ~CuTranspose() + { } + + ComponentType GetType() const + { return TRANSPOSE; } + + const char* GetName() const + { return ""; } + + void ReadFromStream(std::istream& rIn) + { + rIn >> std::ws >> mContext; + + if(GetNInputs() != GetNOutputs()) { + Error("Input dim must be same as output dim"); + } + if(GetNInputs() % mContext != 0) { + Error("Number of inputs must be divisible by context length"); + } + + Vector vec(GetNInputs()); + int channels = GetNInputs() / mContext; + for(int i=0, ch=0; ch& X, CuMatrix& Y) + { CuMath::Rearrange(Y,X,mCopyFromIndices); } + + void BackpropagateFnc(const CuMatrix& X, CuMatrix& Y) + { Error("__func__ Nonsense"); } + + protected: + int mContext; + CuVector mCopyFromIndices; + }; + + + /** + * CuBlockLinearity is used for the blockwise multiplication by + * DCT transform loaded from disk + */ + class CuBlockLinearity : public CuComponent + { + public: + CuBlockLinearity(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred) + { } + + ~CuBlockLinearity() + { } + + + ComponentType GetType() const + { return CuComponent::BLOCK_LINEARITY; } + + const char* GetName() const + { return ""; } + + + void PropagateFnc(const CuMatrix& X, CuMatrix& Y) + { CuMath::BlockLinearity(Y,X,mBlockLinearity); } + + void BackpropagateFnc(const CuMatrix& X, CuMatrix& Y) + { Error("__func__ Not implemented"); } + + + void ReadFromStream(std::istream& rIn) + { + Matrix mat; + rIn >> mat; + Matrix trans(mat,TRANS); + mBlockLinearity.CopyFrom(trans); + + if((GetNOutputs() % mBlockLinearity.Cols() != 0) || + (GetNInputs() % mBlockLinearity.Rows() != 0) || + ((GetNOutputs() / mBlockLinearity.Cols()) != + (GetNInputs() / mBlockLinearity.Rows()))) + { + Error("BlockLinearity matrix dimensions must divide IO dims"); + } + } + + void WriteToStream(std::ostream& rOut) + { + Matrix mat; + mBlockLinearity.CopyTo(mat); + Matrix trans(mat,TRANS); + rOut << trans; + } + + private: + CuMatrix mBlockLinearity; + }; + + + + class CuBias : public CuComponent + { + public: + CuBias(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred) + { } + + ~CuBias() + { } + + + ComponentType GetType() const + { return CuComponent::BIAS; } + + const char* GetName() const + { return ""; } + + + void PropagateFnc(const CuMatrix& X, CuMatrix& Y) + { Y.CopyFrom(X); Y.AddScaledRow(1.0, mBias, 1.0); } + + void BackpropagateFnc(const CuMatrix& X, CuMatrix& Y) + { Y.CopyFrom(X); } + + + void ReadFromStream(std::istream& rIn) + { Vector vec; rIn >> vec; mBias.CopyFrom(vec); } + + void WriteToStream(std::ostream& rOut) + { Vector vec; mBias.CopyTo(vec); rOut << vec; } + + private: + CuVector mBias; + }; + + + + class CuWindow : public CuComponent + { + public: + CuWindow(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs, nOutputs, pPred) + { } + + ~CuWindow() + { } + + + ComponentType GetType() const + { return CuComponent::WINDOW; } + + const char* GetName() const + { return ""; } + + + void PropagateFnc(const CuMatrix& X, CuMatrix& Y) + { Y.CopyFrom(X); Y.ScaleCols(mWindow); } + + void BackpropagateFnc(const CuMatrix& X, CuMatrix& Y) + { Error("__func__ Not implemented"); } + + + void ReadFromStream(std::istream& rIn) + { Vector vec; rIn >> vec; mWindow.CopyFrom(vec); } + + void WriteToStream(std::ostream& rOut) + { Vector vec; mWindow.CopyTo(vec); rOut << vec; } + + private: + CuVector mWindow; + }; + + class CuLog : public CuComponent + { + public: + CuLog(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs, nOutputs, pPred) + { } + + ~CuLog() + { } + + + ComponentType GetType() const + { return CuComponent::LOG; } + + const char* GetName() const + { return ""; } + + + void PropagateFnc(const CuMatrix& X, CuMatrix& Y) + { Y.CopyFrom(X); Y.ApplyLog(); } + + void BackpropagateFnc(const CuMatrix& X, CuMatrix& Y) + { Error("__func__ Not implemented"); } + + + void ReadFromStream(std::istream& rIn) + { } + + void WriteToStream(std::ostream& rOut) + { } + + }; + +} + + +#endif + diff --git a/src/CuTNetLib/.svn/text-base/cuCache.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuCache.cc.svn-base new file mode 100644 index 0000000..f96b3b1 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuCache.cc.svn-base @@ -0,0 +1,203 @@ + + +#include "cuCache.h" +#include "cumath.h" + + + +namespace TNet { + + CuCache:: + CuCache() + : mState(EMPTY), mIntakePos(0), mExhaustPos(0), mDiscarded(0), + mRandomized(false), mTrace(0) + { } + + CuCache:: + ~CuCache() + { } + + void + CuCache:: + Init(size_t cachesize, size_t bunchsize) + { + if((cachesize % bunchsize) != 0) { + Error("Non divisible cachesize by bunchsize"); + } + + mCachesize = cachesize; + mBunchsize = bunchsize; + + mState = EMPTY; + + mIntakePos = 0; + mExhaustPos = 0; + + mRandomized = false; + + } + + void + CuCache:: + AddData(const CuMatrix& rFeatures, const CuMatrix& rDesired) + { + assert(rFeatures.Rows() == rDesired.Rows()); + + //lazy buffers allocation + if(mFeatures.Rows() != mCachesize) { + mFeatures.Init(mCachesize,rFeatures.Cols()); + mDesired.Init(mCachesize,rDesired.Cols()); + } + + //warn if segment longer than half-cache + if(rFeatures.Rows() > mCachesize/2) { + std::ostringstream os; + os << "Too long segment and small feature cache! " + << " cachesize: " << mCachesize + << " segmentsize: " << rFeatures.Rows(); + Warning(os.str()); + } + + //change state + if(mState == EMPTY) { + if(mTrace&3) std::cout << "/" << std::flush; + mState = INTAKE; mIntakePos = 0; + + //check for leftover from previous segment + int leftover = mFeaturesLeftover.Rows(); + //check if leftover is not bigger than cachesize + if(leftover > mCachesize) { + std::ostringstream os; + os << "Too small feature cache: " << mCachesize + << ", truncating: " + << leftover - mCachesize << " frames from previous segment leftover"; + //Error(os.str()); + Warning(os.str()); + leftover = mCachesize; + } + //prefill cache with leftover + if(leftover > 0) { + mFeatures.CopyRows(leftover,0,mFeaturesLeftover,0); + mDesired.CopyRows(leftover,0,mDesiredLeftover,0); + mFeaturesLeftover.Destroy(); + mDesiredLeftover.Destroy(); + mIntakePos += leftover; + } + } + + assert(mState == INTAKE); + assert(rFeatures.Rows() == rDesired.Rows()); + if(mTrace&2) std::cout << "F" << std::flush; + + int cache_space = mCachesize - mIntakePos; + int feature_length = rFeatures.Rows(); + int fill_rows = (cache_space 0); + + //copy the data to cache + mFeatures.CopyRows(fill_rows,0,rFeatures,mIntakePos); + mDesired.CopyRows(fill_rows,0,rDesired,mIntakePos); + + //copy leftovers + if(leftover > 0) { + mFeaturesLeftover.Init(leftover,mFeatures.Cols()); + mDesiredLeftover.Init(leftover,mDesired.Cols()); + mFeaturesLeftover.CopyRows(leftover,fill_rows,rFeatures,0); + mDesiredLeftover.CopyRows(leftover,fill_rows,rDesired,0); + } + + //update cursor + mIntakePos += fill_rows; + + //change state + if(mIntakePos == mCachesize) { + if(mTrace&3) std::cout << "\\" << std::flush; + mState = FULL; + } + } + + + + void + CuCache:: + Randomize() + { + assert(mState == FULL || mState == INTAKE); + + if(mTrace&3) std::cout << "R" << std::flush; + + //lazy initialization of hte output buffers + mFeaturesRandom.Init(mCachesize,mFeatures.Cols()); + mDesiredRandom.Init(mCachesize,mDesired.Cols()); + + //generate random series of integers + Vector randmask(mIntakePos); + for(unsigned int i=0; i cu_randmask; + cu_randmask.CopyFrom(randmask); + + //randomize + CuMath::Randomize(mFeaturesRandom,mFeatures,cu_randmask); + CuMath::Randomize(mDesiredRandom,mDesired,cu_randmask); + + mRandomized = true; + + } + + void + CuCache:: + GetBunch(CuMatrix& rFeatures, CuMatrix& rDesired) + { + if(mState == EMPTY) { + Error("GetBunch on empty cache!!!"); + } + + //change state if full... + if(mState == FULL) { + if(mTrace&3) std::cout << "\\" << std::flush; + mState = EXHAUST; mExhaustPos = 0; + } + + //final cache is not completely filled + if(mState == INTAKE) //&& mpFeatures->EndOfList() + { + if(mTrace&3) std::cout << "\\-LAST\n" << std::flush; + mState = EXHAUST; mExhaustPos = 0; + } + + assert(mState == EXHAUST); + + //init the output + rFeatures.Init(mBunchsize,mFeatures.Cols()); + rDesired.Init(mBunchsize,mDesired.Cols()); + + //copy the output + if(mRandomized) { + rFeatures.CopyRows(mBunchsize,mExhaustPos,mFeaturesRandom,0); + rDesired.CopyRows(mBunchsize,mExhaustPos,mDesiredRandom,0); + } else { + rFeatures.CopyRows(mBunchsize,mExhaustPos,mFeatures,0); + rDesired.CopyRows(mBunchsize,mExhaustPos,mDesired,0); + } + + //update cursor + mExhaustPos += mBunchsize; + + //change state to EMPTY + if(mExhaustPos > mIntakePos-mBunchsize) { + //we don't have more complete bunches... + mDiscarded += mIntakePos - mExhaustPos; + + mState = EMPTY; + } + } + + +} diff --git a/src/CuTNetLib/.svn/text-base/cuCache.h.svn-base b/src/CuTNetLib/.svn/text-base/cuCache.h.svn-base new file mode 100644 index 0000000..cef2dd9 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuCache.h.svn-base @@ -0,0 +1,74 @@ +#ifndef _CUCACHE_H_ +#define _CUCACHE_H_ + +#include "cumatrix.h" + +namespace TNet { + + + /** + * The feature-target pair cache + */ + class CuCache { + typedef enum { EMPTY, INTAKE, FULL, EXHAUST } State; + public: + CuCache(); + ~CuCache(); + + /// Initialize the cache + void Init(size_t cachesize, size_t bunchsize); + + /// Add data to cache, returns number of added vectors + void AddData(const CuMatrix& rFeatures, const CuMatrix& rDesired); + /// Randomizes the cache + void Randomize(); + /// Get the bunch of training data + void GetBunch(CuMatrix& rFeatures, CuMatrix& rDesired); + + + /// Returns true if the cache was completely filled + bool Full() + { return (mState == FULL); } + + /// Returns true if the cache is empty + bool Empty() + { return (mState == EMPTY || mIntakePos < mBunchsize); } + + /// Number of discarded frames + int Discarded() + { return mDiscarded; } + + /// Set the trace message level + void Trace(int trace) + { mTrace = trace; } + + private: + + static long int GenerateRandom(int max) + { return lrand48() % max; } + + State mState; ///< Current state of the cache + + size_t mIntakePos; ///< Number of intaken vectors by AddData + size_t mExhaustPos; ///< Number of exhausted vectors by GetBunch + + size_t mCachesize; ///< Size of cache + size_t mBunchsize; ///< Size of bunch + int mDiscarded; ///< Number of discarded frames + + CuMatrix mFeatures; ///< Feature cache + CuMatrix mFeaturesRandom; ///< Feature cache + CuMatrix mFeaturesLeftover; ///< Feature cache + + CuMatrix mDesired; ///< Desired vector cache + CuMatrix mDesiredRandom; ///< Desired vector cache + CuMatrix mDesiredLeftover; ///< Desired vector cache + + bool mRandomized; + + int mTrace; + }; + +} + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuComponent.h.svn-base b/src/CuTNetLib/.svn/text-base/cuComponent.h.svn-base new file mode 100644 index 0000000..332c156 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuComponent.h.svn-base @@ -0,0 +1,384 @@ +#ifndef _CUNETWORK_COMPONENT_I_H +#define _CUNETWORK_COMPONENT_I_H + + +#include "Vector.h" +#include "Matrix.h" +#include "Error.h" + +#include "cumatrix.h" + +#include +#include + + +namespace TNet { + + + /** + * Basic element of the network, + * it is a box with defined inputs and outputs, + * and functions to refresh outputs + * + * it is able to compute tranformation function (forward pass) + * and jacobian function (backward pass), + * which is to be implemented in descendents + */ + class CuComponent + { + public: + /// Types of the net components + typedef enum { + UPDATABLE_COMPONENT = 0x0100, + BIASED_LINEARITY, + DISCRETE_LINEARITY, + SHARED_LINEARITY, + SPARSE_LINEARITY, + RBM, + RBM_SPARSE, + RECURRENT, + + ACT_FUN = 0x0200, + SOFTMAX, + SIGMOID, + + OTHER = 0x0400, + EXPAND, + COPY, + TRANSPOSE, + BLOCK_LINEARITY, + WINDOW, + BIAS, + LOG, + + BLOCK_ARRAY, + } ComponentType; + + ////////////////////////////////////////////////////////////// + // Constructor & Destructor + public: + CuComponent(size_t nInputs, size_t nOutputs, CuComponent *pPred); + virtual ~CuComponent(); + + ////////////////////////////////////////////////////////////// + // Interface specification (public) + public: + /// Get Type Identification of the component + virtual ComponentType GetType() const = 0; + /// Get Type Label of the component + virtual const char* GetName() const = 0; + /// + virtual bool IsUpdatable() const + { return false; } + + /// Get size of input vectors + size_t GetNInputs() const; + /// Get size of output vectors + size_t GetNOutputs() const; + + /// IO Data getters + const CuMatrix& GetInput() const; + const CuMatrix& GetOutput() const; + const CuMatrix& GetErrorInput() const; + const CuMatrix& GetErrorOutput() const; + + /// Set input vector (bind with the preceding NetworkComponent) + void SetInput(const CuMatrix& rInput); + /// Set error input vector (bind with the following NetworkComponent) + void SetErrorInput(const CuMatrix& rErrorInput); + + /// Perform forward pass propagateion Input->Output + void Propagate(); + /// Perform backward pass propagateion ErrorInput->ErrorOutput + void Backpropagate(); + + /// Reads the component parameters from stream + virtual void ReadFromStream(std::istream& rIn) { } + /// Writes the components parameters to stream + virtual void WriteToStream(std::ostream& rOut) { } + + + /////////////////////////////////////////////////////////////// + // Nonpublic member functions used to update data outputs + protected: + /// Forward pass transformation (to be implemented by descendents...) + virtual void PropagateFnc(const CuMatrix& X, CuMatrix& Y) = 0; + /// Backward pass transformation (to be implemented by descendents...) + virtual void BackpropagateFnc(const CuMatrix& X, CuMatrix& Y) = 0; + + + /////////////////////////////////////////////////////////////// + // data members + protected: + + size_t mNInputs; ///< Size of input vectors + size_t mNOutputs; ///< Size of output vectors + + const CuMatrix* mpInput; ///< inputs are NOT OWNED by component + const CuMatrix* mpErrorInput;///< inputs are NOT OWNED by component + + CuMatrix mOutput; ///< outputs are OWNED by component + CuMatrix mErrorOutput; ///< outputs are OWNED by component + + }; + + + /** + * Class UpdatableComponent is a box which has some + * parameters adjustable by learning + * + * you can set the learning rate, lock the params, + * and learn from each data observation + */ + class CuUpdatableComponent : public CuComponent + { + ////////////////////////////////////////////////////////////// + // Constructor & Destructor + public: + CuUpdatableComponent(size_t nInputs, size_t nOutputs, CuComponent *pPred); + virtual ~CuUpdatableComponent(); + + + ////////////////////////////////////////////////////////////// + // Interface specification (public) + public: + /// + virtual bool IsUpdatable() const + { return true; } + + /// get gradient and update the parameters in one step + virtual void Update() = 0; + + /// Sets the learning rate of gradient descent + void LearnRate(BaseFloat rate); + /// Gets the learning rate of gradient descent + BaseFloat LearnRate(); + + void Momentum(BaseFloat mmt); + BaseFloat Momentum(); + + void Weightcost(BaseFloat cost); + BaseFloat Weightcost(); + + void GradDivFrm(bool div); + bool GradDivFrm(); + + protected: + BaseFloat mLearningRate; + BaseFloat mMomentum; + BaseFloat mWeightcost; + bool mGradDivFrm; + + }; + + + + + ////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuComponent:: + inline + CuComponent:: + CuComponent(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : mNInputs(nInputs), mNOutputs(nOutputs), + mpInput(NULL), mpErrorInput(NULL), + mOutput(), mErrorOutput() + { + /* DOUBLE LINK the Components */ + if (pPred != NULL) { + SetInput(pPred->GetOutput()); + pPred->SetErrorInput(GetErrorOutput()); + } + } + + + inline + CuComponent:: + ~CuComponent() + { + ; + } + + inline void + CuComponent:: + Propagate() + { + //initialize output buffer + mOutput.Init(GetInput().Rows(),GetNOutputs()); + //do the dimensionality test + if(GetNInputs() != GetInput().Cols()) { + KALDI_ERR << "Non-matching INPUT dim!!! Network dim: " << GetNInputs() + << " Data dim: " << GetInput().Cols(); + } + //run transform + PropagateFnc(GetInput(),mOutput); + } + + + inline void + CuComponent:: + Backpropagate() + { + //re-initialize the output buffer + mErrorOutput.Init(GetErrorInput().Rows(),GetNInputs()); + + //do the dimensionality test + assert(GetErrorInput().Cols() == mNOutputs); + assert(mErrorOutput.Cols() == mNInputs); + assert(mErrorOutput.Rows() == GetErrorInput().Rows()); + + //transform + BackpropagateFnc(GetErrorInput(),mErrorOutput); + } + + + inline void + CuComponent:: + SetInput(const CuMatrix& rInput) + { + mpInput = &rInput; + } + + + inline void + CuComponent:: + SetErrorInput(const CuMatrix& rErrorInput) + { + mpErrorInput = &rErrorInput; + } + + + inline const CuMatrix& + CuComponent:: + GetInput() const + { + if (NULL == mpInput) Error("mpInput is NULL"); + return *mpInput; + } + + inline const CuMatrix& + CuComponent:: + GetOutput() const + { + return mOutput; + } + + inline const CuMatrix& + CuComponent:: + GetErrorInput() const + { + if (NULL == mpErrorInput) Error("mpErrorInput is NULL"); + return *mpErrorInput; + } + + inline const CuMatrix& + CuComponent:: + GetErrorOutput() const + { + return mErrorOutput; + } + + inline size_t + CuComponent:: + GetNInputs() const + { + return mNInputs; + } + + inline size_t + CuComponent:: + GetNOutputs() const + { + return mNOutputs; + } + + + + ////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // UpdatableComponent:: + + inline + CuUpdatableComponent:: + CuUpdatableComponent(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuComponent(nInputs, nOutputs, pPred), + mLearningRate(0.0), mMomentum(0), mWeightcost(0), mGradDivFrm(true) + { + ; + } + + + inline + CuUpdatableComponent:: + ~CuUpdatableComponent() + { + ; + } + + + inline void + CuUpdatableComponent:: + LearnRate(BaseFloat rate) + { + mLearningRate = rate; + } + + + inline BaseFloat + CuUpdatableComponent:: + LearnRate() + { + return mLearningRate; + } + + + inline void + CuUpdatableComponent:: + Momentum(BaseFloat mmt) + { + mMomentum = mmt; + } + + + inline BaseFloat + CuUpdatableComponent:: + Momentum() + { + return mMomentum; + } + + + inline void + CuUpdatableComponent:: + Weightcost(BaseFloat cost) + { + mWeightcost = cost; + } + + + inline BaseFloat + CuUpdatableComponent:: + Weightcost() + { + return mWeightcost; + } + + + inline void + CuUpdatableComponent:: + GradDivFrm(bool div) + { + mGradDivFrm = div; + } + + inline bool + CuUpdatableComponent:: + GradDivFrm() + { + return mGradDivFrm; + } + +} // namespace TNet + + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuDiscreteLinearity.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuDiscreteLinearity.cc.svn-base new file mode 100644 index 0000000..befde24 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuDiscreteLinearity.cc.svn-base @@ -0,0 +1,160 @@ + + +#include "cuDiscreteLinearity.h" +#include "cumath.h" + +namespace TNet +{ + + void + CuDiscreteLinearity:: + PropagateFnc(const CuMatrix& X, CuMatrix& Y) + { + //Y.SetConst(0.0); + + //precopy bias + Y.AddScaledRow(1.0,mBias,0.0); + + //mulitply with the matrices + int offset_in=0, offset_out=0; + for (int i=0; i::OffsetGemm('N','N', 1.0, X, mLinearity[i], 1.0, Y, + offset_in, 0, offset_out); + offset_in += mLinearity[i].Rows(); + offset_out += mLinearity[i].Cols(); + } + } + + + void + CuDiscreteLinearity:: + BackpropagateFnc(const CuMatrix& X, CuMatrix& Y) + { + //Y.SetConst(0.0); + + int offset_in=0, offset_out=0; + for(int i=0; i::OffsetGemm('N', 'T', 1.0, X, mLinearity[i], 0.0, Y, + offset_in, 0, offset_out); + offset_in += mLinearity[i].Cols(); + offset_out += mLinearity[i].Rows(); + } + } + + + void + CuDiscreteLinearity:: + Update() + { + //new implementation + BaseFloat N = 1; + if(mGradDivFrm) { + N = static_cast(GetInput().Rows()); + } + BaseFloat mmt_gain = static_cast(1.0/(1.0-mMomentum)); + N *= mmt_gain; //compensate higher gradient estimates due to momentum + + //get gradients of discrete linearities + int offset_in=0, offset_out=0; + for(int i=0; i::OffsetGemm('T','N',1.0, + GetInput(),GetErrorInput(), + mMomentum, mLinearityCorrection[i], + offset_in,offset_out,0); + offset_in += mLinearity[i].Rows(); + offset_out += mLinearity[i].Cols(); + } + for(int i=0; i> std::ws >> mNBlocks; + if(mNBlocks < 1) { + KALDI_ERR << "Bad number of blocks:" << mNBlocks; + } + + mLinearity.resize(mNBlocks); + mLinearityCorrection.resize(mNBlocks); + + int in_dim = 0, out_dim = 0; + for(int i=0; i> transpose; + mLinearity[i].CopyFrom(BfMatrix(transpose, TRANS)); + + if(transpose.Cols()*transpose.Rows() == 0) { + Error("Missing linearity matrix in network file"); + } + //allocate training buffers + mLinearityCorrection[i].Init(mLinearity[i].Rows(),mLinearity[i].Cols()); + mLinearityCorrection[i].SetConst(0.0); + + in_dim += transpose.Cols(); + out_dim += transpose.Rows(); + } + + //biases stored normally + BfVector bias; + rIn >> bias; + mBias.CopyFrom(bias); + if(bias.Dim() == 0) { + Error("Missing bias vector in network file"); + } + mBiasCorrection.Init(mBias.Dim()); + mBiasCorrection.SetConst(0.0); + + if(out_dim != GetNOutputs() || + in_dim != GetNInputs() || + mBias.Dim() != GetNOutputs() + ){ + std::ostringstream os; + os << "Wrong dimensionalities of matrix/vector in network file\n" + << "Inputs:" << GetNInputs() + << "Outputs:" << GetNOutputs() + << "\n" + << "linearityCols:" << in_dim + << "linearityRows:" << out_dim + << "biasDims:" << mBias.Dim() + << "\n"; + Error(os.str()); + } + } + + + void + CuDiscreteLinearity:: + WriteToStream(std::ostream& rOut) + { + rOut << mNBlocks << "\n"; + for(int i=0; i< mNBlocks; i++) { + //matrix is stored transposed as SNet does + BfMatrix tmp; + mLinearity[i].CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + } + //biases stored normally + BfVector vec; + mBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + } + + +} //namespace + diff --git a/src/CuTNetLib/.svn/text-base/cuDiscreteLinearity.h.svn-base b/src/CuTNetLib/.svn/text-base/cuDiscreteLinearity.h.svn-base new file mode 100644 index 0000000..06c8d74 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuDiscreteLinearity.h.svn-base @@ -0,0 +1,90 @@ +#ifndef _CUDISCRETE_LINEARITY_H_ +#define _CUDISCRETE_LINEARITY_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +#include "Matrix.h" +#include "Vector.h" + +#include + + +namespace TNet { + + class CuDiscreteLinearity : public CuUpdatableComponent + { + public: + + CuDiscreteLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuDiscreteLinearity(); + + ComponentType GetType() const; + const char* GetName() const; + + void PropagateFnc(const CuMatrix& X, CuMatrix& Y); + void BackpropagateFnc(const CuMatrix& X, CuMatrix& Y); + + void Update(); + + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + std::vector > mLinearity; ///< Matrix with neuron weights + CuVector mBias; ///< Vector with biases + + std::vector > mLinearityCorrection; ///< Matrix for linearity updates + CuVector mBiasCorrection; ///< Vector for bias updates + + size_t mNBlocks; + + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuDiscreteLinearity:: + inline + CuDiscreteLinearity:: + CuDiscreteLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred), + //mLinearity(nInputs,nOutputs), mBias(nOutputs), + //mLinearityCorrection(nInputs,nOutputs), mBiasCorrection(nOutputs) + mNBlocks(0) + { + //mLinearityCorrection.SetConst(0.0); + //mBiasCorrection.SetConst(0.0); + } + + + inline + CuDiscreteLinearity:: + ~CuDiscreteLinearity() + { } + + inline CuComponent::ComponentType + CuDiscreteLinearity:: + GetType() const + { + return CuComponent::DISCRETE_LINEARITY; + } + + inline const char* + CuDiscreteLinearity:: + GetName() const + { + return ""; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuNetwork.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuNetwork.cc.svn-base new file mode 100644 index 0000000..e245699 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuNetwork.cc.svn-base @@ -0,0 +1,380 @@ + +#include +//#include +#include +#include +#include + +#include "cuNetwork.h" + +#include "cuDiscreteLinearity.h" +#include "cuSharedLinearity.h" +#include "cuSparseLinearity.h" +#include "cuRbm.h" +#include "cuRbmSparse.h" +#include "cuRecurrent.h" +#include "cuBlockArray.h" + +namespace TNet { + + + + + void + CuNetwork:: + ReadNetwork(const char* pSrc) + { + std::ifstream in(pSrc); + if(!in.good()) { + Error(std::string("Error, cannot read model: ")+pSrc); + } + ReadNetwork(in); + in.close(); + } + + + + void + CuNetwork:: + WriteNetwork(const char* pDst) + { + std::ofstream out(pDst); + if(!out.good()) { + Error(std::string("Error, cannot write model: ")+pDst); + } + WriteNetwork(out); + out.close(); + } + + + + void + CuNetwork:: + ReadNetwork(std::istream& rIn) + { + //get the network elements from a factory + CuComponent *pComp; + while(NULL != (pComp = ComponentFactory(rIn))) { + mNetComponents.push_back(pComp); + } + } + + + + void + CuNetwork:: + WriteNetwork(std::ostream& rOut) + { + //dump all the componetns + LayeredType::iterator it; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + ComponentDumper(rOut, **it); + } + } + + + void + CuNetwork:: + SetLearnRate(BaseFloat learnRate, const char* pLearnRateFactors) + { + //parse the learn rate factors: "0.1:0.5:0.6:1.0" to std::list + std::list lr_factors; + if(NULL != pLearnRateFactors) { + //replace ':' by ' ' + std::string str(pLearnRateFactors); + size_t pos = 0; + while((pos = str.find(':',pos)) != std::string::npos) str[pos] = ' '; + while((pos = str.find(',',pos)) != std::string::npos) str[pos] = ' '; + + //parse to std::list + std::istringstream is(str); + is >> std::skipws; + BaseFloat f; + while(!is.eof()) { + if(!(is >> f).fail()) { lr_factors.push_back(f); } + else break; + } + } + + //initialize rate factors iterator + BaseFloat scale = 1.0f; + + //store global learning rate + mGlobLearnRate = learnRate; + mpLearnRateFactors = pLearnRateFactors; + + //give scaled learning rate to components + LayeredType::iterator it; + bool stopper_given = false; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + if((*it)->IsUpdatable()) { + //get next scale factor + if(NULL != pLearnRateFactors) { + if(!(lr_factors.size() > 0)) { + Error("Too few learninig rate scale factors"); + } + scale = lr_factors.front(); + lr_factors.pop_front(); + } + //set scaled learning rate to the component + dynamic_cast(*it)->LearnRate(learnRate*scale); + //set the stopper component for backpropagation + if(!stopper_given && (learnRate*scale > 0.0)) { + mpPropagErrorStopper = *it; stopper_given = true; + } + } + } + if(lr_factors.size() > 0) { + Error("Too much learninig rate scale factors"); + } + } + + + BaseFloat + CuNetwork:: + GetLearnRate() + { + return mGlobLearnRate; + } + + + void + CuNetwork:: + PrintLearnRate() + { + assert(mNetComponents.size() > 0); + std::cout << "Learning rate: global " << mGlobLearnRate; + std::cout << " components' "; + for(size_t i=0; iIsUpdatable()) { + std::cout << " " << dynamic_cast(mNetComponents[i])->LearnRate(); + } + } + std::cout << "\n" << std::flush; + } + + + + void + CuNetwork:: + SetMomentum(BaseFloat momentum) + { + LayeredType::iterator it; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + if((*it)->IsUpdatable()) { + dynamic_cast(*it)->Momentum(momentum); + } + } + } + + void + CuNetwork:: + SetWeightcost(BaseFloat weightcost) + { + LayeredType::iterator it; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + if((*it)->IsUpdatable()) { + dynamic_cast(*it)->Weightcost(weightcost); + } + } + } + + void + CuNetwork:: + SetL1(BaseFloat l1) + { + LayeredType::iterator it; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + if((*it)->GetType() == CuComponent::SPARSE_LINEARITY) { + dynamic_cast(*it)->L1(l1); + } + } + } + + void + CuNetwork:: + SetGradDivFrm(bool div) + { + LayeredType::iterator it; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + if((*it)->IsUpdatable()) { + dynamic_cast(*it)->GradDivFrm(div); + } + } + } + + + CuComponent* + CuNetwork:: + ComponentFactory(std::istream& rIn) + { + rIn >> std::ws; + if(rIn.eof()) return NULL; + + CuComponent* pRet=NULL; + CuComponent* pPred=NULL; + + std::string componentTag; + size_t nInputs, nOutputs; + + rIn >> std::ws; + rIn >> componentTag; + if(componentTag == "") return NULL; //nothing left in the file + + //make it lowercase + std::transform(componentTag.begin(), componentTag.end(), + componentTag.begin(), tolower); + + if(componentTag[0] != '<' || componentTag[componentTag.size()-1] != '>') { + Error(std::string("Invalid component tag:")+componentTag); + } + + //the 'endblock' tag terminates the network + if(componentTag == "") return NULL; + + rIn >> std::ws; + rIn >> nOutputs; + rIn >> std::ws; + rIn >> nInputs; + assert(nInputs > 0 && nOutputs > 0); + + //make coupling with predecessor + if(mNetComponents.size() != 0) { + pPred = mNetComponents.back(); + } + + //array with list of component tags + static const std::string TAGS[] = { + "", + "", + "", + "", + "", + "", + "", + + "", + "", + + "", + "", + "", + "", + "", + "", + "", + + "", + }; + + static const int n_tags = sizeof(TAGS) / sizeof(TAGS[0]); + int i; + for(i=0; iReadFromStream(rIn); + + //return + return pRet; + } + + + void + CuNetwork:: + ComponentDumper(std::ostream& rOut, CuComponent& rComp) + { + //use tags of all the components; or the identification codes + //array with list of component tags + static const CuComponent::ComponentType TYPES[] = { + CuComponent::BIASED_LINEARITY, + CuComponent::DISCRETE_LINEARITY, + CuComponent::SHARED_LINEARITY, + CuComponent::SPARSE_LINEARITY, + CuComponent::RBM, + CuComponent::RBM_SPARSE, + CuComponent::RECURRENT, + + CuComponent::SIGMOID, + CuComponent::SOFTMAX, + + CuComponent::EXPAND, + CuComponent::COPY, + CuComponent::TRANSPOSE, + CuComponent::BLOCK_LINEARITY, + CuComponent::BIAS, + CuComponent::WINDOW, + CuComponent::LOG, + + CuComponent::BLOCK_ARRAY, + }; + static const std::string TAGS[] = { + "", + "", + "", + "", + "", + "", + "", + + "", + "", + + "", + "", + "", + "", + "", + "", + "", + + "", + }; + static const int MAX = sizeof TYPES / sizeof TYPES[0]; + + int i; + for(i=0; i + + +namespace TNet { + + class CuNetwork + { + ////////////////////////////////////// + // Typedefs + typedef std::vector LayeredType; + + ////////////////////////////////////// + // Disable copy construction, assignment and default constructor + private: + CuNetwork(CuNetwork&); + CuNetwork& operator=(CuNetwork&); + + public: + CuNetwork() { } + CuNetwork(std::istream& rIn); + ~CuNetwork(); + + void AddLayer(CuComponent* layer); + + int Layers() + { return mNetComponents.size(); } + + CuComponent& Layer(int i) + { return *mNetComponents[i]; } + + /// forward the data to the output + void Propagate(const CuMatrix& in, CuMatrix& out); + + /// backpropagate the error while updating weights + void Backpropagate(const CuMatrix& globerr); + + void ReadNetwork(const char* pSrc); ///< read the network from file + void WriteNetwork(const char* pDst); ///< write network to file + + void ReadNetwork(std::istream& rIn); ///< read the network from stream + void WriteNetwork(std::ostream& rOut); ///< write network to stream + + size_t GetNInputs() const; ///< Dimensionality of the input features + size_t GetNOutputs() const; ///< Dimensionality of the desired vectors + + /// set the learning rate + void SetLearnRate(BaseFloat learnRate, const char* pLearnRateFactors = NULL); + BaseFloat GetLearnRate(); ///< get the learning rate value + void PrintLearnRate(); ///< log the learning rate values + + void SetMomentum(BaseFloat momentum); + void SetWeightcost(BaseFloat weightcost); + void SetL1(BaseFloat l1); + + void SetGradDivFrm(bool div); + + + private: + /// Creates a component by reading from stream + CuComponent* ComponentFactory(std::istream& In); + /// Dumps component into a stream + void ComponentDumper(std::ostream& rOut, CuComponent& rComp); + + + + private: + LayeredType mNetComponents; ///< container with the network layers + CuComponent* mpPropagErrorStopper; + BaseFloat mGlobLearnRate; ///< The global (unscaled) learn rate of the network + const char* mpLearnRateFactors; ///< The global (unscaled) learn rate of the network + + + //friend class NetworkGenerator; //<< For generating networks... + + }; + + + ////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuNetwork:: + inline + CuNetwork:: + CuNetwork(std::istream& rSource) + : mpPropagErrorStopper(NULL), mGlobLearnRate(0.0), mpLearnRateFactors(NULL) + { + ReadNetwork(rSource); + } + + + inline + CuNetwork:: + ~CuNetwork() + { + //delete all the components + LayeredType::iterator it; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + delete *it; + *it = NULL; + } + mNetComponents.resize(0); + } + + + inline void + CuNetwork:: + AddLayer(CuComponent* layer) + { + if(mNetComponents.size() > 0) { + if(GetNOutputs() != layer->GetNInputs()) { + Error("Nonmatching dims"); + } + layer->SetInput(mNetComponents.back()->GetOutput()); + mNetComponents.back()->SetErrorInput(layer->GetErrorOutput()); + } + mNetComponents.push_back(layer); + } + + + inline void + CuNetwork:: + Propagate(const CuMatrix& in, CuMatrix& out) + { + //empty network => copy input + if(mNetComponents.size() == 0) { + out.CopyFrom(in); + return; + } + + //check dims + if(in.Cols() != GetNInputs()) { + std::ostringstream os; + os << "Nonmatching dims" + << " data dim is: " << in.Cols() + << " network needs: " << GetNInputs(); + Error(os.str()); + } + mNetComponents.front()->SetInput(in); + + //propagate + LayeredType::iterator it; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + (*it)->Propagate(); + } + + //copy the output + out.CopyFrom(mNetComponents.back()->GetOutput()); + } + + + + + inline void + CuNetwork:: + Backpropagate(const CuMatrix& globerr) + { + mNetComponents.back()->SetErrorInput(globerr); + + // back-propagation + LayeredType::reverse_iterator it; + for(it=mNetComponents.rbegin(); it!=mNetComponents.rend(); ++it) { + //stopper component does not propagate error (no updatable predecessors) + if(*it != mpPropagErrorStopper) { + //compute errors for preceding network components + (*it)->Backpropagate(); + } + //update weights if updatable component + if((*it)->IsUpdatable()) { + CuUpdatableComponent& rComp = dynamic_cast(**it); + if(rComp.LearnRate() > 0.0f) { + rComp.Update(); + } + } + //stop backprop if no updatable components precede current component + if(mpPropagErrorStopper == *it) break; + } + } + + + inline size_t + CuNetwork:: + GetNInputs() const + { + if(!mNetComponents.size() > 0) return 0; + return mNetComponents.front()->GetNInputs(); + } + + + inline size_t + CuNetwork:: + GetNOutputs() const + { + if(!mNetComponents.size() > 0) return 0; + return mNetComponents.back()->GetNOutputs(); + } + + + + + +} //namespace + +#endif + + diff --git a/src/CuTNetLib/.svn/text-base/cuObjectiveFunction.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuObjectiveFunction.cc.svn-base new file mode 100644 index 0000000..e2b0a1d --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuObjectiveFunction.cc.svn-base @@ -0,0 +1,87 @@ + +#include "cuObjectiveFunction.h" + +#include "Error.h" +#include "cumath.h" + + +namespace TNet +{ + + + + CuObjectiveFunction* + CuObjectiveFunction:: + Factory(ObjFunType type) { + CuObjectiveFunction* ret = NULL; + switch(type) { + case MEAN_SQUARE_ERROR: ret = new CuMeanSquareError; break; + case CROSS_ENTROPY: ret = new CuCrossEntropy; break; + default: Error("Unknown ObjFun type"); + } + return ret; + } + + + void + CuMeanSquareError:: + Evaluate(const CuMatrix& rNetOutput, const CuMatrix& rDesired, CuMatrix& rNetError) + { + //get the global error + rNetError.CopyFrom(rNetOutput); + rNetError.AddScaled(-1.0,rDesired,1.0); + + //calculate the MSE + mAuxMat.CopyFrom(rNetError); + mAuxMat.MulElem(mAuxMat); + + mAuxVec.Init(mAuxMat.Cols()); + mAuxVec.AddColSum(1.0,mAuxMat,0.0); + mAuxVec.CopyTo(mAuxVecHost); + + mError += mAuxVecHost.Sum(); + + //count the frames + mFrames += rNetError.Rows(); + } + + void + CuCrossEntropy:: + Evaluate(const CuMatrix& rNetOutput, const CuMatrix& rDesired, CuMatrix& rNetError) + { + if(rDesired.Cols() != rNetOutput.Cols()) { + std::ostringstream os; + os << "Non-matching dimensions of network output with training targets!!!" + << " Netoutput:" << rNetOutput.Cols() + << " Targets:" << rDesired.Cols(); + Error(os.str()); + } + + //get the global error + //dXent/dSoftmax_in = y-d + rNetError.CopyFrom(rNetOutput); + rNetError.AddScaled(-1.0,rDesired,1.0); + + //check classification + mClassifyVec.Init(rNetOutput.Rows()); + CuMath::CheckClass(rNetOutput,rDesired,mClassifyVec); + mClassifyVec.CopyTo(mClassifyVecHost); + mCorrect += mClassifyVecHost.Sum(); + + //calculate Xent + mAuxMat.CopyFrom(rNetOutput); + mAuxMat.LogElem(); + mAuxMat.MulElem(rDesired); + + mAuxVec.Init(mAuxMat.Cols()); + mAuxVec.AddColSum(-1.0,mAuxMat,0.0); + mAuxVec.CopyTo(mAuxVecHost); + + mError += mAuxVecHost.Sum(); + + //count the frames + mFrames += rNetError.Rows(); + } + + +} // namespace TNet diff --git a/src/CuTNetLib/.svn/text-base/cuObjectiveFunction.h.svn-base b/src/CuTNetLib/.svn/text-base/cuObjectiveFunction.h.svn-base new file mode 100644 index 0000000..6b425e8 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuObjectiveFunction.h.svn-base @@ -0,0 +1,166 @@ +#ifndef _CUOBJ_FUN_I_ +#define _CUOBJ_FUN_I_ + +#include +#include +#include +#include + +#include "Vector.h" +#include "cuvector.h" +#include "cumatrix.h" + +namespace TNet +{ + + + /** + * General interface for objective functions + */ + class CuObjectiveFunction + { + public: + /// Enum with objective function types + typedef enum { + OBJ_FUN_I = 0x0300, + MEAN_SQUARE_ERROR, + CROSS_ENTROPY, + } ObjFunType; + + /// Factory for creating objective function instances + static CuObjectiveFunction* Factory(ObjFunType type); + + ////////////////////////////////////////////////////////////// + // Interface specification + public: + CuObjectiveFunction() + { } + + virtual ~CuObjectiveFunction() + { } + + virtual ObjFunType GetTypeId() = 0; + virtual const char* GetTypeLabel() = 0; + + /// evaluates the data, calculate global error + virtual void Evaluate(const CuMatrix& rNetOutput, const CuMatrix& rDesired, CuMatrix& rNetError) = 0; + + ///get the average per frame error + virtual double GetError() = 0; + ///the number of processed frames + virtual size_t GetFrames() = 0; + ///report the error to std::cout + virtual std::string Report() = 0; + }; + + + + + /** + * Means square error, useful for autoencoders, RBMs et al. + */ + class CuMeanSquareError : public CuObjectiveFunction + { + public: + CuMeanSquareError() + : mError(0), mFrames(0) + { } + + virtual ~CuMeanSquareError() + { } + + ObjFunType GetTypeId() + { return CuObjectiveFunction::MEAN_SQUARE_ERROR; } + + const char* GetTypeLabel() + { return ""; } + + void Evaluate(const CuMatrix& rNetOutput, const CuMatrix& rDesired, CuMatrix& rNetError); + + double GetError() + { return mError; } + + size_t GetFrames() + { return mFrames; } + + std::string Report() + { + std::ostringstream ss; + ss << "Mse:" << mError << " frames:" << mFrames + << " err/frm:" << mError/mFrames << "\n"; + return ss.str(); + } + + private: + double mError; + size_t mFrames; + + CuMatrix mAuxMat; + CuVector mAuxVec; + Vector mAuxVecHost; + + }; + + + /** + * Cross entropy, it assumes desired vectors as output values + */ + class CuCrossEntropy : public CuObjectiveFunction + { + public: + CuCrossEntropy() + : mError(0), mFrames(0), mCorrect(0) + { } + + ~CuCrossEntropy() + { } + + ObjFunType GetTypeId() + { return CuObjectiveFunction::CROSS_ENTROPY; } + + const char* GetTypeLabel() + { return ""; } + + void Evaluate(const CuMatrix& rNetOutput, const CuMatrix& rDesired, CuMatrix& rNetError); + + double GetError() + { return mError; } + + size_t GetFrames() + { return mFrames; } + + std::string Report() + { + std::ostringstream ss; + //for compatibility with SNet + //ss << " correct: >> " << 100.0*mCorrect/mFrames << "% <<\n"; + + //current new format... + ss << "Xent:" << mError << " frames:" << mFrames + << " err/frm:" << mError/mFrames + << " correct[" << 100.0*mCorrect/mFrames << "%]" + << "\n"; + return ss.str(); + } + + private: + double mError; + size_t mFrames; + size_t mCorrect; + + CuMatrix mAuxMat; + CuVector mAuxVec; + Vector mAuxVecHost; + + CuVector mClassifyVec; + Vector mClassifyVecHost; + }; + + + + + +} //namespace TNet + + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuRbm.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuRbm.cc.svn-base new file mode 100644 index 0000000..3d0699d --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuRbm.cc.svn-base @@ -0,0 +1,244 @@ + +#include +#include + +#include "cuRbm.h" + +#include "cumath.h" + + +namespace TNet +{ + + void + CuRbm:: + PropagateFnc(const CuMatrix& X, CuMatrix& Y) + { + Y.SetConst(0.0); + Y.AddScaledRow(1.0,mHidBias,0.0); + Y.Gemm('N','N', 1.0, X, mVisHid, 1.0); + if(mHidType == BERNOULLI) { + CuMath::Sigmoid(Y,Y); + } + } + + + void + CuRbm:: + BackpropagateFnc(const CuMatrix& X, CuMatrix& Y) + { + if(mHidType == BERNOULLI) { + mBackpropErrBuf.Init(X.Rows(),X.Cols()); + CuMath::DiffSigmoid(mBackpropErrBuf,X,GetOutput()); + } else { + mBackpropErrBuf.CopyFrom(X); + } + + Y.SetConst(0.0); + Y.Gemm('N', 'T', 1.0, mBackpropErrBuf, mVisHid, 0.0); + } + + + void + CuRbm:: + Update() + { + //THIS IS DONE TWICE BECAUSE OF THE BACKPROP STOPPER!!! + if(mHidType == BERNOULLI) { + mBackpropErrBuf.Init(GetErrorInput().Rows(),GetErrorInput().Cols()); + CuMath::DiffSigmoid(mBackpropErrBuf,GetErrorInput(),GetOutput()); + } else { + mBackpropErrBuf.CopyFrom(GetErrorInput()); + } + +/* + std::cout << " " << GetInput().Rows() + << " " << GetInput().Cols() + << " " << mBackpropErrBuf.Rows() + << " " << mBackpropErrBuf.Cols() + << " " << mVisHidCorrection.Rows() + << " " << mVisHidCorrection.Cols() + ; +*/ + +#if 0 + //former implementation + BaseFloat N = static_cast(GetInput().Rows()); + + mVisHidCorrection.Gemm('T','N',-mLearningRate/N,GetInput(),mBackpropErrBuf,mMomentum); + mHidBiasCorrection.AddColSum(-mLearningRate/N,mBackpropErrBuf,mMomentum); + + //regularization weight decay + mVisHidCorrection.AddScaled(-mLearningRate*mWeightcost,mVisHid,1.0); + + mVisHid.AddScaled(1.0,mVisHidCorrection,1.0); + mHidBias.AddScaled(1.0,mHidBiasCorrection,1.0); +#endif + +#if 1 + //new implementation + BaseFloat N = 1; + if(mGradDivFrm) { + N = static_cast(GetInput().Rows()); + } + BaseFloat mmt_gain = static_cast(1.0/(1.0-mMomentum)); + N *= mmt_gain; + + mVisHidCorrection.Gemm('T','N',1.0,GetInput(),mBackpropErrBuf,mMomentum); + mHidBiasCorrection.AddColSum(1.0,mBackpropErrBuf,mMomentum); + + mVisHid.AddScaled(-mLearningRate/N,mVisHidCorrection,1.0); + mHidBias.AddScaled(-mLearningRate/N,mHidBiasCorrection,1.0); + + //regularization weight decay (from actual weights only) + mVisHid.AddScaled(-mLearningRate*mWeightcost,mVisHid,1.0); +#endif + + } + + + + void + CuRbm:: + Propagate(const CuMatrix& visProbs, CuMatrix& hidProbs) + { + if(visProbs.Cols() != GetNInputs()) { + std::ostringstream os; + os << " Nonmatching input dim, needs:" << GetNInputs() + << " got:" << visProbs.Cols() << "\n"; + Error(os.str()); + } + + hidProbs.Init(visProbs.Rows(),GetNOutputs()); + + PropagateFnc(visProbs, hidProbs); + } + + void + CuRbm:: + Reconstruct(const CuMatrix& hidState, CuMatrix& visProbs) + { + visProbs.Init(hidState.Rows(),mNInputs); + visProbs.SetConst(0.0); + visProbs.AddScaledRow(1.0,mVisBias,0.0); + visProbs.Gemm('N','T', 1.0, hidState, mVisHid, 1.0); + if(mVisType == BERNOULLI) { + CuMath::Sigmoid(visProbs,visProbs); + } + } + + + void + CuRbm:: + RbmUpdate(const CuMatrix& pos_vis, const CuMatrix& pos_hid, const CuMatrix& neg_vis, const CuMatrix& neg_hid) + { + assert(pos_vis.Rows() == pos_hid.Rows() && + pos_vis.Rows() == neg_vis.Rows() && + pos_vis.Rows() == neg_hid.Rows() && + pos_vis.Cols() == neg_vis.Cols() && + pos_hid.Cols() == neg_hid.Cols() && + pos_vis.Cols() == mNInputs && + pos_hid.Cols() == mNOutputs); + + // UPDATE vishid matrix + // + // vishidinc = momentum*vishidinc + ... + // epsilonw*( (posprods-negprods)/numcases - weightcost*vishid); + // + // vishidinc[t] = -(epsilonw/numcases)*negprods + momentum*vishidinc[t-1] + // +(epsilonw/numcases)*posprods + // -(epsilonw*weightcost)*vishid[t-1] + // + BaseFloat N = static_cast(pos_vis.Rows()); + mVisHidCorrection.Gemm('T','N',-mLearningRate/N,neg_vis,neg_hid,mMomentum); + mVisHidCorrection.Gemm('T','N',+mLearningRate/N,pos_vis,pos_hid,1.0); + mVisHidCorrection.AddScaled(-mLearningRate*mWeightcost,mVisHid,1.0); + mVisHid.AddScaled(1.0,mVisHidCorrection,1.0); + + // UPDATE visbias vector + // + // visbiasinc = momentum*visbiasinc + (epsilonvb/numcases)*(posvisact-negvisact); + // + mVisBiasCorrection.AddColSum(-mLearningRate/N,neg_vis,mMomentum); + mVisBiasCorrection.AddColSum(+mLearningRate/N,pos_vis,1.0); + mVisBias.AddScaled(1.0,mVisBiasCorrection,1.0); + + // UPDATE hidbias vector + // + // hidbiasinc = momentum*hidbiasinc + (epsilonhb/numcases)*(poshidact-neghidact); + // + mHidBiasCorrection.AddColSum(-mLearningRate/N,neg_hid,mMomentum); + mHidBiasCorrection.AddColSum(+mLearningRate/N,pos_hid,1.0); + mHidBias.AddScaled(1.0/*0.0*/,mHidBiasCorrection,1.0); + + } + + + void + CuRbm:: + ReadFromStream(std::istream& rIn) + { + //type of the units + std::string str; + + rIn >> std::ws >> str; + if(0 == str.compare("bern")) { + mVisType = BERNOULLI; + } else if(0 == str.compare("gauss")) { + mVisType = GAUSSIAN; + } else Error(std::string("Invalid unit type: ")+str); + + rIn >> std::ws >> str; + if(0 == str.compare("bern")) { + mHidType = BERNOULLI; + } else if(0 == str.compare("gauss")) { + mHidType = GAUSSIAN; + } else Error(std::string("Invalid unit type: ")+str); + + + //matrix is stored transposed as SNet does + BfMatrix transpose; + rIn >> transpose; + mVisHid.CopyFrom(BfMatrix(transpose, TRANS)); + //biases stored normally + BfVector bias; + rIn >> bias; + mVisBias.CopyFrom(bias); + rIn >> bias; + mHidBias.CopyFrom(bias); + } + + + void + CuRbm:: + WriteToStream(std::ostream& rOut) + { + //store unit type info + if(mVisType == BERNOULLI) { + rOut << " bern "; + } else { + rOut << " gauss "; + } + if(mHidType == BERNOULLI) { + rOut << " bern\n"; + } else { + rOut << " gauss\n"; + } + + //matrix is stored transposed as SNet does + BfMatrix tmp; + mVisHid.CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + //biases stored normally + BfVector vec; + mVisBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + mHidBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + } + + +} //namespace diff --git a/src/CuTNetLib/.svn/text-base/cuRbm.h.svn-base b/src/CuTNetLib/.svn/text-base/cuRbm.h.svn-base new file mode 100644 index 0000000..c1e984b --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuRbm.h.svn-base @@ -0,0 +1,146 @@ +#ifndef _CU_RBM_H_ +#define _CU_RBM_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + + class CuRbmBase : public CuUpdatableComponent + { + public: + typedef enum { + BERNOULLI, + GAUSSIAN + } RbmUnitType; + + CuRbmBase(size_t nInputs, size_t nOutputs, CuComponent *pPred) : + CuUpdatableComponent(nInputs, nOutputs, pPred) + { } + + + virtual void Propagate( + const CuMatrix& visProbs, + CuMatrix& hidProbs + ) = 0; + virtual void Reconstruct( + const CuMatrix& hidState, + CuMatrix& visProbs + ) = 0; + virtual void RbmUpdate( + const CuMatrix& pos_vis, + const CuMatrix& pos_hid, + const CuMatrix& neg_vis, + const CuMatrix& neg_hid + ) = 0; + + virtual RbmUnitType VisType() = 0; + virtual RbmUnitType HidType() = 0; + }; + + + class CuRbm : public CuRbmBase + { + public: + + CuRbm(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuRbm(); + + ComponentType GetType() const; + const char* GetName() const; + + //CuUpdatableComponent API + void PropagateFnc(const CuMatrix& X, CuMatrix& Y); + void BackpropagateFnc(const CuMatrix& X, CuMatrix& Y); + + void Update(); + + //RBM training API + void Propagate(const CuMatrix& visProbs, CuMatrix& hidProbs); + void Reconstruct(const CuMatrix& hidState, CuMatrix& visProbs); + void RbmUpdate(const CuMatrix& pos_vis, const CuMatrix& pos_hid, const CuMatrix& neg_vis, const CuMatrix& neg_hid); + + RbmUnitType VisType() + { return mVisType; } + + RbmUnitType HidType() + { return mHidType; } + + //static void BinarizeProbs(const CuMatrix& probs, CuMatrix& states); + + //I/O + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + CuMatrix mVisHid; ///< Matrix with neuron weights + CuVector mVisBias; ///< Vector with biases + CuVector mHidBias; ///< Vector with biases + + CuMatrix mVisHidCorrection; ///< Matrix for linearity updates + CuVector mVisBiasCorrection; ///< Vector for bias updates + CuVector mHidBiasCorrection; ///< Vector for bias updates + + CuMatrix mBackpropErrBuf; + + RbmUnitType mVisType; + RbmUnitType mHidType; + + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuRbm:: + inline + CuRbm:: + CuRbm(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuRbmBase(nInputs, nOutputs, pPred), + mVisHid(nInputs,nOutputs), + mVisBias(nInputs), mHidBias(nOutputs), + mVisHidCorrection(nInputs,nOutputs), + mVisBiasCorrection(nInputs), mHidBiasCorrection(nOutputs), + mBackpropErrBuf(), + mVisType(BERNOULLI), + mHidType(BERNOULLI) + { + mVisHidCorrection.SetConst(0.0); + mVisBiasCorrection.SetConst(0.0); + mHidBiasCorrection.SetConst(0.0); + } + + + inline + CuRbm:: + ~CuRbm() + { } + + inline CuComponent::ComponentType + CuRbm:: + GetType() const + { + return CuComponent::RBM; + } + + inline const char* + CuRbm:: + GetName() const + { + return ""; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuRbmSparse.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuRbmSparse.cc.svn-base new file mode 100644 index 0000000..e0b7352 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuRbmSparse.cc.svn-base @@ -0,0 +1,269 @@ + +#include +#include + +#include "cuRbmSparse.h" + +#include "cumath.h" + + +namespace TNet +{ + + void + CuRbmSparse:: + PropagateFnc(const CuMatrix& X, CuMatrix& Y) + { + Y.SetConst(0.0); + Y.AddScaledRow(1.0,mHidBias,0.0); + Y.Gemm('N','N', 1.0, X, mVisHid, 1.0); + if(mHidType == BERNOULLI) { + CuMath::Sigmoid(Y,Y); + } + } + + + void + CuRbmSparse:: + BackpropagateFnc(const CuMatrix& X, CuMatrix& Y) + { + if(mHidType == BERNOULLI) { + mBackpropErrBuf.Init(X.Rows(),X.Cols()); + CuMath::DiffSigmoid(mBackpropErrBuf,X,GetOutput()); + } else { + mBackpropErrBuf.CopyFrom(X); + } + + Y.SetConst(0.0); + Y.Gemm('N', 'T', 1.0, mBackpropErrBuf, mVisHid, 0.0); + } + + + void + CuRbmSparse:: + Update() + { + //THIS IS DONE TWICE BECAUSE OF THE BACKPROP STOPPER!!! + if(mHidType == BERNOULLI) { + mBackpropErrBuf.Init(GetErrorInput().Rows(),GetErrorInput().Cols()); + CuMath::DiffSigmoid(mBackpropErrBuf,GetErrorInput(),GetOutput()); + } else { + mBackpropErrBuf.CopyFrom(GetErrorInput()); + } + +/* + std::cout << " " << GetInput().Rows() + << " " << GetInput().Cols() + << " " << mBackpropErrBuf.Rows() + << " " << mBackpropErrBuf.Cols() + << " " << mVisHidCorrection.Rows() + << " " << mVisHidCorrection.Cols() + ; +*/ + +#if 0 + //former implementation + BaseFloat N = static_cast(GetInput().Rows()); + + mVisHidCorrection.Gemm('T','N',-mLearningRate/N,GetInput(),mBackpropErrBuf,mMomentum); + mHidBiasCorrection.AddColSum(-mLearningRate/N,mBackpropErrBuf,mMomentum); + + //regularization weight decay + mVisHidCorrection.AddScaled(-mLearningRate*mWeightcost,mVisHid,1.0); + + mVisHid.AddScaled(1.0,mVisHidCorrection,1.0); + mHidBias.AddScaled(1.0,mHidBiasCorrection,1.0); +#endif + +#if 1 + //new implementation + BaseFloat N = 1; + if(mGradDivFrm) { + N = static_cast(GetInput().Rows()); + } + BaseFloat mmt_gain = static_cast(1.0/(1.0-mMomentum)); + N *= mmt_gain; + + mVisHidCorrection.Gemm('T','N',1.0,GetInput(),mBackpropErrBuf,mMomentum); + mHidBiasCorrection.AddColSum(1.0,mBackpropErrBuf,mMomentum); + + mVisHid.AddScaled(-mLearningRate/N,mVisHidCorrection,1.0); + mHidBias.AddScaled(-mLearningRate/N,mHidBiasCorrection,1.0); + + //regularization weight decay (from actual weights only) + mVisHid.AddScaled(-mLearningRate*mWeightcost,mVisHid,1.0); +#endif + + } + + + + void + CuRbmSparse:: + Propagate(const CuMatrix& visProbs, CuMatrix& hidProbs) + { + if(visProbs.Cols() != GetNInputs()) { + std::ostringstream os; + os << " Nonmatching input dim, needs:" << GetNInputs() + << " got:" << visProbs.Cols() << "\n"; + Error(os.str()); + } + + hidProbs.Init(visProbs.Rows(),GetNOutputs()); + + PropagateFnc(visProbs, hidProbs); + } + + void + CuRbmSparse:: + Reconstruct(const CuMatrix& hidState, CuMatrix& visProbs) + { + visProbs.Init(hidState.Rows(),mNInputs); + visProbs.SetConst(0.0); + visProbs.AddScaledRow(1.0,mVisBias,0.0); + visProbs.Gemm('N','T', 1.0, hidState, mVisHid, 1.0); + if(mVisType == BERNOULLI) { + CuMath::Sigmoid(visProbs,visProbs); + } + } + + + void + CuRbmSparse:: + RbmUpdate(const CuMatrix& pos_vis, const CuMatrix& pos_hid, const CuMatrix& neg_vis, const CuMatrix& neg_hid) + { + assert(pos_vis.Rows() == pos_hid.Rows() && + pos_vis.Rows() == neg_vis.Rows() && + pos_vis.Rows() == neg_hid.Rows() && + pos_vis.Cols() == neg_vis.Cols() && + pos_hid.Cols() == neg_hid.Cols() && + pos_vis.Cols() == mNInputs && + pos_hid.Cols() == mNOutputs); + + //:SPARSITY: + if(mHidType==BERNOULLI) { + //get expected node activity from current batch + mSparsityQCurrent.AddColSum(1.0/pos_hid.Rows(),pos_hid,0.0); + //get smoothed expected node activity + mSparsityQ.AddScaled(1.0-mLambda,mSparsityQCurrent,mLambda); + //subtract the prior: (q-p) + mSparsityQCurrent.SetConst(-mSparsityPrior); + mSparsityQCurrent.AddScaled(1.0,mSparsityQ,1.0); + //get mean pos_vis + mVisMean.AddColSum(1.0/pos_vis.Rows(),pos_vis,0.0); + } + + // UPDATE vishid matrix + // + // vishidinc = momentum*vishidinc + ... + // epsilonw*( (posprods-negprods)/numcases - weightcost*vishid) + // -sparsitycost*mean_posvis'*(q-p); + // + // vishidinc[t] = -(epsilonw/numcases)*negprods + momentum*vishidinc[t-1] + // +(epsilonw/numcases)*posprods + // -(epsilonw*weightcost)*vishid[t-1] + // + BaseFloat N = static_cast(pos_vis.Rows()); + mVisHidCorrection.Gemm('T','N',-mLearningRate/N,neg_vis,neg_hid,mMomentum); + mVisHidCorrection.Gemm('T','N',+mLearningRate/N,pos_vis,pos_hid,1.0); + mVisHidCorrection.AddScaled(-mLearningRate*mWeightcost,mVisHid,1.0);//L2 + if(mHidType==BERNOULLI) { + mVisHidCorrection.BlasGer(-mSparsityCost,mVisMean,mSparsityQCurrent);//sparsity + } + mVisHid.AddScaled(1.0,mVisHidCorrection,1.0); + + // UPDATE visbias vector + // + // visbiasinc = momentum*visbiasinc + (epsilonvb/numcases)*(posvisact-negvisact); + // + mVisBiasCorrection.AddColSum(-mLearningRate/N,neg_vis,mMomentum); + mVisBiasCorrection.AddColSum(+mLearningRate/N,pos_vis,1.0); + mVisBias.AddScaled(1.0,mVisBiasCorrection,1.0); + + // UPDATE hidbias vector + // + // hidbiasinc = momentum*hidbiasinc + (epsilonhb/numcases)*(poshidact-neghidact); + // + mHidBiasCorrection.AddColSum(-mLearningRate/N,neg_hid,mMomentum); + mHidBiasCorrection.AddColSum(+mLearningRate/N,pos_hid,1.0); + if(mHidType==BERNOULLI) { + mHidBiasCorrection.AddScaled(-mSparsityCost,mSparsityQCurrent,1.0);//sparsity + } + mHidBias.AddScaled(1.0/*0.0*/,mHidBiasCorrection,1.0); + + } + + + void + CuRbmSparse:: + ReadFromStream(std::istream& rIn) + { + //type of the units + std::string str; + + rIn >> std::ws >> str; + if(0 == str.compare("bern")) { + mVisType = BERNOULLI; + } else if(0 == str.compare("gauss")) { + mVisType = GAUSSIAN; + } else Error(std::string("Invalid unit type: ")+str); + + rIn >> std::ws >> str; + if(0 == str.compare("bern")) { + mHidType = BERNOULLI; + } else if(0 == str.compare("gauss")) { + mHidType = GAUSSIAN; + } else Error(std::string("Invalid unit type: ")+str); + + + //matrix is stored transposed as SNet does + BfMatrix transpose; + rIn >> transpose; + mVisHid.CopyFrom(BfMatrix(transpose, TRANS)); + //biases stored normally + BfVector bias; + rIn >> bias; + mVisBias.CopyFrom(bias); + rIn >> bias; + mHidBias.CopyFrom(bias); + + rIn >> std::ws >> mSparsityCost; + std::cout << "RBM::mSparsityCost=" << mSparsityCost; + } + + + void + CuRbmSparse:: + WriteToStream(std::ostream& rOut) + { + //store unit type info + if(mVisType == BERNOULLI) { + rOut << " bern "; + } else { + rOut << " gauss "; + } + if(mHidType == BERNOULLI) { + rOut << " bern\n"; + } else { + rOut << " gauss\n"; + } + + //matrix is stored transposed as SNet does + BfMatrix tmp; + mVisHid.CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + //biases stored normally + BfVector vec; + mVisBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + mHidBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + //store the sparsity cost + rOut << mSparsityCost << std::endl; + } + + +} //namespace diff --git a/src/CuTNetLib/.svn/text-base/cuRbmSparse.h.svn-base b/src/CuTNetLib/.svn/text-base/cuRbmSparse.h.svn-base new file mode 100644 index 0000000..9d7e304 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuRbmSparse.h.svn-base @@ -0,0 +1,134 @@ +#ifndef _CU_RBM_SPARSE_H_ +#define _CU_RBM_SPARSE_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" +#include "cuRbm.h" + + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + + class CuRbmSparse : public CuRbmBase + { + public: + + CuRbmSparse(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuRbmSparse(); + + ComponentType GetType() const; + const char* GetName() const; + + //CuUpdatableComponent API + void PropagateFnc(const CuMatrix& X, CuMatrix& Y); + void BackpropagateFnc(const CuMatrix& X, CuMatrix& Y); + + void Update(); + + //RBM training API + void Propagate(const CuMatrix& visProbs, CuMatrix& hidProbs); + void Reconstruct(const CuMatrix& hidState, CuMatrix& visProbs); + void RbmUpdate(const CuMatrix& pos_vis, const CuMatrix& pos_hid, const CuMatrix& neg_vis, const CuMatrix& neg_hid); + + RbmUnitType VisType() + { return mVisType; } + + RbmUnitType HidType() + { return mHidType; } + + //static void BinarizeProbs(const CuMatrix& probs, CuMatrix& states); + + //I/O + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + CuMatrix mVisHid; ///< Matrix with neuron weights + CuVector mVisBias; ///< Vector with biases + CuVector mHidBias; ///< Vector with biases + + CuMatrix mVisHidCorrection; ///< Matrix for linearity updates + CuVector mVisBiasCorrection; ///< Vector for bias updates + CuVector mHidBiasCorrection; ///< Vector for bias updates + + CuMatrix mBackpropErrBuf; + + RbmUnitType mVisType; + RbmUnitType mHidType; + + ////// sparsity + BaseFloat mSparsityPrior; ///< sparsity target (unit activity prior) + BaseFloat mLambda; ///< exponential decay factor for q (observed probability of unit to be active) + BaseFloat mSparsityCost; ///< sparsity cost coef. + + CuVector mSparsityQ; + CuVector mSparsityQCurrent; + CuVector mVisMean; ///< buffer for mean visible + + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuRbmSparse:: + inline + CuRbmSparse:: + CuRbmSparse(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuRbmBase(nInputs, nOutputs, pPred), + mVisHid(nInputs,nOutputs), + mVisBias(nInputs), mHidBias(nOutputs), + mVisHidCorrection(nInputs,nOutputs), + mVisBiasCorrection(nInputs), mHidBiasCorrection(nOutputs), + mBackpropErrBuf(), + mVisType(BERNOULLI), + mHidType(BERNOULLI), + + mSparsityPrior(0.0001), + mLambda(0.95), + mSparsityCost(1e-7), + mSparsityQ(nOutputs), + mSparsityQCurrent(nOutputs), + mVisMean(nInputs) + { + mVisHidCorrection.SetConst(0.0); + mVisBiasCorrection.SetConst(0.0); + mHidBiasCorrection.SetConst(0.0); + + mSparsityQ.SetConst(mSparsityPrior); + mSparsityQCurrent.SetConst(0.0); + mVisMean.SetConst(0.0); + } + + + inline + CuRbmSparse:: + ~CuRbmSparse() + { } + + inline CuComponent::ComponentType + CuRbmSparse:: + GetType() const + { + return CuComponent::RBM_SPARSE; + } + + inline const char* + CuRbmSparse:: + GetName() const + { + return ""; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuRecurrent.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuRecurrent.cc.svn-base new file mode 100644 index 0000000..428df2c --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuRecurrent.cc.svn-base @@ -0,0 +1,191 @@ + +#include +#include + +#include "cuRecurrent.h" + +#include "cumath.h" +#include "cuda_runtime.h" + + +namespace TNet +{ + + void + CuRecurrent:: + PropagateFnc(const CuMatrix& X, CuMatrix& Y) + { + assert(X.Rows() == 1); + assert(Y.Rows() == 1); + if(mInputHistory.Rows() == 0) { + Error("Bptt order was not set"); + } + + //pushback the history + CuMatrix tmp(mInputHistory.Rows()-1,mInputHistory.Cols()); + tmp.CopyRows(tmp.Rows(),0,mInputHistory,0); + mInputHistory.CopyRows(tmp.Rows(),0,tmp,1); + + //compose the input vector to 0th row, use input X and previous Y + cudaMemcpy(mInputHistory.pCUData(), X.pCUData(), + sizeof(BaseFloat)*X.Cols(), cudaMemcpyDeviceToDevice); + cudaMemcpy(mInputHistory.pCUData()+X.Cols(), Y.pCUData(), + sizeof(BaseFloat)*Y.Cols(), cudaMemcpyDeviceToDevice); + + //extract first row + //CuMatrix first_row(1,mInputHistory.Cols()); + //first_row.CopyRows(1,0,mInputHistory,0); + + //calculate the output + Y.AddScaledRow(1.0,mBias,0.0); + //take 0th vector of history, propagate + CuMath::OffsetGemv('T',1.0,mLinearity,mInputHistory.pCUData(),mInputHistory.Cols(),1.0,Y.pCUData(),Y.Cols(),0); + //Y.Gemm('N','N', 1.0, first_row, mLinearity, 1.0); + CuMath::Sigmoid(Y,Y); + + /* + std::cout << "-------------------------------------" << std::endl; + X.Print(); + Y.Print(); + mInputHistory.Print(); + */ + + } + + + void + CuRecurrent:: + BackpropagateFnc(const CuMatrix& X, CuMatrix& Y) + { + assert(Y.Rows() == 1); + assert(X.Rows() == 1); + + //apply diff sigmoid + CuMatrix diff_sigm(1,X.Cols()); + CuMath::DiffSigmoid(diff_sigm,X,GetOutput()); + + //:TODO: inefficent to calculate all the input errors!!! + // we need only part of them! + // + //backward-multiply by weights + /* + CuMatrix err_prev(1,mLinearity.Rows()); + err_prev.Gemm('N', 'T', 1.0, diff_sigm, mLinearity, 0.0); + + //copy out the interval + cudaMemcpy(Y.pCUData(),err_prev.pCUData(), + sizeof(BaseFloat)*Y.Cols(),cudaMemcpyDeviceToDevice); + */ + + //backward-multiply by weights + CuMath::OffsetGemv('N',1.0,mLinearity,diff_sigm.pCUData(),diff_sigm.Cols(),1.0,Y.pCUData(),Y.Cols(),0); + + } + + + void + CuRecurrent:: + Update() + { + // + //correction from PRESENT input x error pair + // + //apply diff sigmoid + CuMatrix diff_sigm(1,GetOutput().Cols()); + CuMath::DiffSigmoid(diff_sigm,GetErrorInput(),GetOutput()); + + //get 0th row of history (present time) + CuMatrix history_row(1,mInputHistory.Cols()); + history_row.CopyRows(1,0,mInputHistory,0); + + //calculate update + //mLinearityCorrection.Gemm('T','N',-mLearningRate,history_row,diff_sigm,mMomentum); + mLinearityCorrection.SetConst(0.0); //:TODO: should be scale/momentum + CuMath::BlasGer(-mLearningRate,history_row.pCUData(),history_row.Cols(),diff_sigm.pCUData(),diff_sigm.Cols(),mLinearityCorrection); + + mBiasCorrection.AddColSum(-mLearningRate,diff_sigm,mMomentum); + + // + //BPTT (backprop through time) + // + CuMatrix err_prev(1,mLinearity.Rows()); + CuMatrix err_prev_part(1,diff_sigm.Cols()); + CuMatrix history_output(1,GetOutput().Cols()); + for(int i=1; i<=mBpttOrder; i++) { + //:TODO: inefficent to calculate all the input errors!!! + // we need only part of them! + // + /* + //get previous error + err_prev.Gemm('N','T',1.0,diff_sigm,mLinearity,0.0); + //select interval + cudaMemcpy(err_prev_part.pCUData(),err_prev.pCUData()+GetNInputs(), + sizeof(BaseFloat)*err_prev_part.Cols(),cudaMemcpyDeviceToDevice); + */ + + //backward-multiply by weights + CuMath::OffsetGemv('N',1.0,mLinearity,diff_sigm.pCUData(),diff_sigm.Cols(),0.0,err_prev_part.pCUData(),err_prev_part.Cols(),GetInput().Cols()); + + //apply diff sigmoid with activations of HISTORY frame!!! + cudaMemcpy(history_output.pCUData(), mInputHistory.pCURowData(i-1)+GetInput().Cols(), + sizeof(BaseFloat)*history_output.Cols(), cudaMemcpyDeviceToDevice); + CuMath::DiffSigmoid(diff_sigm,err_prev_part,history_output); + + //get history row + history_row.CopyRows(1,i,mInputHistory,0); + + //accu the update + //mLinearityCorrection.Gemm('T','N',-mLearningRate,history_row,diff_sigm,1.0); + CuMath::BlasGer(-mLearningRate,history_row.pCUData(),history_row.Cols(),diff_sigm.pCUData(),diff_sigm.Cols(),mLinearityCorrection); + mBiasCorrection.AddColSum(-mLearningRate,diff_sigm,1.0); + } + + // + //update the weights + // + //regularization weight decay + mLinearityCorrection.AddScaled(-mLearningRate*mWeightcost,mLinearity,1.0); + + //perform update + mLinearity.AddScaled(1.0,mLinearityCorrection,1.0); + mBias.AddScaled(1.0,mBiasCorrection,1.0); + + } + + + + + void + CuRecurrent:: + ReadFromStream(std::istream& rIn) + { + //matrix is stored transposed as SNet does + BfMatrix transpose; + rIn >> transpose; + mLinearity.CopyFrom(BfMatrix(transpose, TRANS)); + //biases stored normally + BfVector bias; + rIn >> bias; + mBias.CopyFrom(bias); + } + + + void + CuRecurrent:: + WriteToStream(std::ostream& rOut) + { + //matrix is stored transposed as SNet does + BfMatrix tmp; + mLinearity.CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + //biases stored normally + BfVector vec; + mBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + } + + +} //namespace + diff --git a/src/CuTNetLib/.svn/text-base/cuRecurrent.h.svn-base b/src/CuTNetLib/.svn/text-base/cuRecurrent.h.svn-base new file mode 100644 index 0000000..e487b27 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuRecurrent.h.svn-base @@ -0,0 +1,101 @@ +#ifndef _CU_RECURRENT_H_ +#define _CU_RECURRENT_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + + class CuRecurrent : public CuUpdatableComponent + { + public: + + CuRecurrent(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuRecurrent(); + + ComponentType GetType() const; + const char* GetName() const; + + //CuUpdatableComponent API + void PropagateFnc(const CuMatrix& X, CuMatrix& Y); + void BackpropagateFnc(const CuMatrix& X, CuMatrix& Y); + + void Update(); + + //Recurrent training API + void BpttOrder(int ord) { + mBpttOrder = ord; + mInputHistory.Init(ord+1,GetNInputs()+GetNOutputs()); + } + void ClearHistory() { + mInputHistory.SetConst(0.0); + if(mOutput.MSize() > 0) { + mOutput.SetConst(0.0); + } + } + + //I/O + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + CuMatrix mLinearity; + CuVector mBias; + + CuMatrix mLinearityCorrection; + CuVector mBiasCorrection; + + CuMatrix mInputHistory; + + int mBpttOrder; + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuRecurrent:: + inline + CuRecurrent:: + CuRecurrent(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred), + mLinearity(nInputs+nOutputs,nOutputs), + mBias(nOutputs), + mLinearityCorrection(nInputs+nOutputs,nOutputs), + mBiasCorrection(nOutputs) + { } + + + inline + CuRecurrent:: + ~CuRecurrent() + { } + + inline CuComponent::ComponentType + CuRecurrent:: + GetType() const + { + return CuComponent::RECURRENT; + } + + inline const char* + CuRecurrent:: + GetName() const + { + return ""; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuSharedLinearity.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuSharedLinearity.cc.svn-base new file mode 100644 index 0000000..8d5ec09 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuSharedLinearity.cc.svn-base @@ -0,0 +1,179 @@ + + +#include "cuSharedLinearity.h" +#include "cumath.h" + + +namespace TNet +{ + + void + CuSharedLinearity:: + PropagateFnc(const CuMatrix& X, CuMatrix& Y) + { + CuMath::VecExpand(mBias,mBiasExpand); /// [ 1 2 3 ] -> [ 1 2 3 1 2 3 ... ] + Y.AddScaledRow(1.0,mBiasExpand,0.0); + + //mBiasExpand.Print(); + + for(int i=0; i::OffsetGemm('N','N', 1.0, X, mLinearity, 1.0, Y, + i*mLinearity.Rows(), 0, i*mLinearity.Cols()); + } + //std::cout << CuDevice::Instantiate().GetFreeMemory(); + //GetInput().Print(); + //GetOutput().Print(); + } + + + void + CuSharedLinearity:: + BackpropagateFnc(const CuMatrix& X, CuMatrix& Y) + { + for(int i=0; i::OffsetGemm('N', 'T', 1.0, X, mLinearity, 0.0, Y, + i*mLinearity.Cols(), 0, i*mLinearity.Rows()); + } + } + + + void + CuSharedLinearity:: + Update() + { +#if 0 + //former implementation + BaseFloat N = static_cast(GetInput().Rows()); + + for(int i=0; i::OffsetGemm('T','N',-mLearningRate/(N*mNInstances), + GetInput(),GetErrorInput(), + ((i==0)?mMomentum:1.0f), mLinearityCorrection, + i*mLinearity.Rows(),i*mLinearity.Cols(),0); + } + mBiasCorrectionExpand.AddColSum(1.0,GetErrorInput(),0.0); + CuMath::VecAddColSum(-mLearningRate/(N*mNInstances),mBiasCorrectionExpand,mMomentum,mBiasCorrection); + + + //regularization weight decay + mLinearityCorrection.AddScaled(-mLearningRate*mWeightcost,mLinearity,1.0); + + mLinearity.AddScaled(1.0,mLinearityCorrection,1.0); + mBias.AddScaled(1.0,mBiasCorrection,1.0); +#endif + +#if 1 + //new implementation + BaseFloat N = 1; + if(mGradDivFrm) { + N = static_cast(GetInput().Rows()); + } + BaseFloat mmt_gain = static_cast(1.0/(1.0-mMomentum)); + N *= mmt_gain; //compensate higher gradient estimates due to momentum + + //compensate augmented dyn. range of gradient caused by multiple instances + N *= static_cast(mNInstances); + + //get gradient of shared linearity + for(int i=0; i::OffsetGemm('T','N',1.0, + GetInput(),GetErrorInput(), + ((i==0)?mMomentum:1.0f), mLinearityCorrection, + i*mLinearity.Rows(),i*mLinearity.Cols(),0); + } + //get gradient of shared bias + mBiasCorrectionExpand.AddColSum(1.0,GetErrorInput(),0.0); + CuMath::VecAddColSum(1.0,mBiasCorrectionExpand,mMomentum,mBiasCorrection); + + //perform update + mLinearity.AddScaled(-mLearningRate/N,mLinearityCorrection,1.0); + mBias.AddScaled(-mLearningRate/N,mBiasCorrection,1.0); + + //regularization weight decay + mLinearity.AddScaled(-mLearningRate*mWeightcost,mLinearity,1.0); +#endif + + } + + + void + CuSharedLinearity:: + ReadFromStream(std::istream& rIn) + { + //number of instances of shared weights in layer + rIn >> std::ws >> mNInstances; + if(mNInstances < 1) { + std::ostringstream os; + os << "Bad number of instances:" << mNInstances; + Error(os.str()); + } + if(GetNInputs() % mNInstances != 0 || GetNOutputs() % mNInstances != 0) { + std::ostringstream os; + os << "Number of Inputs/Outputs must be divisible by number of instances" + << " Inputs:" << GetNInputs() + << " Outputs" << GetNOutputs() + << " Intances:" << mNInstances; + Error(os.str()); + } + + //matrix is stored transposed as SNet does + BfMatrix transpose; + rIn >> transpose; + mLinearity.CopyFrom(BfMatrix(transpose, TRANS)); + //biases stored normally + BfVector bias; + rIn >> bias; + mBias.CopyFrom(bias); + + if(transpose.Cols()*transpose.Rows() == 0) { + Error("Missing linearity matrix in network file"); + } + if(bias.Dim() == 0) { + Error("Missing bias vector in network file"); + } + + + if(mLinearity.Cols() != GetNOutputs() / mNInstances || + mLinearity.Rows() != GetNInputs() / mNInstances || + mBias.Dim() != GetNOutputs() / mNInstances + ){ + std::ostringstream os; + os << "Wrong dimensionalities of matrix/vector in network file\n" + << "Inputs:" << GetNInputs() + << "Outputs:" << GetNOutputs() + << "\n" + << "linearityCols:" << mLinearity.Cols() + << "linearityRows:" << mLinearity.Rows() + << "biasDims:" << mBias.Dim() + << "\n"; + Error(os.str()); + } + + mLinearityCorrection.Init(mLinearity.Rows(),mLinearity.Cols()); + mBiasCorrection.Init(mBias.Dim()); + + mBiasExpand.Init(mBias.Dim()*mNInstances); + mBiasCorrectionExpand.Init(mBias.Dim()*mNInstances); + } + + + void + CuSharedLinearity:: + WriteToStream(std::ostream& rOut) + { + rOut << mNInstances << std::endl; + + //matrix is stored transposed as SNet does + BfMatrix tmp; + mLinearity.CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + //biases stored normally + BfVector vec; + mBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + } + + +} //namespace diff --git a/src/CuTNetLib/.svn/text-base/cuSharedLinearity.h.svn-base b/src/CuTNetLib/.svn/text-base/cuSharedLinearity.h.svn-base new file mode 100644 index 0000000..4aa022a --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuSharedLinearity.h.svn-base @@ -0,0 +1,85 @@ +#ifndef _CUSHARED_LINEARITY_H_ +#define _CUSHARED_LINEARITY_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + + class CuSharedLinearity : public CuUpdatableComponent + { + public: + + CuSharedLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuSharedLinearity(); + + ComponentType GetType() const; + const char* GetName() const; + + void PropagateFnc(const CuMatrix& X, CuMatrix& Y); + void BackpropagateFnc(const CuMatrix& X, CuMatrix& Y); + + void Update(); + + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + CuMatrix mLinearity; ///< Matrix with neuron weights + CuVector mBias; ///< Vector with biases + + CuMatrix mLinearityCorrection; ///< Matrix for linearity updates + CuVector mBiasCorrection; ///< Vector for bias updates + + int mNInstances; + CuVector mBiasExpand; + CuVector mBiasCorrectionExpand; + + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuSharedLinearity:: + inline + CuSharedLinearity:: + CuSharedLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred), + mNInstances(0) + { } + + + inline + CuSharedLinearity:: + ~CuSharedLinearity() + { } + + inline CuComponent::ComponentType + CuSharedLinearity:: + GetType() const + { + return CuComponent::SHARED_LINEARITY; + } + + inline const char* + CuSharedLinearity:: + GetName() const + { + return ""; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuSparseLinearity.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuSparseLinearity.cc.svn-base new file mode 100644 index 0000000..2f1159b --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuSparseLinearity.cc.svn-base @@ -0,0 +1,190 @@ + + +#include "cuSparseLinearity.h" +#include +#include + + +namespace TNet +{ + + void + CuSparseLinearity:: + PropagateFnc(const CuMatrix& X, CuMatrix& Y) + { + Y.AddScaledRow(1.0,mBias,0.0); + Y.Gemm('N','N', 1.0, X, mLinearity, 1.0); + } + + + void + CuSparseLinearity:: + BackpropagateFnc(const CuMatrix& X, CuMatrix& Y) + { + Y.Gemm('N', 'T', 1.0, X, mLinearity, 0.0); + } + + + void + CuSparseLinearity:: + Update() + { + BaseFloat N = 1; + if(mGradDivFrm) { + N = static_cast(GetInput().Rows()); + } + BaseFloat mmt_gain = static_cast(1.0/(1.0-mMomentum)); + N *= mmt_gain; + + mLinearityCorrection.Gemm('T','N',1.0,GetInput(),GetErrorInput(),mMomentum); + mBiasCorrection.AddColSum(1.0,GetErrorInput(),mMomentum); + + mLinearity.AddScaled(-mLearningRate/N,mLinearityCorrection,1.0); + mBias.AddScaled(-mLearningRate/N,mBiasCorrection,1.0); + + mLinearityCorrectionAccu.AddScaled(1.0,mLinearityCorrection,1.0); + mLinearity.ApplyMask(mSparsityMask); + + //L1 regularization lassoo... + //each update? everty 1000th update? + if(mL1Const > 0) { + BaseFloat L1_const = mLearningRate*mL1Const*(mGradDivFrm?1.0:GetInput().Rows()); + mLinearity.ApplyL1(L1_const); + } + + //L2 regularization weight decay (from actual weights only) + if(mWeightcost > 0) { + BaseFloat L2_decay = -mLearningRate*mWeightcost*(mGradDivFrm?1.0:GetInput().Rows()); + mLinearity.AddScaled(L2_decay, mLinearity,1.0); + } + + mNFrames += GetInput().Rows(); + + } + + + void + CuSparseLinearity:: + UpdateMask() + { + //move data to host + Matrix linearity, linearity_correction_accu; + Matrix sparsity_mask; + + mLinearity.CopyTo(linearity); + mLinearityCorrectionAccu.CopyTo(linearity_correction_accu); + mSparsityMask.CopyTo(sparsity_mask); + + //decide on new sparsity mask + for(size_t r=0; r mUnsparsifyAccu) { + sparsity_mask(r,c) = 1;//activate + } + } + } + } + + //move data to the device + mLinearity.CopyFrom(linearity); + mSparsityMask.CopyFrom(sparsity_mask); + } + + + void + CuSparseLinearity:: + ReadFromStream(std::istream& rIn) + { + //matrix is stored transposed as SNet does + BfMatrix transpose; + rIn >> transpose; + mLinearity.CopyFrom(BfMatrix(transpose, TRANS)); + //biases stored normally + BfVector bias; + rIn >> bias; + mBias.CopyFrom(bias); + + //sparsity mask + rIn >> std::ws; + Matrix mask_transp; + if(rIn.peek() == 'm') {//load from file + rIn >> mask_transp; + } else {//or set all elements active + mask_transp.Init(transpose.Rows(),transpose.Cols()); + int items=transpose.Rows()*transpose.Stride(); + BaseFloat* p = mask_transp.pData(); + for(int i=0; i> std::ws; + if(rIn.peek() == 'm') {//load from file + BfMatrix dummy; + rIn >> dummy; + } + + if(transpose.Cols()*transpose.Rows() == 0) { + Error("Missing linearity matrix in network file"); + } + if(bias.Dim() == 0) { + Error("Missing bias vector in network file"); + } + if(mLinearity.Cols() != GetNOutputs() || + mLinearity.Rows() != GetNInputs() || + mBias.Dim() != GetNOutputs() + ){ + std::ostringstream os; + os << "Wrong dimensionalities of matrix/vector in network file\n" + << "Inputs:" << GetNInputs() + << "Outputs:" << GetNOutputs() + << "\n" + << "linearityCols:" << mLinearity.Cols() + << "linearityRows:" << mLinearity.Rows() + << "biasDims:" << mBias.Dim() + << "\n"; + Error(os.str()); + } + + assert(mLinearity.Rows() == mSparsityMask.Rows()); + assert(mLinearity.Cols() == mSparsityMask.Cols()); + + } + + + void + CuSparseLinearity:: + WriteToStream(std::ostream& rOut) + { + UpdateMask(); + + //matrix is stored transposed as SNet does + BfMatrix tmp; + mLinearity.CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + //biases stored normally + BfVector vec; + mBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + //store mask + mSparsityMask.CopyTo(tmp); + rOut << BfMatrix(tmp,TRANS); + //store accu + mLinearityCorrectionAccu.CopyTo(tmp); + rOut << BfMatrix(tmp,TRANS); + + } + + +} //namespace + diff --git a/src/CuTNetLib/.svn/text-base/cuSparseLinearity.h.svn-base b/src/CuTNetLib/.svn/text-base/cuSparseLinearity.h.svn-base new file mode 100644 index 0000000..c2b6d6f --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuSparseLinearity.h.svn-base @@ -0,0 +1,104 @@ +#ifndef _CUSPARSE_LINEARITY_H_ +#define _CUSPARSE_LINEARITY_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + + class CuSparseLinearity : public CuUpdatableComponent + { + public: + + CuSparseLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuSparseLinearity(); + + ComponentType GetType() const; + const char* GetName() const; + + void PropagateFnc(const CuMatrix& X, CuMatrix& Y); + void BackpropagateFnc(const CuMatrix& X, CuMatrix& Y); + + void Update(); + void UpdateMask(); + + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + void L1(BaseFloat l1) { + mL1Const = l1; + } + + protected: + CuMatrix mLinearity; ///< Matrix with neuron weights + CuVector mBias; ///< Vector with biases + CuMatrix mSparsityMask; ///< Mask which selects active weights + + CuMatrix mLinearityCorrection; ///< Matrix for linearity updates + CuVector mBiasCorrection; ///< Vector for bias updates + + CuMatrix mLinearityCorrectionAccu; ///< Accumulator for linearity updates + + BaseFloat mL1Const; ///< L1 regularization constant + + size_t mNFrames; ///< Number of accumulated frames + BaseFloat mSparsifyWeightThreshold; ///< Cutoff + BaseFloat mUnsparsifyAccu; ///< Threshold to unsparsify the Cutoff + + + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuSparseLinearity:: + inline + CuSparseLinearity:: + CuSparseLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred), + mLinearity(nInputs,nOutputs), mBias(nOutputs), mSparsityMask(nInputs,nOutputs), + mLinearityCorrection(nInputs,nOutputs), mBiasCorrection(nOutputs), + mLinearityCorrectionAccu(nInputs,nOutputs), + mNFrames(0), mSparsifyWeightThreshold(1.0e-3), + mUnsparsifyAccu(1e20f) + { + mLinearityCorrection.SetConst(0.0f); + mBiasCorrection.SetConst(0.0f); + mLinearityCorrectionAccu.SetConst(0.0f); + } + + + inline + CuSparseLinearity:: + ~CuSparseLinearity() + { } + + inline CuComponent::ComponentType + CuSparseLinearity:: + GetType() const + { + return CuComponent::SPARSE_LINEARITY; + } + + inline const char* + CuSparseLinearity:: + GetName() const + { + return ""; + } + + + +} //namespace + + + +#endif -- cgit v1.2.3-70-g09d2