diff options
Diffstat (limited to 'src/CuTNetLib')
84 files changed, 13952 insertions, 0 deletions
diff --git a/src/CuTNetLib/.depend.mk b/src/CuTNetLib/.depend.mk new file mode 100644 index 0000000..bcf9cb2 --- /dev/null +++ b/src/CuTNetLib/.depend.mk @@ -0,0 +1,2250 @@ +cuActivation.o: cuActivation.cc cuActivation.h cuComponent.h \ + ../KaldiLib/Vector.h /usr/include/c++/4.6/cstddef \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++config.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/os_defines.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-32.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/cpu_defines.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stddef.h \ + /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ + /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ + /usr/include/endian.h /usr/include/bits/endian.h \ + /usr/include/bits/byteswap.h /usr/include/xlocale.h \ + /usr/include/sys/types.h /usr/include/bits/types.h \ + /usr/include/bits/typesizes.h /usr/include/time.h \ + /usr/include/sys/select.h /usr/include/bits/select.h \ + /usr/include/bits/sigset.h /usr/include/bits/time.h \ + /usr/include/bits/select2.h /usr/include/sys/sysmacros.h \ + /usr/include/bits/pthreadtypes.h /usr/include/alloca.h \ + /usr/include/bits/stdlib.h /usr/include/c++/4.6/stdexcept \ + /usr/include/c++/4.6/exception /usr/include/c++/4.6/string \ + /usr/include/c++/4.6/bits/stringfwd.h \ + /usr/include/c++/4.6/bits/char_traits.h \ + /usr/include/c++/4.6/bits/stl_algobase.h \ + /usr/include/c++/4.6/bits/functexcept.h \ + /usr/include/c++/4.6/bits/exception_defines.h \ + /usr/include/c++/4.6/bits/cpp_type_traits.h \ + /usr/include/c++/4.6/ext/type_traits.h \ + /usr/include/c++/4.6/ext/numeric_traits.h \ + /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ + /usr/include/c++/4.6/bits/concept_check.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.6/bits/stl_iterator.h \ + /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/postypes.h \ + /usr/include/c++/4.6/cwchar /usr/include/wchar.h /usr/include/stdio.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stdarg.h \ + /usr/include/bits/wchar.h /usr/include/bits/wchar2.h \ + /usr/include/c++/4.6/bits/allocator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++allocator.h \ + /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ + /usr/include/c++/4.6/bits/localefwd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++locale.h \ + /usr/include/c++/4.6/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ + /usr/include/c++/4.6/cctype /usr/include/ctype.h \ + /usr/include/c++/4.6/bits/ostream_insert.h \ + /usr/include/c++/4.6/bits/cxxabi_forced.h \ + /usr/include/c++/4.6/bits/stl_function.h \ + /usr/include/c++/4.6/backward/binders.h \ + /usr/include/c++/4.6/bits/range_access.h \ + /usr/include/c++/4.6/bits/basic_string.h \ + /usr/include/c++/4.6/ext/atomicity.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/bits/sched.h \ + /usr/include/bits/timex.h /usr/include/bits/setjmp.h \ + /usr/include/unistd.h /usr/include/bits/posix_opt.h \ + /usr/include/bits/environments.h /usr/include/bits/confname.h \ + /usr/include/getopt.h /usr/include/bits/unistd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/atomic_word.h \ + /usr/include/c++/4.6/initializer_list \ + /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/iostream \ + /usr/include/c++/4.6/ostream /usr/include/c++/4.6/ios \ + /usr/include/c++/4.6/bits/ios_base.h \ + /usr/include/c++/4.6/bits/locale_classes.h \ + /usr/include/c++/4.6/bits/locale_classes.tcc \ + /usr/include/c++/4.6/streambuf /usr/include/c++/4.6/bits/streambuf.tcc \ + /usr/include/c++/4.6/bits/basic_ios.h \ + /usr/include/c++/4.6/bits/locale_facets.h /usr/include/c++/4.6/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_base.h \ + /usr/include/c++/4.6/bits/streambuf_iterator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_inline.h \ + /usr/include/c++/4.6/bits/locale_facets.tcc \ + /usr/include/c++/4.6/bits/basic_ios.tcc \ + /usr/include/c++/4.6/bits/ostream.tcc /usr/include/c++/4.6/istream \ + /usr/include/c++/4.6/bits/istream.tcc ../KaldiLib/cblas.h \ + ../KaldiLib/clapack.h ../KaldiLib/cblas.h ../KaldiLib/Common.h \ + /usr/include/string.h /usr/include/bits/string3.h \ + /usr/include/c++/4.6/sstream /usr/include/c++/4.6/bits/sstream.tcc \ + ../KaldiLib/MathAux.h /usr/include/c++/4.6/cmath /usr/include/math.h \ + /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ + /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ + /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ + /usr/include/bits/mathcalls.h /usr/include/bits/mathinline.h \ + ../KaldiLib/Types.h ../KaldiLib/Error.h /usr/include/execinfo.h \ + ../KaldiLib/Vector.tcc /usr/include/c++/4.6/cstring \ + /usr/include/c++/4.6/fstream /usr/include/c++/4.6/bits/codecvt.h \ + /usr/include/c++/4.6/cstdio /usr/include/libio.h \ + /usr/include/_G_config.h /usr/include/bits/stdio_lim.h \ + /usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \ + /usr/include/bits/stdio2.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/basic_file.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++io.h \ + /usr/include/c++/4.6/bits/fstream.tcc /usr/include/c++/4.6/iomanip \ + ../KaldiLib/Matrix.h ../KaldiLib/Matrix.tcc /usr/include/c++/4.6/cfloat \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/float.h \ + /usr/include/c++/4.6/typeinfo /usr/include/c++/4.6/algorithm \ + /usr/include/c++/4.6/utility /usr/include/c++/4.6/bits/stl_relops.h \ + /usr/include/c++/4.6/bits/stl_algo.h \ + /usr/include/c++/4.6/bits/algorithmfwd.h \ + /usr/include/c++/4.6/bits/stl_heap.h \ + /usr/include/c++/4.6/bits/stl_tempbuf.h \ + /usr/include/c++/4.6/bits/stl_construct.h /usr/include/c++/4.6/limits \ + /usr/include/c++/4.6/vector \ + /usr/include/c++/4.6/bits/stl_uninitialized.h \ + /usr/include/c++/4.6/bits/stl_vector.h \ + /usr/include/c++/4.6/bits/stl_bvector.h \ + /usr/include/c++/4.6/bits/vector.tcc ../KaldiLib/Vector.h \ + ../KaldiLib/Matrix.h ../KaldiLib/Error.h ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cukernels.h /usr/local/cuda-5.0/include/vector_types.h \ + /usr/local/cuda-5.0/include/builtin_types.h \ + /usr/local/cuda-5.0/include/device_types.h \ + /usr/local/cuda-5.0/include/host_defines.h \ + /usr/local/cuda-5.0/include/driver_types.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/limits.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/syslimits.h \ + /usr/include/limits.h /usr/include/bits/posix1_lim.h \ + /usr/include/bits/local_lim.h /usr/include/linux/limits.h \ + /usr/include/bits/posix2_lim.h /usr/include/bits/xopen_lim.h \ + /usr/local/cuda-5.0/include/surface_types.h \ + /usr/local/cuda-5.0/include/texture_types.h \ + /usr/local/cuda-5.0/include/vector_types.h ../CuBaseLib/cumatrix.tcc \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/cuda_device_runtime_api.h \ + /usr/local/cuda-5.0/include/cublas.h \ + /usr/local/cuda-5.0/include/cuda_runtime.h \ + /usr/local/cuda-5.0/include/host_config.h \ + /usr/local/cuda-5.0/include/channel_descriptor.h \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/driver_functions.h \ + /usr/local/cuda-5.0/include/vector_functions.h \ + /usr/local/cuda-5.0/include/cublas_api.h \ + /usr/local/cuda-5.0/include/cuComplex.h ../KaldiLib/Timer.h \ + /usr/include/sys/time.h ../CuBaseLib/cucommon.h ../CuBaseLib/cuvector.h \ + ../CuBaseLib/cuvector.tcc ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cudevice.h /usr/include/c++/4.6/map \ + /usr/include/c++/4.6/bits/stl_tree.h /usr/include/c++/4.6/bits/stl_map.h \ + /usr/include/c++/4.6/bits/stl_multimap.h ../CuBaseLib/cumath.h +cuBiasedLinearity.o: cuBiasedLinearity.cc cuBiasedLinearity.h \ + cuComponent.h ../KaldiLib/Vector.h /usr/include/c++/4.6/cstddef \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++config.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/os_defines.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-32.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/cpu_defines.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stddef.h \ + /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ + /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ + /usr/include/endian.h /usr/include/bits/endian.h \ + /usr/include/bits/byteswap.h /usr/include/xlocale.h \ + /usr/include/sys/types.h /usr/include/bits/types.h \ + /usr/include/bits/typesizes.h /usr/include/time.h \ + /usr/include/sys/select.h /usr/include/bits/select.h \ + /usr/include/bits/sigset.h /usr/include/bits/time.h \ + /usr/include/bits/select2.h /usr/include/sys/sysmacros.h \ + /usr/include/bits/pthreadtypes.h /usr/include/alloca.h \ + /usr/include/bits/stdlib.h /usr/include/c++/4.6/stdexcept \ + /usr/include/c++/4.6/exception /usr/include/c++/4.6/string \ + /usr/include/c++/4.6/bits/stringfwd.h \ + /usr/include/c++/4.6/bits/char_traits.h \ + /usr/include/c++/4.6/bits/stl_algobase.h \ + /usr/include/c++/4.6/bits/functexcept.h \ + /usr/include/c++/4.6/bits/exception_defines.h \ + /usr/include/c++/4.6/bits/cpp_type_traits.h \ + /usr/include/c++/4.6/ext/type_traits.h \ + /usr/include/c++/4.6/ext/numeric_traits.h \ + /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ + /usr/include/c++/4.6/bits/concept_check.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.6/bits/stl_iterator.h \ + /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/postypes.h \ + /usr/include/c++/4.6/cwchar /usr/include/wchar.h /usr/include/stdio.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stdarg.h \ + /usr/include/bits/wchar.h /usr/include/bits/wchar2.h \ + /usr/include/c++/4.6/bits/allocator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++allocator.h \ + /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ + /usr/include/c++/4.6/bits/localefwd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++locale.h \ + /usr/include/c++/4.6/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ + /usr/include/c++/4.6/cctype /usr/include/ctype.h \ + /usr/include/c++/4.6/bits/ostream_insert.h \ + /usr/include/c++/4.6/bits/cxxabi_forced.h \ + /usr/include/c++/4.6/bits/stl_function.h \ + /usr/include/c++/4.6/backward/binders.h \ + /usr/include/c++/4.6/bits/range_access.h \ + /usr/include/c++/4.6/bits/basic_string.h \ + /usr/include/c++/4.6/ext/atomicity.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/bits/sched.h \ + /usr/include/bits/timex.h /usr/include/bits/setjmp.h \ + /usr/include/unistd.h /usr/include/bits/posix_opt.h \ + /usr/include/bits/environments.h /usr/include/bits/confname.h \ + /usr/include/getopt.h /usr/include/bits/unistd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/atomic_word.h \ + /usr/include/c++/4.6/initializer_list \ + /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/iostream \ + /usr/include/c++/4.6/ostream /usr/include/c++/4.6/ios \ + /usr/include/c++/4.6/bits/ios_base.h \ + /usr/include/c++/4.6/bits/locale_classes.h \ + /usr/include/c++/4.6/bits/locale_classes.tcc \ + /usr/include/c++/4.6/streambuf /usr/include/c++/4.6/bits/streambuf.tcc \ + /usr/include/c++/4.6/bits/basic_ios.h \ + /usr/include/c++/4.6/bits/locale_facets.h /usr/include/c++/4.6/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_base.h \ + /usr/include/c++/4.6/bits/streambuf_iterator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_inline.h \ + /usr/include/c++/4.6/bits/locale_facets.tcc \ + /usr/include/c++/4.6/bits/basic_ios.tcc \ + /usr/include/c++/4.6/bits/ostream.tcc /usr/include/c++/4.6/istream \ + /usr/include/c++/4.6/bits/istream.tcc ../KaldiLib/cblas.h \ + ../KaldiLib/clapack.h ../KaldiLib/cblas.h ../KaldiLib/Common.h \ + /usr/include/string.h /usr/include/bits/string3.h \ + /usr/include/c++/4.6/sstream /usr/include/c++/4.6/bits/sstream.tcc \ + ../KaldiLib/MathAux.h /usr/include/c++/4.6/cmath /usr/include/math.h \ + /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ + /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ + /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ + /usr/include/bits/mathcalls.h /usr/include/bits/mathinline.h \ + ../KaldiLib/Types.h ../KaldiLib/Error.h /usr/include/execinfo.h \ + ../KaldiLib/Vector.tcc /usr/include/c++/4.6/cstring \ + /usr/include/c++/4.6/fstream /usr/include/c++/4.6/bits/codecvt.h \ + /usr/include/c++/4.6/cstdio /usr/include/libio.h \ + /usr/include/_G_config.h /usr/include/bits/stdio_lim.h \ + /usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \ + /usr/include/bits/stdio2.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/basic_file.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++io.h \ + /usr/include/c++/4.6/bits/fstream.tcc /usr/include/c++/4.6/iomanip \ + ../KaldiLib/Matrix.h ../KaldiLib/Matrix.tcc /usr/include/c++/4.6/cfloat \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/float.h \ + /usr/include/c++/4.6/typeinfo /usr/include/c++/4.6/algorithm \ + /usr/include/c++/4.6/utility /usr/include/c++/4.6/bits/stl_relops.h \ + /usr/include/c++/4.6/bits/stl_algo.h \ + /usr/include/c++/4.6/bits/algorithmfwd.h \ + /usr/include/c++/4.6/bits/stl_heap.h \ + /usr/include/c++/4.6/bits/stl_tempbuf.h \ + /usr/include/c++/4.6/bits/stl_construct.h /usr/include/c++/4.6/limits \ + /usr/include/c++/4.6/vector \ + /usr/include/c++/4.6/bits/stl_uninitialized.h \ + /usr/include/c++/4.6/bits/stl_vector.h \ + /usr/include/c++/4.6/bits/stl_bvector.h \ + /usr/include/c++/4.6/bits/vector.tcc ../KaldiLib/Vector.h \ + ../KaldiLib/Matrix.h ../KaldiLib/Error.h ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cukernels.h /usr/local/cuda-5.0/include/vector_types.h \ + /usr/local/cuda-5.0/include/builtin_types.h \ + /usr/local/cuda-5.0/include/device_types.h \ + /usr/local/cuda-5.0/include/host_defines.h \ + /usr/local/cuda-5.0/include/driver_types.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/limits.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/syslimits.h \ + /usr/include/limits.h /usr/include/bits/posix1_lim.h \ + /usr/include/bits/local_lim.h /usr/include/linux/limits.h \ + /usr/include/bits/posix2_lim.h /usr/include/bits/xopen_lim.h \ + /usr/local/cuda-5.0/include/surface_types.h \ + /usr/local/cuda-5.0/include/texture_types.h \ + /usr/local/cuda-5.0/include/vector_types.h ../CuBaseLib/cumatrix.tcc \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/cuda_device_runtime_api.h \ + /usr/local/cuda-5.0/include/cublas.h \ + /usr/local/cuda-5.0/include/cuda_runtime.h \ + /usr/local/cuda-5.0/include/host_config.h \ + /usr/local/cuda-5.0/include/channel_descriptor.h \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/driver_functions.h \ + /usr/local/cuda-5.0/include/vector_functions.h \ + /usr/local/cuda-5.0/include/cublas_api.h \ + /usr/local/cuda-5.0/include/cuComplex.h ../KaldiLib/Timer.h \ + /usr/include/sys/time.h ../CuBaseLib/cucommon.h ../CuBaseLib/cuvector.h \ + ../CuBaseLib/cuvector.tcc ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cudevice.h /usr/include/c++/4.6/map \ + /usr/include/c++/4.6/bits/stl_tree.h /usr/include/c++/4.6/bits/stl_map.h \ + /usr/include/c++/4.6/bits/stl_multimap.h +cuBlockArray.o: cuBlockArray.cc cuBlockArray.h cuComponent.h \ + ../KaldiLib/Vector.h /usr/include/c++/4.6/cstddef \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++config.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/os_defines.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-32.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/cpu_defines.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stddef.h \ + /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ + /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ + /usr/include/endian.h /usr/include/bits/endian.h \ + /usr/include/bits/byteswap.h /usr/include/xlocale.h \ + /usr/include/sys/types.h /usr/include/bits/types.h \ + /usr/include/bits/typesizes.h /usr/include/time.h \ + /usr/include/sys/select.h /usr/include/bits/select.h \ + /usr/include/bits/sigset.h /usr/include/bits/time.h \ + /usr/include/bits/select2.h /usr/include/sys/sysmacros.h \ + /usr/include/bits/pthreadtypes.h /usr/include/alloca.h \ + /usr/include/bits/stdlib.h /usr/include/c++/4.6/stdexcept \ + /usr/include/c++/4.6/exception /usr/include/c++/4.6/string \ + /usr/include/c++/4.6/bits/stringfwd.h \ + /usr/include/c++/4.6/bits/char_traits.h \ + /usr/include/c++/4.6/bits/stl_algobase.h \ + /usr/include/c++/4.6/bits/functexcept.h \ + /usr/include/c++/4.6/bits/exception_defines.h \ + /usr/include/c++/4.6/bits/cpp_type_traits.h \ + /usr/include/c++/4.6/ext/type_traits.h \ + /usr/include/c++/4.6/ext/numeric_traits.h \ + /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ + /usr/include/c++/4.6/bits/concept_check.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.6/bits/stl_iterator.h \ + /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/postypes.h \ + /usr/include/c++/4.6/cwchar /usr/include/wchar.h /usr/include/stdio.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stdarg.h \ + /usr/include/bits/wchar.h /usr/include/bits/wchar2.h \ + /usr/include/c++/4.6/bits/allocator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++allocator.h \ + /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ + /usr/include/c++/4.6/bits/localefwd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++locale.h \ + /usr/include/c++/4.6/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ + /usr/include/c++/4.6/cctype /usr/include/ctype.h \ + /usr/include/c++/4.6/bits/ostream_insert.h \ + /usr/include/c++/4.6/bits/cxxabi_forced.h \ + /usr/include/c++/4.6/bits/stl_function.h \ + /usr/include/c++/4.6/backward/binders.h \ + /usr/include/c++/4.6/bits/range_access.h \ + /usr/include/c++/4.6/bits/basic_string.h \ + /usr/include/c++/4.6/ext/atomicity.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/bits/sched.h \ + /usr/include/bits/timex.h /usr/include/bits/setjmp.h \ + /usr/include/unistd.h /usr/include/bits/posix_opt.h \ + /usr/include/bits/environments.h /usr/include/bits/confname.h \ + /usr/include/getopt.h /usr/include/bits/unistd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/atomic_word.h \ + /usr/include/c++/4.6/initializer_list \ + /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/iostream \ + /usr/include/c++/4.6/ostream /usr/include/c++/4.6/ios \ + /usr/include/c++/4.6/bits/ios_base.h \ + /usr/include/c++/4.6/bits/locale_classes.h \ + /usr/include/c++/4.6/bits/locale_classes.tcc \ + /usr/include/c++/4.6/streambuf /usr/include/c++/4.6/bits/streambuf.tcc \ + /usr/include/c++/4.6/bits/basic_ios.h \ + /usr/include/c++/4.6/bits/locale_facets.h /usr/include/c++/4.6/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_base.h \ + /usr/include/c++/4.6/bits/streambuf_iterator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_inline.h \ + /usr/include/c++/4.6/bits/locale_facets.tcc \ + /usr/include/c++/4.6/bits/basic_ios.tcc \ + /usr/include/c++/4.6/bits/ostream.tcc /usr/include/c++/4.6/istream \ + /usr/include/c++/4.6/bits/istream.tcc ../KaldiLib/cblas.h \ + ../KaldiLib/clapack.h ../KaldiLib/cblas.h ../KaldiLib/Common.h \ + /usr/include/string.h /usr/include/bits/string3.h \ + /usr/include/c++/4.6/sstream /usr/include/c++/4.6/bits/sstream.tcc \ + ../KaldiLib/MathAux.h /usr/include/c++/4.6/cmath /usr/include/math.h \ + /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ + /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ + /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ + /usr/include/bits/mathcalls.h /usr/include/bits/mathinline.h \ + ../KaldiLib/Types.h ../KaldiLib/Error.h /usr/include/execinfo.h \ + ../KaldiLib/Vector.tcc /usr/include/c++/4.6/cstring \ + /usr/include/c++/4.6/fstream /usr/include/c++/4.6/bits/codecvt.h \ + /usr/include/c++/4.6/cstdio /usr/include/libio.h \ + /usr/include/_G_config.h /usr/include/bits/stdio_lim.h \ + /usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \ + /usr/include/bits/stdio2.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/basic_file.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++io.h \ + /usr/include/c++/4.6/bits/fstream.tcc /usr/include/c++/4.6/iomanip \ + ../KaldiLib/Matrix.h ../KaldiLib/Matrix.tcc /usr/include/c++/4.6/cfloat \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/float.h \ + /usr/include/c++/4.6/typeinfo /usr/include/c++/4.6/algorithm \ + /usr/include/c++/4.6/utility /usr/include/c++/4.6/bits/stl_relops.h \ + /usr/include/c++/4.6/bits/stl_algo.h \ + /usr/include/c++/4.6/bits/algorithmfwd.h \ + /usr/include/c++/4.6/bits/stl_heap.h \ + /usr/include/c++/4.6/bits/stl_tempbuf.h \ + /usr/include/c++/4.6/bits/stl_construct.h /usr/include/c++/4.6/limits \ + /usr/include/c++/4.6/vector \ + /usr/include/c++/4.6/bits/stl_uninitialized.h \ + /usr/include/c++/4.6/bits/stl_vector.h \ + /usr/include/c++/4.6/bits/stl_bvector.h \ + /usr/include/c++/4.6/bits/vector.tcc ../KaldiLib/Vector.h \ + ../KaldiLib/Matrix.h ../KaldiLib/Error.h ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cukernels.h /usr/local/cuda-5.0/include/vector_types.h \ + /usr/local/cuda-5.0/include/builtin_types.h \ + /usr/local/cuda-5.0/include/device_types.h \ + /usr/local/cuda-5.0/include/host_defines.h \ + /usr/local/cuda-5.0/include/driver_types.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/limits.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/syslimits.h \ + /usr/include/limits.h /usr/include/bits/posix1_lim.h \ + /usr/include/bits/local_lim.h /usr/include/linux/limits.h \ + /usr/include/bits/posix2_lim.h /usr/include/bits/xopen_lim.h \ + /usr/local/cuda-5.0/include/surface_types.h \ + /usr/local/cuda-5.0/include/texture_types.h \ + /usr/local/cuda-5.0/include/vector_types.h ../CuBaseLib/cumatrix.tcc \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/cuda_device_runtime_api.h \ + /usr/local/cuda-5.0/include/cublas.h \ + /usr/local/cuda-5.0/include/cuda_runtime.h \ + /usr/local/cuda-5.0/include/host_config.h \ + /usr/local/cuda-5.0/include/channel_descriptor.h \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/driver_functions.h \ + /usr/local/cuda-5.0/include/vector_functions.h \ + /usr/local/cuda-5.0/include/cublas_api.h \ + /usr/local/cuda-5.0/include/cuComplex.h ../KaldiLib/Timer.h \ + /usr/include/sys/time.h ../CuBaseLib/cucommon.h ../CuBaseLib/cuvector.h \ + ../CuBaseLib/cuvector.tcc ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cudevice.h /usr/include/c++/4.6/map \ + /usr/include/c++/4.6/bits/stl_tree.h /usr/include/c++/4.6/bits/stl_map.h \ + /usr/include/c++/4.6/bits/stl_multimap.h cuNetwork.h cuBiasedLinearity.h \ + cuActivation.h cuCRBEDctFeat.h ../CuBaseLib/cumath.h +cuCache.o: cuCache.cc cuCache.h ../CuBaseLib/cumatrix.h \ + /usr/include/c++/4.6/sstream /usr/include/c++/4.6/istream \ + /usr/include/c++/4.6/ios /usr/include/c++/4.6/iosfwd \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++config.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/os_defines.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-32.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/cpu_defines.h \ + /usr/include/c++/4.6/bits/stringfwd.h \ + /usr/include/c++/4.6/bits/postypes.h /usr/include/c++/4.6/cwchar \ + /usr/include/wchar.h /usr/include/stdio.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stdarg.h \ + /usr/include/bits/wchar.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stddef.h \ + /usr/include/xlocale.h /usr/include/bits/wchar2.h \ + /usr/include/c++/4.6/exception /usr/include/c++/4.6/bits/char_traits.h \ + /usr/include/c++/4.6/bits/stl_algobase.h \ + /usr/include/c++/4.6/bits/functexcept.h \ + /usr/include/c++/4.6/bits/exception_defines.h \ + /usr/include/c++/4.6/bits/cpp_type_traits.h \ + /usr/include/c++/4.6/ext/type_traits.h \ + /usr/include/c++/4.6/ext/numeric_traits.h \ + /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ + /usr/include/c++/4.6/bits/concept_check.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.6/bits/stl_iterator.h \ + /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/localefwd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++locale.h \ + /usr/include/c++/4.6/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.6/cctype \ + /usr/include/ctype.h /usr/include/bits/types.h \ + /usr/include/bits/typesizes.h /usr/include/endian.h \ + /usr/include/bits/endian.h /usr/include/bits/byteswap.h \ + /usr/include/c++/4.6/bits/ios_base.h \ + /usr/include/c++/4.6/ext/atomicity.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/time.h \ + /usr/include/bits/sched.h /usr/include/bits/time.h \ + /usr/include/bits/timex.h /usr/include/bits/pthreadtypes.h \ + /usr/include/bits/setjmp.h /usr/include/unistd.h \ + /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ + /usr/include/bits/confname.h /usr/include/getopt.h \ + /usr/include/bits/unistd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/atomic_word.h \ + /usr/include/c++/4.6/bits/locale_classes.h /usr/include/c++/4.6/string \ + /usr/include/c++/4.6/bits/allocator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++allocator.h \ + /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ + /usr/include/c++/4.6/bits/ostream_insert.h \ + /usr/include/c++/4.6/bits/cxxabi_forced.h \ + /usr/include/c++/4.6/bits/stl_function.h \ + /usr/include/c++/4.6/backward/binders.h \ + /usr/include/c++/4.6/bits/range_access.h \ + /usr/include/c++/4.6/bits/basic_string.h \ + /usr/include/c++/4.6/initializer_list \ + /usr/include/c++/4.6/bits/basic_string.tcc \ + /usr/include/c++/4.6/bits/locale_classes.tcc \ + /usr/include/c++/4.6/streambuf /usr/include/c++/4.6/bits/streambuf.tcc \ + /usr/include/c++/4.6/bits/basic_ios.h \ + /usr/include/c++/4.6/bits/locale_facets.h /usr/include/c++/4.6/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_base.h \ + /usr/include/c++/4.6/bits/streambuf_iterator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_inline.h \ + /usr/include/c++/4.6/bits/locale_facets.tcc \ + /usr/include/c++/4.6/bits/basic_ios.tcc /usr/include/c++/4.6/ostream \ + /usr/include/c++/4.6/bits/ostream.tcc \ + /usr/include/c++/4.6/bits/istream.tcc \ + /usr/include/c++/4.6/bits/sstream.tcc ../KaldiLib/Matrix.h \ + /usr/include/stdlib.h /usr/include/bits/waitflags.h \ + /usr/include/bits/waitstatus.h /usr/include/sys/types.h \ + /usr/include/sys/select.h /usr/include/bits/select.h \ + /usr/include/bits/sigset.h /usr/include/bits/select2.h \ + /usr/include/sys/sysmacros.h /usr/include/alloca.h \ + /usr/include/bits/stdlib.h /usr/include/c++/4.6/stdexcept \ + /usr/include/c++/4.6/iostream ../KaldiLib/cblas.h ../KaldiLib/clapack.h \ + ../KaldiLib/cblas.h ../KaldiLib/Common.h /usr/include/c++/4.6/cstdlib \ + /usr/include/string.h /usr/include/bits/string3.h ../KaldiLib/MathAux.h \ + /usr/include/c++/4.6/cmath /usr/include/math.h \ + /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ + /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ + /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ + /usr/include/bits/mathcalls.h /usr/include/bits/mathinline.h \ + ../KaldiLib/Types.h ../KaldiLib/Error.h /usr/include/execinfo.h \ + ../KaldiLib/Matrix.tcc /usr/include/c++/4.6/cfloat \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/float.h \ + /usr/include/c++/4.6/fstream /usr/include/c++/4.6/bits/codecvt.h \ + /usr/include/c++/4.6/cstdio /usr/include/libio.h \ + /usr/include/_G_config.h /usr/include/bits/stdio_lim.h \ + /usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \ + /usr/include/bits/stdio2.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/basic_file.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++io.h \ + /usr/include/c++/4.6/bits/fstream.tcc /usr/include/c++/4.6/iomanip \ + /usr/include/c++/4.6/typeinfo /usr/include/c++/4.6/algorithm \ + /usr/include/c++/4.6/utility /usr/include/c++/4.6/bits/stl_relops.h \ + /usr/include/c++/4.6/bits/stl_algo.h \ + /usr/include/c++/4.6/bits/algorithmfwd.h \ + /usr/include/c++/4.6/bits/stl_heap.h \ + /usr/include/c++/4.6/bits/stl_tempbuf.h \ + /usr/include/c++/4.6/bits/stl_construct.h /usr/include/c++/4.6/limits \ + /usr/include/c++/4.6/vector \ + /usr/include/c++/4.6/bits/stl_uninitialized.h \ + /usr/include/c++/4.6/bits/stl_vector.h \ + /usr/include/c++/4.6/bits/stl_bvector.h \ + /usr/include/c++/4.6/bits/vector.tcc ../KaldiLib/Vector.h \ + /usr/include/c++/4.6/cstddef ../KaldiLib/Vector.tcc \ + /usr/include/c++/4.6/cstring ../KaldiLib/Matrix.h \ + ../CuBaseLib/cukernels.h /usr/local/cuda-5.0/include/vector_types.h \ + /usr/local/cuda-5.0/include/builtin_types.h \ + /usr/local/cuda-5.0/include/device_types.h \ + /usr/local/cuda-5.0/include/host_defines.h \ + /usr/local/cuda-5.0/include/driver_types.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/limits.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/syslimits.h \ + /usr/include/limits.h /usr/include/bits/posix1_lim.h \ + /usr/include/bits/local_lim.h /usr/include/linux/limits.h \ + /usr/include/bits/posix2_lim.h /usr/include/bits/xopen_lim.h \ + /usr/local/cuda-5.0/include/surface_types.h \ + /usr/local/cuda-5.0/include/texture_types.h \ + /usr/local/cuda-5.0/include/vector_types.h ../CuBaseLib/cumatrix.tcc \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/cuda_device_runtime_api.h \ + /usr/local/cuda-5.0/include/cublas.h \ + /usr/local/cuda-5.0/include/cuda_runtime.h \ + /usr/local/cuda-5.0/include/host_config.h \ + /usr/local/cuda-5.0/include/channel_descriptor.h \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/driver_functions.h \ + /usr/local/cuda-5.0/include/vector_functions.h \ + /usr/local/cuda-5.0/include/cublas_api.h \ + /usr/local/cuda-5.0/include/cuComplex.h ../KaldiLib/Timer.h \ + /usr/include/sys/time.h ../CuBaseLib/cucommon.h ../KaldiLib/Error.h \ + ../CuBaseLib/cuvector.h ../KaldiLib/Vector.h ../CuBaseLib/cuvector.tcc \ + ../CuBaseLib/cumatrix.h ../CuBaseLib/cudevice.h /usr/include/c++/4.6/map \ + /usr/include/c++/4.6/bits/stl_tree.h /usr/include/c++/4.6/bits/stl_map.h \ + /usr/include/c++/4.6/bits/stl_multimap.h ../CuBaseLib/cumath.h +cuCompDisc.o: cuCompDisc.cc cuCompDisc.h cuComponent.h \ + ../KaldiLib/Vector.h /usr/include/c++/4.6/cstddef \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++config.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/os_defines.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-32.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/cpu_defines.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stddef.h \ + /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ + /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ + /usr/include/endian.h /usr/include/bits/endian.h \ + /usr/include/bits/byteswap.h /usr/include/xlocale.h \ + /usr/include/sys/types.h /usr/include/bits/types.h \ + /usr/include/bits/typesizes.h /usr/include/time.h \ + /usr/include/sys/select.h /usr/include/bits/select.h \ + /usr/include/bits/sigset.h /usr/include/bits/time.h \ + /usr/include/bits/select2.h /usr/include/sys/sysmacros.h \ + /usr/include/bits/pthreadtypes.h /usr/include/alloca.h \ + /usr/include/bits/stdlib.h /usr/include/c++/4.6/stdexcept \ + /usr/include/c++/4.6/exception /usr/include/c++/4.6/string \ + /usr/include/c++/4.6/bits/stringfwd.h \ + /usr/include/c++/4.6/bits/char_traits.h \ + /usr/include/c++/4.6/bits/stl_algobase.h \ + /usr/include/c++/4.6/bits/functexcept.h \ + /usr/include/c++/4.6/bits/exception_defines.h \ + /usr/include/c++/4.6/bits/cpp_type_traits.h \ + /usr/include/c++/4.6/ext/type_traits.h \ + /usr/include/c++/4.6/ext/numeric_traits.h \ + /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ + /usr/include/c++/4.6/bits/concept_check.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.6/bits/stl_iterator.h \ + /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/postypes.h \ + /usr/include/c++/4.6/cwchar /usr/include/wchar.h /usr/include/stdio.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stdarg.h \ + /usr/include/bits/wchar.h /usr/include/bits/wchar2.h \ + /usr/include/c++/4.6/bits/allocator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++allocator.h \ + /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ + /usr/include/c++/4.6/bits/localefwd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++locale.h \ + /usr/include/c++/4.6/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ + /usr/include/c++/4.6/cctype /usr/include/ctype.h \ + /usr/include/c++/4.6/bits/ostream_insert.h \ + /usr/include/c++/4.6/bits/cxxabi_forced.h \ + /usr/include/c++/4.6/bits/stl_function.h \ + /usr/include/c++/4.6/backward/binders.h \ + /usr/include/c++/4.6/bits/range_access.h \ + /usr/include/c++/4.6/bits/basic_string.h \ + /usr/include/c++/4.6/ext/atomicity.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/bits/sched.h \ + /usr/include/bits/timex.h /usr/include/bits/setjmp.h \ + /usr/include/unistd.h /usr/include/bits/posix_opt.h \ + /usr/include/bits/environments.h /usr/include/bits/confname.h \ + /usr/include/getopt.h /usr/include/bits/unistd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/atomic_word.h \ + /usr/include/c++/4.6/initializer_list \ + /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/iostream \ + /usr/include/c++/4.6/ostream /usr/include/c++/4.6/ios \ + /usr/include/c++/4.6/bits/ios_base.h \ + /usr/include/c++/4.6/bits/locale_classes.h \ + /usr/include/c++/4.6/bits/locale_classes.tcc \ + /usr/include/c++/4.6/streambuf /usr/include/c++/4.6/bits/streambuf.tcc \ + /usr/include/c++/4.6/bits/basic_ios.h \ + /usr/include/c++/4.6/bits/locale_facets.h /usr/include/c++/4.6/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_base.h \ + /usr/include/c++/4.6/bits/streambuf_iterator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_inline.h \ + /usr/include/c++/4.6/bits/locale_facets.tcc \ + /usr/include/c++/4.6/bits/basic_ios.tcc \ + /usr/include/c++/4.6/bits/ostream.tcc /usr/include/c++/4.6/istream \ + /usr/include/c++/4.6/bits/istream.tcc ../KaldiLib/cblas.h \ + ../KaldiLib/clapack.h ../KaldiLib/cblas.h ../KaldiLib/Common.h \ + /usr/include/string.h /usr/include/bits/string3.h \ + /usr/include/c++/4.6/sstream /usr/include/c++/4.6/bits/sstream.tcc \ + ../KaldiLib/MathAux.h /usr/include/c++/4.6/cmath /usr/include/math.h \ + /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ + /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ + /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ + /usr/include/bits/mathcalls.h /usr/include/bits/mathinline.h \ + ../KaldiLib/Types.h ../KaldiLib/Error.h /usr/include/execinfo.h \ + ../KaldiLib/Vector.tcc /usr/include/c++/4.6/cstring \ + /usr/include/c++/4.6/fstream /usr/include/c++/4.6/bits/codecvt.h \ + /usr/include/c++/4.6/cstdio /usr/include/libio.h \ + /usr/include/_G_config.h /usr/include/bits/stdio_lim.h \ + /usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \ + /usr/include/bits/stdio2.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/basic_file.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++io.h \ + /usr/include/c++/4.6/bits/fstream.tcc /usr/include/c++/4.6/iomanip \ + ../KaldiLib/Matrix.h ../KaldiLib/Matrix.tcc /usr/include/c++/4.6/cfloat \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/float.h \ + /usr/include/c++/4.6/typeinfo /usr/include/c++/4.6/algorithm \ + /usr/include/c++/4.6/utility /usr/include/c++/4.6/bits/stl_relops.h \ + /usr/include/c++/4.6/bits/stl_algo.h \ + /usr/include/c++/4.6/bits/algorithmfwd.h \ + /usr/include/c++/4.6/bits/stl_heap.h \ + /usr/include/c++/4.6/bits/stl_tempbuf.h \ + /usr/include/c++/4.6/bits/stl_construct.h /usr/include/c++/4.6/limits \ + /usr/include/c++/4.6/vector \ + /usr/include/c++/4.6/bits/stl_uninitialized.h \ + /usr/include/c++/4.6/bits/stl_vector.h \ + /usr/include/c++/4.6/bits/stl_bvector.h \ + /usr/include/c++/4.6/bits/vector.tcc ../KaldiLib/Vector.h \ + ../KaldiLib/Matrix.h ../KaldiLib/Error.h ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cukernels.h /usr/local/cuda-5.0/include/vector_types.h \ + /usr/local/cuda-5.0/include/builtin_types.h \ + /usr/local/cuda-5.0/include/device_types.h \ + /usr/local/cuda-5.0/include/host_defines.h \ + /usr/local/cuda-5.0/include/driver_types.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/limits.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/syslimits.h \ + /usr/include/limits.h /usr/include/bits/posix1_lim.h \ + /usr/include/bits/local_lim.h /usr/include/linux/limits.h \ + /usr/include/bits/posix2_lim.h /usr/include/bits/xopen_lim.h \ + /usr/local/cuda-5.0/include/surface_types.h \ + /usr/local/cuda-5.0/include/texture_types.h \ + /usr/local/cuda-5.0/include/vector_types.h ../CuBaseLib/cumatrix.tcc \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/cuda_device_runtime_api.h \ + /usr/local/cuda-5.0/include/cublas.h \ + /usr/local/cuda-5.0/include/cuda_runtime.h \ + /usr/local/cuda-5.0/include/host_config.h \ + /usr/local/cuda-5.0/include/channel_descriptor.h \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/driver_functions.h \ + /usr/local/cuda-5.0/include/vector_functions.h \ + /usr/local/cuda-5.0/include/cublas_api.h \ + /usr/local/cuda-5.0/include/cuComplex.h ../KaldiLib/Timer.h \ + /usr/include/sys/time.h ../CuBaseLib/cucommon.h ../CuBaseLib/cuvector.h \ + ../CuBaseLib/cuvector.tcc ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cudevice.h /usr/include/c++/4.6/map \ + /usr/include/c++/4.6/bits/stl_tree.h /usr/include/c++/4.6/bits/stl_map.h \ + /usr/include/c++/4.6/bits/stl_multimap.h cuNetwork.h cuBiasedLinearity.h \ + cuActivation.h cuCRBEDctFeat.h ../CuBaseLib/cumath.h +cuConcatenate.o: cuConcatenate.cc cuConcatenate.h cuComponent.h \ + ../KaldiLib/Vector.h /usr/include/c++/4.6/cstddef \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++config.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/os_defines.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-32.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/cpu_defines.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stddef.h \ + /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ + /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ + /usr/include/endian.h /usr/include/bits/endian.h \ + /usr/include/bits/byteswap.h /usr/include/xlocale.h \ + /usr/include/sys/types.h /usr/include/bits/types.h \ + /usr/include/bits/typesizes.h /usr/include/time.h \ + /usr/include/sys/select.h /usr/include/bits/select.h \ + /usr/include/bits/sigset.h /usr/include/bits/time.h \ + /usr/include/bits/select2.h /usr/include/sys/sysmacros.h \ + /usr/include/bits/pthreadtypes.h /usr/include/alloca.h \ + /usr/include/bits/stdlib.h /usr/include/c++/4.6/stdexcept \ + /usr/include/c++/4.6/exception /usr/include/c++/4.6/string \ + /usr/include/c++/4.6/bits/stringfwd.h \ + /usr/include/c++/4.6/bits/char_traits.h \ + /usr/include/c++/4.6/bits/stl_algobase.h \ + /usr/include/c++/4.6/bits/functexcept.h \ + /usr/include/c++/4.6/bits/exception_defines.h \ + /usr/include/c++/4.6/bits/cpp_type_traits.h \ + /usr/include/c++/4.6/ext/type_traits.h \ + /usr/include/c++/4.6/ext/numeric_traits.h \ + /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ + /usr/include/c++/4.6/bits/concept_check.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.6/bits/stl_iterator.h \ + /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/postypes.h \ + /usr/include/c++/4.6/cwchar /usr/include/wchar.h /usr/include/stdio.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stdarg.h \ + /usr/include/bits/wchar.h /usr/include/bits/wchar2.h \ + /usr/include/c++/4.6/bits/allocator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++allocator.h \ + /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ + /usr/include/c++/4.6/bits/localefwd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++locale.h \ + /usr/include/c++/4.6/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ + /usr/include/c++/4.6/cctype /usr/include/ctype.h \ + /usr/include/c++/4.6/bits/ostream_insert.h \ + /usr/include/c++/4.6/bits/cxxabi_forced.h \ + /usr/include/c++/4.6/bits/stl_function.h \ + /usr/include/c++/4.6/backward/binders.h \ + /usr/include/c++/4.6/bits/range_access.h \ + /usr/include/c++/4.6/bits/basic_string.h \ + /usr/include/c++/4.6/ext/atomicity.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/bits/sched.h \ + /usr/include/bits/timex.h /usr/include/bits/setjmp.h \ + /usr/include/unistd.h /usr/include/bits/posix_opt.h \ + /usr/include/bits/environments.h /usr/include/bits/confname.h \ + /usr/include/getopt.h /usr/include/bits/unistd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/atomic_word.h \ + /usr/include/c++/4.6/initializer_list \ + /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/iostream \ + /usr/include/c++/4.6/ostream /usr/include/c++/4.6/ios \ + /usr/include/c++/4.6/bits/ios_base.h \ + /usr/include/c++/4.6/bits/locale_classes.h \ + /usr/include/c++/4.6/bits/locale_classes.tcc \ + /usr/include/c++/4.6/streambuf /usr/include/c++/4.6/bits/streambuf.tcc \ + /usr/include/c++/4.6/bits/basic_ios.h \ + /usr/include/c++/4.6/bits/locale_facets.h /usr/include/c++/4.6/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_base.h \ + /usr/include/c++/4.6/bits/streambuf_iterator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_inline.h \ + /usr/include/c++/4.6/bits/locale_facets.tcc \ + /usr/include/c++/4.6/bits/basic_ios.tcc \ + /usr/include/c++/4.6/bits/ostream.tcc /usr/include/c++/4.6/istream \ + /usr/include/c++/4.6/bits/istream.tcc ../KaldiLib/cblas.h \ + ../KaldiLib/clapack.h ../KaldiLib/cblas.h ../KaldiLib/Common.h \ + /usr/include/string.h /usr/include/bits/string3.h \ + /usr/include/c++/4.6/sstream /usr/include/c++/4.6/bits/sstream.tcc \ + ../KaldiLib/MathAux.h /usr/include/c++/4.6/cmath /usr/include/math.h \ + /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ + /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ + /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ + /usr/include/bits/mathcalls.h /usr/include/bits/mathinline.h \ + ../KaldiLib/Types.h ../KaldiLib/Error.h /usr/include/execinfo.h \ + ../KaldiLib/Vector.tcc /usr/include/c++/4.6/cstring \ + /usr/include/c++/4.6/fstream /usr/include/c++/4.6/bits/codecvt.h \ + /usr/include/c++/4.6/cstdio /usr/include/libio.h \ + /usr/include/_G_config.h /usr/include/bits/stdio_lim.h \ + /usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \ + /usr/include/bits/stdio2.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/basic_file.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++io.h \ + /usr/include/c++/4.6/bits/fstream.tcc /usr/include/c++/4.6/iomanip \ + ../KaldiLib/Matrix.h ../KaldiLib/Matrix.tcc /usr/include/c++/4.6/cfloat \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/float.h \ + /usr/include/c++/4.6/typeinfo /usr/include/c++/4.6/algorithm \ + /usr/include/c++/4.6/utility /usr/include/c++/4.6/bits/stl_relops.h \ + /usr/include/c++/4.6/bits/stl_algo.h \ + /usr/include/c++/4.6/bits/algorithmfwd.h \ + /usr/include/c++/4.6/bits/stl_heap.h \ + /usr/include/c++/4.6/bits/stl_tempbuf.h \ + /usr/include/c++/4.6/bits/stl_construct.h /usr/include/c++/4.6/limits \ + /usr/include/c++/4.6/vector \ + /usr/include/c++/4.6/bits/stl_uninitialized.h \ + /usr/include/c++/4.6/bits/stl_vector.h \ + /usr/include/c++/4.6/bits/stl_bvector.h \ + /usr/include/c++/4.6/bits/vector.tcc ../KaldiLib/Vector.h \ + ../KaldiLib/Matrix.h ../KaldiLib/Error.h ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cukernels.h /usr/local/cuda-5.0/include/vector_types.h \ + /usr/local/cuda-5.0/include/builtin_types.h \ + /usr/local/cuda-5.0/include/device_types.h \ + /usr/local/cuda-5.0/include/host_defines.h \ + /usr/local/cuda-5.0/include/driver_types.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/limits.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/syslimits.h \ + /usr/include/limits.h /usr/include/bits/posix1_lim.h \ + /usr/include/bits/local_lim.h /usr/include/linux/limits.h \ + /usr/include/bits/posix2_lim.h /usr/include/bits/xopen_lim.h \ + /usr/local/cuda-5.0/include/surface_types.h \ + /usr/local/cuda-5.0/include/texture_types.h \ + /usr/local/cuda-5.0/include/vector_types.h ../CuBaseLib/cumatrix.tcc \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/cuda_device_runtime_api.h \ + /usr/local/cuda-5.0/include/cublas.h \ + /usr/local/cuda-5.0/include/cuda_runtime.h \ + /usr/local/cuda-5.0/include/host_config.h \ + /usr/local/cuda-5.0/include/channel_descriptor.h \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/driver_functions.h \ + /usr/local/cuda-5.0/include/vector_functions.h \ + /usr/local/cuda-5.0/include/cublas_api.h \ + /usr/local/cuda-5.0/include/cuComplex.h ../KaldiLib/Timer.h \ + /usr/include/sys/time.h ../CuBaseLib/cucommon.h ../CuBaseLib/cuvector.h \ + ../CuBaseLib/cuvector.tcc ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cudevice.h /usr/include/c++/4.6/map \ + /usr/include/c++/4.6/bits/stl_tree.h /usr/include/c++/4.6/bits/stl_map.h \ + /usr/include/c++/4.6/bits/stl_multimap.h cuNetwork.h cuBiasedLinearity.h \ + cuActivation.h cuCRBEDctFeat.h ../CuBaseLib/cumath.h +cuDiscreteLinearity.o: cuDiscreteLinearity.cc cuDiscreteLinearity.h \ + cuComponent.h ../KaldiLib/Vector.h /usr/include/c++/4.6/cstddef \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++config.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/os_defines.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-32.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/cpu_defines.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stddef.h \ + /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ + /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ + /usr/include/endian.h /usr/include/bits/endian.h \ + /usr/include/bits/byteswap.h /usr/include/xlocale.h \ + /usr/include/sys/types.h /usr/include/bits/types.h \ + /usr/include/bits/typesizes.h /usr/include/time.h \ + /usr/include/sys/select.h /usr/include/bits/select.h \ + /usr/include/bits/sigset.h /usr/include/bits/time.h \ + /usr/include/bits/select2.h /usr/include/sys/sysmacros.h \ + /usr/include/bits/pthreadtypes.h /usr/include/alloca.h \ + /usr/include/bits/stdlib.h /usr/include/c++/4.6/stdexcept \ + /usr/include/c++/4.6/exception /usr/include/c++/4.6/string \ + /usr/include/c++/4.6/bits/stringfwd.h \ + /usr/include/c++/4.6/bits/char_traits.h \ + /usr/include/c++/4.6/bits/stl_algobase.h \ + /usr/include/c++/4.6/bits/functexcept.h \ + /usr/include/c++/4.6/bits/exception_defines.h \ + /usr/include/c++/4.6/bits/cpp_type_traits.h \ + /usr/include/c++/4.6/ext/type_traits.h \ + /usr/include/c++/4.6/ext/numeric_traits.h \ + /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ + /usr/include/c++/4.6/bits/concept_check.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.6/bits/stl_iterator.h \ + /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/postypes.h \ + /usr/include/c++/4.6/cwchar /usr/include/wchar.h /usr/include/stdio.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stdarg.h \ + /usr/include/bits/wchar.h /usr/include/bits/wchar2.h \ + /usr/include/c++/4.6/bits/allocator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++allocator.h \ + /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ + /usr/include/c++/4.6/bits/localefwd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++locale.h \ + /usr/include/c++/4.6/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ + /usr/include/c++/4.6/cctype /usr/include/ctype.h \ + /usr/include/c++/4.6/bits/ostream_insert.h \ + /usr/include/c++/4.6/bits/cxxabi_forced.h \ + /usr/include/c++/4.6/bits/stl_function.h \ + /usr/include/c++/4.6/backward/binders.h \ + /usr/include/c++/4.6/bits/range_access.h \ + /usr/include/c++/4.6/bits/basic_string.h \ + /usr/include/c++/4.6/ext/atomicity.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/bits/sched.h \ + /usr/include/bits/timex.h /usr/include/bits/setjmp.h \ + /usr/include/unistd.h /usr/include/bits/posix_opt.h \ + /usr/include/bits/environments.h /usr/include/bits/confname.h \ + /usr/include/getopt.h /usr/include/bits/unistd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/atomic_word.h \ + /usr/include/c++/4.6/initializer_list \ + /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/iostream \ + /usr/include/c++/4.6/ostream /usr/include/c++/4.6/ios \ + /usr/include/c++/4.6/bits/ios_base.h \ + /usr/include/c++/4.6/bits/locale_classes.h \ + /usr/include/c++/4.6/bits/locale_classes.tcc \ + /usr/include/c++/4.6/streambuf /usr/include/c++/4.6/bits/streambuf.tcc \ + /usr/include/c++/4.6/bits/basic_ios.h \ + /usr/include/c++/4.6/bits/locale_facets.h /usr/include/c++/4.6/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_base.h \ + /usr/include/c++/4.6/bits/streambuf_iterator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_inline.h \ + /usr/include/c++/4.6/bits/locale_facets.tcc \ + /usr/include/c++/4.6/bits/basic_ios.tcc \ + /usr/include/c++/4.6/bits/ostream.tcc /usr/include/c++/4.6/istream \ + /usr/include/c++/4.6/bits/istream.tcc ../KaldiLib/cblas.h \ + ../KaldiLib/clapack.h ../KaldiLib/cblas.h ../KaldiLib/Common.h \ + /usr/include/string.h /usr/include/bits/string3.h \ + /usr/include/c++/4.6/sstream /usr/include/c++/4.6/bits/sstream.tcc \ + ../KaldiLib/MathAux.h /usr/include/c++/4.6/cmath /usr/include/math.h \ + /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ + /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ + /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ + /usr/include/bits/mathcalls.h /usr/include/bits/mathinline.h \ + ../KaldiLib/Types.h ../KaldiLib/Error.h /usr/include/execinfo.h \ + ../KaldiLib/Vector.tcc /usr/include/c++/4.6/cstring \ + /usr/include/c++/4.6/fstream /usr/include/c++/4.6/bits/codecvt.h \ + /usr/include/c++/4.6/cstdio /usr/include/libio.h \ + /usr/include/_G_config.h /usr/include/bits/stdio_lim.h \ + /usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \ + /usr/include/bits/stdio2.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/basic_file.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++io.h \ + /usr/include/c++/4.6/bits/fstream.tcc /usr/include/c++/4.6/iomanip \ + ../KaldiLib/Matrix.h ../KaldiLib/Matrix.tcc /usr/include/c++/4.6/cfloat \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/float.h \ + /usr/include/c++/4.6/typeinfo /usr/include/c++/4.6/algorithm \ + /usr/include/c++/4.6/utility /usr/include/c++/4.6/bits/stl_relops.h \ + /usr/include/c++/4.6/bits/stl_algo.h \ + /usr/include/c++/4.6/bits/algorithmfwd.h \ + /usr/include/c++/4.6/bits/stl_heap.h \ + /usr/include/c++/4.6/bits/stl_tempbuf.h \ + /usr/include/c++/4.6/bits/stl_construct.h /usr/include/c++/4.6/limits \ + /usr/include/c++/4.6/vector \ + /usr/include/c++/4.6/bits/stl_uninitialized.h \ + /usr/include/c++/4.6/bits/stl_vector.h \ + /usr/include/c++/4.6/bits/stl_bvector.h \ + /usr/include/c++/4.6/bits/vector.tcc ../KaldiLib/Vector.h \ + ../KaldiLib/Matrix.h ../KaldiLib/Error.h ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cukernels.h /usr/local/cuda-5.0/include/vector_types.h \ + /usr/local/cuda-5.0/include/builtin_types.h \ + /usr/local/cuda-5.0/include/device_types.h \ + /usr/local/cuda-5.0/include/host_defines.h \ + /usr/local/cuda-5.0/include/driver_types.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/limits.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/syslimits.h \ + /usr/include/limits.h /usr/include/bits/posix1_lim.h \ + /usr/include/bits/local_lim.h /usr/include/linux/limits.h \ + /usr/include/bits/posix2_lim.h /usr/include/bits/xopen_lim.h \ + /usr/local/cuda-5.0/include/surface_types.h \ + /usr/local/cuda-5.0/include/texture_types.h \ + /usr/local/cuda-5.0/include/vector_types.h ../CuBaseLib/cumatrix.tcc \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/cuda_device_runtime_api.h \ + /usr/local/cuda-5.0/include/cublas.h \ + /usr/local/cuda-5.0/include/cuda_runtime.h \ + /usr/local/cuda-5.0/include/host_config.h \ + /usr/local/cuda-5.0/include/channel_descriptor.h \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/driver_functions.h \ + /usr/local/cuda-5.0/include/vector_functions.h \ + /usr/local/cuda-5.0/include/cublas_api.h \ + /usr/local/cuda-5.0/include/cuComplex.h ../KaldiLib/Timer.h \ + /usr/include/sys/time.h ../CuBaseLib/cucommon.h ../CuBaseLib/cuvector.h \ + ../CuBaseLib/cuvector.tcc ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cudevice.h /usr/include/c++/4.6/map \ + /usr/include/c++/4.6/bits/stl_tree.h /usr/include/c++/4.6/bits/stl_map.h \ + /usr/include/c++/4.6/bits/stl_multimap.h ../CuBaseLib/cumath.h +cuLinearity.o: cuLinearity.cc cuLinearity.h cuComponent.h \ + ../KaldiLib/Vector.h /usr/include/c++/4.6/cstddef \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++config.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/os_defines.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-32.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/cpu_defines.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stddef.h \ + /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ + /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ + /usr/include/endian.h /usr/include/bits/endian.h \ + /usr/include/bits/byteswap.h /usr/include/xlocale.h \ + /usr/include/sys/types.h /usr/include/bits/types.h \ + /usr/include/bits/typesizes.h /usr/include/time.h \ + /usr/include/sys/select.h /usr/include/bits/select.h \ + /usr/include/bits/sigset.h /usr/include/bits/time.h \ + /usr/include/bits/select2.h /usr/include/sys/sysmacros.h \ + /usr/include/bits/pthreadtypes.h /usr/include/alloca.h \ + /usr/include/bits/stdlib.h /usr/include/c++/4.6/stdexcept \ + /usr/include/c++/4.6/exception /usr/include/c++/4.6/string \ + /usr/include/c++/4.6/bits/stringfwd.h \ + /usr/include/c++/4.6/bits/char_traits.h \ + /usr/include/c++/4.6/bits/stl_algobase.h \ + /usr/include/c++/4.6/bits/functexcept.h \ + /usr/include/c++/4.6/bits/exception_defines.h \ + /usr/include/c++/4.6/bits/cpp_type_traits.h \ + /usr/include/c++/4.6/ext/type_traits.h \ + /usr/include/c++/4.6/ext/numeric_traits.h \ + /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ + /usr/include/c++/4.6/bits/concept_check.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.6/bits/stl_iterator.h \ + /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/postypes.h \ + /usr/include/c++/4.6/cwchar /usr/include/wchar.h /usr/include/stdio.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stdarg.h \ + /usr/include/bits/wchar.h /usr/include/bits/wchar2.h \ + /usr/include/c++/4.6/bits/allocator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++allocator.h \ + /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ + /usr/include/c++/4.6/bits/localefwd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++locale.h \ + /usr/include/c++/4.6/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ + /usr/include/c++/4.6/cctype /usr/include/ctype.h \ + /usr/include/c++/4.6/bits/ostream_insert.h \ + /usr/include/c++/4.6/bits/cxxabi_forced.h \ + /usr/include/c++/4.6/bits/stl_function.h \ + /usr/include/c++/4.6/backward/binders.h \ + /usr/include/c++/4.6/bits/range_access.h \ + /usr/include/c++/4.6/bits/basic_string.h \ + /usr/include/c++/4.6/ext/atomicity.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/bits/sched.h \ + /usr/include/bits/timex.h /usr/include/bits/setjmp.h \ + /usr/include/unistd.h /usr/include/bits/posix_opt.h \ + /usr/include/bits/environments.h /usr/include/bits/confname.h \ + /usr/include/getopt.h /usr/include/bits/unistd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/atomic_word.h \ + /usr/include/c++/4.6/initializer_list \ + /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/iostream \ + /usr/include/c++/4.6/ostream /usr/include/c++/4.6/ios \ + /usr/include/c++/4.6/bits/ios_base.h \ + /usr/include/c++/4.6/bits/locale_classes.h \ + /usr/include/c++/4.6/bits/locale_classes.tcc \ + /usr/include/c++/4.6/streambuf /usr/include/c++/4.6/bits/streambuf.tcc \ + /usr/include/c++/4.6/bits/basic_ios.h \ + /usr/include/c++/4.6/bits/locale_facets.h /usr/include/c++/4.6/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_base.h \ + /usr/include/c++/4.6/bits/streambuf_iterator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_inline.h \ + /usr/include/c++/4.6/bits/locale_facets.tcc \ + /usr/include/c++/4.6/bits/basic_ios.tcc \ + /usr/include/c++/4.6/bits/ostream.tcc /usr/include/c++/4.6/istream \ + /usr/include/c++/4.6/bits/istream.tcc ../KaldiLib/cblas.h \ + ../KaldiLib/clapack.h ../KaldiLib/cblas.h ../KaldiLib/Common.h \ + /usr/include/string.h /usr/include/bits/string3.h \ + /usr/include/c++/4.6/sstream /usr/include/c++/4.6/bits/sstream.tcc \ + ../KaldiLib/MathAux.h /usr/include/c++/4.6/cmath /usr/include/math.h \ + /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ + /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ + /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ + /usr/include/bits/mathcalls.h /usr/include/bits/mathinline.h \ + ../KaldiLib/Types.h ../KaldiLib/Error.h /usr/include/execinfo.h \ + ../KaldiLib/Vector.tcc /usr/include/c++/4.6/cstring \ + /usr/include/c++/4.6/fstream /usr/include/c++/4.6/bits/codecvt.h \ + /usr/include/c++/4.6/cstdio /usr/include/libio.h \ + /usr/include/_G_config.h /usr/include/bits/stdio_lim.h \ + /usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \ + /usr/include/bits/stdio2.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/basic_file.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++io.h \ + /usr/include/c++/4.6/bits/fstream.tcc /usr/include/c++/4.6/iomanip \ + ../KaldiLib/Matrix.h ../KaldiLib/Matrix.tcc /usr/include/c++/4.6/cfloat \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/float.h \ + /usr/include/c++/4.6/typeinfo /usr/include/c++/4.6/algorithm \ + /usr/include/c++/4.6/utility /usr/include/c++/4.6/bits/stl_relops.h \ + /usr/include/c++/4.6/bits/stl_algo.h \ + /usr/include/c++/4.6/bits/algorithmfwd.h \ + /usr/include/c++/4.6/bits/stl_heap.h \ + /usr/include/c++/4.6/bits/stl_tempbuf.h \ + /usr/include/c++/4.6/bits/stl_construct.h /usr/include/c++/4.6/limits \ + /usr/include/c++/4.6/vector \ + /usr/include/c++/4.6/bits/stl_uninitialized.h \ + /usr/include/c++/4.6/bits/stl_vector.h \ + /usr/include/c++/4.6/bits/stl_bvector.h \ + /usr/include/c++/4.6/bits/vector.tcc ../KaldiLib/Vector.h \ + ../KaldiLib/Matrix.h ../KaldiLib/Error.h ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cukernels.h /usr/local/cuda-5.0/include/vector_types.h \ + /usr/local/cuda-5.0/include/builtin_types.h \ + /usr/local/cuda-5.0/include/device_types.h \ + /usr/local/cuda-5.0/include/host_defines.h \ + /usr/local/cuda-5.0/include/driver_types.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/limits.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/syslimits.h \ + /usr/include/limits.h /usr/include/bits/posix1_lim.h \ + /usr/include/bits/local_lim.h /usr/include/linux/limits.h \ + /usr/include/bits/posix2_lim.h /usr/include/bits/xopen_lim.h \ + /usr/local/cuda-5.0/include/surface_types.h \ + /usr/local/cuda-5.0/include/texture_types.h \ + /usr/local/cuda-5.0/include/vector_types.h ../CuBaseLib/cumatrix.tcc \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/cuda_device_runtime_api.h \ + /usr/local/cuda-5.0/include/cublas.h \ + /usr/local/cuda-5.0/include/cuda_runtime.h \ + /usr/local/cuda-5.0/include/host_config.h \ + /usr/local/cuda-5.0/include/channel_descriptor.h \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/driver_functions.h \ + /usr/local/cuda-5.0/include/vector_functions.h \ + /usr/local/cuda-5.0/include/cublas_api.h \ + /usr/local/cuda-5.0/include/cuComplex.h ../KaldiLib/Timer.h \ + /usr/include/sys/time.h ../CuBaseLib/cucommon.h ../CuBaseLib/cuvector.h \ + ../CuBaseLib/cuvector.tcc ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cudevice.h /usr/include/c++/4.6/map \ + /usr/include/c++/4.6/bits/stl_tree.h /usr/include/c++/4.6/bits/stl_map.h \ + /usr/include/c++/4.6/bits/stl_multimap.h +cuNetwork.o: cuNetwork.cc /usr/include/c++/4.6/algorithm \ + /usr/include/c++/4.6/utility \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++config.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/os_defines.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-32.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/cpu_defines.h \ + /usr/include/c++/4.6/bits/stl_relops.h \ + /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ + /usr/include/c++/4.6/bits/concept_check.h \ + /usr/include/c++/4.6/bits/stl_algobase.h \ + /usr/include/c++/4.6/bits/functexcept.h \ + /usr/include/c++/4.6/bits/exception_defines.h \ + /usr/include/c++/4.6/bits/cpp_type_traits.h \ + /usr/include/c++/4.6/ext/type_traits.h \ + /usr/include/c++/4.6/ext/numeric_traits.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.6/bits/stl_iterator.h \ + /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/stl_algo.h \ + /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stddef.h \ + /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ + /usr/include/endian.h /usr/include/bits/endian.h \ + /usr/include/bits/byteswap.h /usr/include/xlocale.h \ + /usr/include/sys/types.h /usr/include/bits/types.h \ + /usr/include/bits/typesizes.h /usr/include/time.h \ + /usr/include/sys/select.h /usr/include/bits/select.h \ + /usr/include/bits/sigset.h /usr/include/bits/time.h \ + /usr/include/bits/select2.h /usr/include/sys/sysmacros.h \ + /usr/include/bits/pthreadtypes.h /usr/include/alloca.h \ + /usr/include/bits/stdlib.h /usr/include/c++/4.6/bits/algorithmfwd.h \ + /usr/include/c++/4.6/initializer_list \ + /usr/include/c++/4.6/bits/stl_heap.h \ + /usr/include/c++/4.6/bits/stl_tempbuf.h \ + /usr/include/c++/4.6/bits/stl_construct.h /usr/include/c++/4.6/new \ + /usr/include/c++/4.6/exception /usr/include/c++/4.6/cctype \ + /usr/include/ctype.h /usr/include/c++/4.6/list \ + /usr/include/c++/4.6/bits/allocator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++allocator.h \ + /usr/include/c++/4.6/ext/new_allocator.h \ + /usr/include/c++/4.6/bits/range_access.h \ + /usr/include/c++/4.6/bits/stl_list.h /usr/include/c++/4.6/bits/list.tcc \ + /usr/include/c++/4.6/sstream /usr/include/c++/4.6/istream \ + /usr/include/c++/4.6/ios /usr/include/c++/4.6/iosfwd \ + /usr/include/c++/4.6/bits/stringfwd.h \ + /usr/include/c++/4.6/bits/postypes.h /usr/include/c++/4.6/cwchar \ + /usr/include/wchar.h /usr/include/stdio.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stdarg.h \ + /usr/include/bits/wchar.h /usr/include/bits/wchar2.h \ + /usr/include/c++/4.6/bits/char_traits.h \ + /usr/include/c++/4.6/bits/localefwd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++locale.h \ + /usr/include/c++/4.6/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.6/bits/ios_base.h \ + /usr/include/c++/4.6/ext/atomicity.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/bits/sched.h \ + /usr/include/bits/timex.h /usr/include/bits/setjmp.h \ + /usr/include/unistd.h /usr/include/bits/posix_opt.h \ + /usr/include/bits/environments.h /usr/include/bits/confname.h \ + /usr/include/getopt.h /usr/include/bits/unistd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/atomic_word.h \ + /usr/include/c++/4.6/bits/locale_classes.h /usr/include/c++/4.6/string \ + /usr/include/c++/4.6/bits/ostream_insert.h \ + /usr/include/c++/4.6/bits/cxxabi_forced.h \ + /usr/include/c++/4.6/bits/stl_function.h \ + /usr/include/c++/4.6/backward/binders.h \ + /usr/include/c++/4.6/bits/basic_string.h \ + /usr/include/c++/4.6/bits/basic_string.tcc \ + /usr/include/c++/4.6/bits/locale_classes.tcc \ + /usr/include/c++/4.6/streambuf /usr/include/c++/4.6/bits/streambuf.tcc \ + /usr/include/c++/4.6/bits/basic_ios.h \ + /usr/include/c++/4.6/bits/locale_facets.h /usr/include/c++/4.6/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_base.h \ + /usr/include/c++/4.6/bits/streambuf_iterator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_inline.h \ + /usr/include/c++/4.6/bits/locale_facets.tcc \ + /usr/include/c++/4.6/bits/basic_ios.tcc /usr/include/c++/4.6/ostream \ + /usr/include/c++/4.6/bits/ostream.tcc \ + /usr/include/c++/4.6/bits/istream.tcc \ + /usr/include/c++/4.6/bits/sstream.tcc cuNetwork.h cuComponent.h \ + ../KaldiLib/Vector.h /usr/include/c++/4.6/cstddef \ + /usr/include/c++/4.6/stdexcept /usr/include/c++/4.6/iostream \ + ../KaldiLib/cblas.h ../KaldiLib/clapack.h ../KaldiLib/cblas.h \ + ../KaldiLib/Common.h /usr/include/string.h /usr/include/bits/string3.h \ + ../KaldiLib/MathAux.h /usr/include/c++/4.6/cmath /usr/include/math.h \ + /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ + /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ + /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ + /usr/include/bits/mathcalls.h /usr/include/bits/mathinline.h \ + ../KaldiLib/Types.h ../KaldiLib/Error.h /usr/include/execinfo.h \ + ../KaldiLib/Vector.tcc /usr/include/c++/4.6/cstring \ + /usr/include/c++/4.6/fstream /usr/include/c++/4.6/bits/codecvt.h \ + /usr/include/c++/4.6/cstdio /usr/include/libio.h \ + /usr/include/_G_config.h /usr/include/bits/stdio_lim.h \ + /usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \ + /usr/include/bits/stdio2.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/basic_file.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++io.h \ + /usr/include/c++/4.6/bits/fstream.tcc /usr/include/c++/4.6/iomanip \ + ../KaldiLib/Matrix.h ../KaldiLib/Matrix.tcc /usr/include/c++/4.6/cfloat \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/float.h \ + /usr/include/c++/4.6/typeinfo /usr/include/c++/4.6/limits \ + /usr/include/c++/4.6/vector \ + /usr/include/c++/4.6/bits/stl_uninitialized.h \ + /usr/include/c++/4.6/bits/stl_vector.h \ + /usr/include/c++/4.6/bits/stl_bvector.h \ + /usr/include/c++/4.6/bits/vector.tcc ../KaldiLib/Vector.h \ + ../KaldiLib/Matrix.h ../KaldiLib/Error.h ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cukernels.h /usr/local/cuda-5.0/include/vector_types.h \ + /usr/local/cuda-5.0/include/builtin_types.h \ + /usr/local/cuda-5.0/include/device_types.h \ + /usr/local/cuda-5.0/include/host_defines.h \ + /usr/local/cuda-5.0/include/driver_types.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/limits.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/syslimits.h \ + /usr/include/limits.h /usr/include/bits/posix1_lim.h \ + /usr/include/bits/local_lim.h /usr/include/linux/limits.h \ + /usr/include/bits/posix2_lim.h /usr/include/bits/xopen_lim.h \ + /usr/local/cuda-5.0/include/surface_types.h \ + /usr/local/cuda-5.0/include/texture_types.h \ + /usr/local/cuda-5.0/include/vector_types.h ../CuBaseLib/cumatrix.tcc \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/cuda_device_runtime_api.h \ + /usr/local/cuda-5.0/include/cublas.h \ + /usr/local/cuda-5.0/include/cuda_runtime.h \ + /usr/local/cuda-5.0/include/host_config.h \ + /usr/local/cuda-5.0/include/channel_descriptor.h \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/driver_functions.h \ + /usr/local/cuda-5.0/include/vector_functions.h \ + /usr/local/cuda-5.0/include/cublas_api.h \ + /usr/local/cuda-5.0/include/cuComplex.h ../KaldiLib/Timer.h \ + /usr/include/sys/time.h ../CuBaseLib/cucommon.h ../CuBaseLib/cuvector.h \ + ../CuBaseLib/cuvector.tcc ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cudevice.h /usr/include/c++/4.6/map \ + /usr/include/c++/4.6/bits/stl_tree.h /usr/include/c++/4.6/bits/stl_map.h \ + /usr/include/c++/4.6/bits/stl_multimap.h cuBiasedLinearity.h \ + cuActivation.h cuCRBEDctFeat.h ../CuBaseLib/cumath.h \ + cuDiscreteLinearity.h cuSharedLinearity.h cuSparseLinearity.h cuRbm.h \ + cuRbmSparse.h cuRecurrent.h cuBlockArray.h cuLinearity.h \ + cuUpdatableBias.h cuMisc.h cuCompDisc.h +cuObjectiveFunction.o: cuObjectiveFunction.cc cuObjectiveFunction.h \ + /usr/include/c++/4.6/cassert /usr/include/assert.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-32.h \ + /usr/include/c++/4.6/limits \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++config.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/os_defines.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/cpu_defines.h \ + /usr/include/c++/4.6/cmath /usr/include/c++/4.6/bits/cpp_type_traits.h \ + /usr/include/c++/4.6/ext/type_traits.h /usr/include/math.h \ + /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ + /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ + /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ + /usr/include/bits/mathcalls.h /usr/include/bits/mathinline.h \ + /usr/include/c++/4.6/sstream /usr/include/c++/4.6/istream \ + /usr/include/c++/4.6/ios /usr/include/c++/4.6/iosfwd \ + /usr/include/c++/4.6/bits/stringfwd.h \ + /usr/include/c++/4.6/bits/postypes.h /usr/include/c++/4.6/cwchar \ + /usr/include/wchar.h /usr/include/stdio.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stdarg.h \ + /usr/include/bits/wchar.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stddef.h \ + /usr/include/xlocale.h /usr/include/bits/wchar2.h \ + /usr/include/c++/4.6/exception /usr/include/c++/4.6/bits/char_traits.h \ + /usr/include/c++/4.6/bits/stl_algobase.h \ + /usr/include/c++/4.6/bits/functexcept.h \ + /usr/include/c++/4.6/bits/exception_defines.h \ + /usr/include/c++/4.6/ext/numeric_traits.h \ + /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ + /usr/include/c++/4.6/bits/concept_check.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.6/bits/stl_iterator.h \ + /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/localefwd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++locale.h \ + /usr/include/c++/4.6/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.6/cctype \ + /usr/include/ctype.h /usr/include/bits/types.h \ + /usr/include/bits/typesizes.h /usr/include/endian.h \ + /usr/include/bits/endian.h /usr/include/bits/byteswap.h \ + /usr/include/c++/4.6/bits/ios_base.h \ + /usr/include/c++/4.6/ext/atomicity.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/time.h \ + /usr/include/bits/sched.h /usr/include/bits/time.h \ + /usr/include/bits/timex.h /usr/include/bits/pthreadtypes.h \ + /usr/include/bits/setjmp.h /usr/include/unistd.h \ + /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ + /usr/include/bits/confname.h /usr/include/getopt.h \ + /usr/include/bits/unistd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/atomic_word.h \ + /usr/include/c++/4.6/bits/locale_classes.h /usr/include/c++/4.6/string \ + /usr/include/c++/4.6/bits/allocator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++allocator.h \ + /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ + /usr/include/c++/4.6/bits/ostream_insert.h \ + /usr/include/c++/4.6/bits/cxxabi_forced.h \ + /usr/include/c++/4.6/bits/stl_function.h \ + /usr/include/c++/4.6/backward/binders.h \ + /usr/include/c++/4.6/bits/range_access.h \ + /usr/include/c++/4.6/bits/basic_string.h \ + /usr/include/c++/4.6/initializer_list \ + /usr/include/c++/4.6/bits/basic_string.tcc \ + /usr/include/c++/4.6/bits/locale_classes.tcc \ + /usr/include/c++/4.6/streambuf /usr/include/c++/4.6/bits/streambuf.tcc \ + /usr/include/c++/4.6/bits/basic_ios.h \ + /usr/include/c++/4.6/bits/locale_facets.h /usr/include/c++/4.6/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_base.h \ + /usr/include/c++/4.6/bits/streambuf_iterator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_inline.h \ + /usr/include/c++/4.6/bits/locale_facets.tcc \ + /usr/include/c++/4.6/bits/basic_ios.tcc /usr/include/c++/4.6/ostream \ + /usr/include/c++/4.6/bits/ostream.tcc \ + /usr/include/c++/4.6/bits/istream.tcc \ + /usr/include/c++/4.6/bits/sstream.tcc ../KaldiLib/Vector.h \ + /usr/include/c++/4.6/cstddef /usr/include/c++/4.6/cstdlib \ + /usr/include/stdlib.h /usr/include/bits/waitflags.h \ + /usr/include/bits/waitstatus.h /usr/include/sys/types.h \ + /usr/include/sys/select.h /usr/include/bits/select.h \ + /usr/include/bits/sigset.h /usr/include/bits/select2.h \ + /usr/include/sys/sysmacros.h /usr/include/alloca.h \ + /usr/include/bits/stdlib.h /usr/include/c++/4.6/stdexcept \ + /usr/include/c++/4.6/iostream ../KaldiLib/cblas.h ../KaldiLib/clapack.h \ + ../KaldiLib/cblas.h ../KaldiLib/Common.h /usr/include/string.h \ + /usr/include/bits/string3.h ../KaldiLib/MathAux.h ../KaldiLib/Types.h \ + ../KaldiLib/Error.h /usr/include/execinfo.h ../KaldiLib/Vector.tcc \ + /usr/include/c++/4.6/cstring /usr/include/c++/4.6/fstream \ + /usr/include/c++/4.6/bits/codecvt.h /usr/include/c++/4.6/cstdio \ + /usr/include/libio.h /usr/include/_G_config.h \ + /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \ + /usr/include/bits/stdio.h /usr/include/bits/stdio2.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/basic_file.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++io.h \ + /usr/include/c++/4.6/bits/fstream.tcc /usr/include/c++/4.6/iomanip \ + ../KaldiLib/Matrix.h ../KaldiLib/Matrix.tcc /usr/include/c++/4.6/cfloat \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/float.h \ + /usr/include/c++/4.6/typeinfo /usr/include/c++/4.6/algorithm \ + /usr/include/c++/4.6/utility /usr/include/c++/4.6/bits/stl_relops.h \ + /usr/include/c++/4.6/bits/stl_algo.h \ + /usr/include/c++/4.6/bits/algorithmfwd.h \ + /usr/include/c++/4.6/bits/stl_heap.h \ + /usr/include/c++/4.6/bits/stl_tempbuf.h \ + /usr/include/c++/4.6/bits/stl_construct.h /usr/include/c++/4.6/vector \ + /usr/include/c++/4.6/bits/stl_uninitialized.h \ + /usr/include/c++/4.6/bits/stl_vector.h \ + /usr/include/c++/4.6/bits/stl_bvector.h \ + /usr/include/c++/4.6/bits/vector.tcc ../KaldiLib/Vector.h \ + ../CuBaseLib/cuvector.h ../CuBaseLib/cuvector.tcc \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/host_defines.h \ + /usr/local/cuda-5.0/include/builtin_types.h \ + /usr/local/cuda-5.0/include/device_types.h \ + /usr/local/cuda-5.0/include/driver_types.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/limits.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/syslimits.h \ + /usr/include/limits.h /usr/include/bits/posix1_lim.h \ + /usr/include/bits/local_lim.h /usr/include/linux/limits.h \ + /usr/include/bits/posix2_lim.h /usr/include/bits/xopen_lim.h \ + /usr/local/cuda-5.0/include/surface_types.h \ + /usr/local/cuda-5.0/include/texture_types.h \ + /usr/local/cuda-5.0/include/vector_types.h \ + /usr/local/cuda-5.0/include/cuda_device_runtime_api.h \ + ../KaldiLib/Timer.h /usr/include/sys/time.h ../CuBaseLib/cucommon.h \ + ../KaldiLib/Error.h ../CuBaseLib/cumatrix.h ../KaldiLib/Matrix.h \ + ../CuBaseLib/cukernels.h /usr/local/cuda-5.0/include/vector_types.h \ + ../CuBaseLib/cumatrix.tcc /usr/local/cuda-5.0/include/cublas.h \ + /usr/local/cuda-5.0/include/cuda_runtime.h \ + /usr/local/cuda-5.0/include/host_config.h \ + /usr/local/cuda-5.0/include/channel_descriptor.h \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/driver_functions.h \ + /usr/local/cuda-5.0/include/vector_functions.h \ + /usr/local/cuda-5.0/include/cublas_api.h \ + /usr/local/cuda-5.0/include/cuComplex.h ../CuBaseLib/cuvector.h \ + ../CuBaseLib/cudevice.h /usr/include/c++/4.6/map \ + /usr/include/c++/4.6/bits/stl_tree.h /usr/include/c++/4.6/bits/stl_map.h \ + /usr/include/c++/4.6/bits/stl_multimap.h ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cumath.h +cuRbm.o: cuRbm.cc /usr/include/c++/4.6/string \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++config.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/os_defines.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-32.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/cpu_defines.h \ + /usr/include/c++/4.6/bits/stringfwd.h \ + /usr/include/c++/4.6/bits/char_traits.h \ + /usr/include/c++/4.6/bits/stl_algobase.h \ + /usr/include/c++/4.6/bits/functexcept.h \ + /usr/include/c++/4.6/bits/exception_defines.h \ + /usr/include/c++/4.6/bits/cpp_type_traits.h \ + /usr/include/c++/4.6/ext/type_traits.h \ + /usr/include/c++/4.6/ext/numeric_traits.h \ + /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ + /usr/include/c++/4.6/bits/concept_check.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.6/bits/stl_iterator.h \ + /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/postypes.h \ + /usr/include/c++/4.6/cwchar /usr/include/wchar.h /usr/include/stdio.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stdarg.h \ + /usr/include/bits/wchar.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stddef.h \ + /usr/include/xlocale.h /usr/include/bits/wchar2.h \ + /usr/include/c++/4.6/bits/allocator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++allocator.h \ + /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ + /usr/include/c++/4.6/exception /usr/include/c++/4.6/bits/localefwd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++locale.h \ + /usr/include/c++/4.6/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ + /usr/include/c++/4.6/cctype /usr/include/ctype.h \ + /usr/include/bits/types.h /usr/include/bits/typesizes.h \ + /usr/include/endian.h /usr/include/bits/endian.h \ + /usr/include/bits/byteswap.h /usr/include/c++/4.6/bits/ostream_insert.h \ + /usr/include/c++/4.6/bits/cxxabi_forced.h \ + /usr/include/c++/4.6/bits/stl_function.h \ + /usr/include/c++/4.6/backward/binders.h \ + /usr/include/c++/4.6/bits/range_access.h \ + /usr/include/c++/4.6/bits/basic_string.h \ + /usr/include/c++/4.6/ext/atomicity.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/time.h \ + /usr/include/bits/sched.h /usr/include/bits/time.h \ + /usr/include/bits/timex.h /usr/include/bits/pthreadtypes.h \ + /usr/include/bits/setjmp.h /usr/include/unistd.h \ + /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ + /usr/include/bits/confname.h /usr/include/getopt.h \ + /usr/include/bits/unistd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/atomic_word.h \ + /usr/include/c++/4.6/initializer_list \ + /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/sstream \ + /usr/include/c++/4.6/istream /usr/include/c++/4.6/ios \ + /usr/include/c++/4.6/bits/ios_base.h \ + /usr/include/c++/4.6/bits/locale_classes.h \ + /usr/include/c++/4.6/bits/locale_classes.tcc \ + /usr/include/c++/4.6/streambuf /usr/include/c++/4.6/bits/streambuf.tcc \ + /usr/include/c++/4.6/bits/basic_ios.h \ + /usr/include/c++/4.6/bits/locale_facets.h /usr/include/c++/4.6/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_base.h \ + /usr/include/c++/4.6/bits/streambuf_iterator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_inline.h \ + /usr/include/c++/4.6/bits/locale_facets.tcc \ + /usr/include/c++/4.6/bits/basic_ios.tcc /usr/include/c++/4.6/ostream \ + /usr/include/c++/4.6/bits/ostream.tcc \ + /usr/include/c++/4.6/bits/istream.tcc \ + /usr/include/c++/4.6/bits/sstream.tcc cuRbm.h cuComponent.h \ + ../KaldiLib/Vector.h /usr/include/c++/4.6/cstddef \ + /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ + /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ + /usr/include/sys/types.h /usr/include/sys/select.h \ + /usr/include/bits/select.h /usr/include/bits/sigset.h \ + /usr/include/bits/select2.h /usr/include/sys/sysmacros.h \ + /usr/include/alloca.h /usr/include/bits/stdlib.h \ + /usr/include/c++/4.6/stdexcept /usr/include/c++/4.6/iostream \ + ../KaldiLib/cblas.h ../KaldiLib/clapack.h ../KaldiLib/cblas.h \ + ../KaldiLib/Common.h /usr/include/string.h /usr/include/bits/string3.h \ + ../KaldiLib/MathAux.h /usr/include/c++/4.6/cmath /usr/include/math.h \ + /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ + /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ + /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ + /usr/include/bits/mathcalls.h /usr/include/bits/mathinline.h \ + ../KaldiLib/Types.h ../KaldiLib/Error.h /usr/include/execinfo.h \ + ../KaldiLib/Vector.tcc /usr/include/c++/4.6/cstring \ + /usr/include/c++/4.6/fstream /usr/include/c++/4.6/bits/codecvt.h \ + /usr/include/c++/4.6/cstdio /usr/include/libio.h \ + /usr/include/_G_config.h /usr/include/bits/stdio_lim.h \ + /usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \ + /usr/include/bits/stdio2.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/basic_file.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++io.h \ + /usr/include/c++/4.6/bits/fstream.tcc /usr/include/c++/4.6/iomanip \ + ../KaldiLib/Matrix.h ../KaldiLib/Matrix.tcc /usr/include/c++/4.6/cfloat \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/float.h \ + /usr/include/c++/4.6/typeinfo /usr/include/c++/4.6/algorithm \ + /usr/include/c++/4.6/utility /usr/include/c++/4.6/bits/stl_relops.h \ + /usr/include/c++/4.6/bits/stl_algo.h \ + /usr/include/c++/4.6/bits/algorithmfwd.h \ + /usr/include/c++/4.6/bits/stl_heap.h \ + /usr/include/c++/4.6/bits/stl_tempbuf.h \ + /usr/include/c++/4.6/bits/stl_construct.h /usr/include/c++/4.6/limits \ + /usr/include/c++/4.6/vector \ + /usr/include/c++/4.6/bits/stl_uninitialized.h \ + /usr/include/c++/4.6/bits/stl_vector.h \ + /usr/include/c++/4.6/bits/stl_bvector.h \ + /usr/include/c++/4.6/bits/vector.tcc ../KaldiLib/Vector.h \ + ../KaldiLib/Matrix.h ../KaldiLib/Error.h ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cukernels.h /usr/local/cuda-5.0/include/vector_types.h \ + /usr/local/cuda-5.0/include/builtin_types.h \ + /usr/local/cuda-5.0/include/device_types.h \ + /usr/local/cuda-5.0/include/host_defines.h \ + /usr/local/cuda-5.0/include/driver_types.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/limits.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/syslimits.h \ + /usr/include/limits.h /usr/include/bits/posix1_lim.h \ + /usr/include/bits/local_lim.h /usr/include/linux/limits.h \ + /usr/include/bits/posix2_lim.h /usr/include/bits/xopen_lim.h \ + /usr/local/cuda-5.0/include/surface_types.h \ + /usr/local/cuda-5.0/include/texture_types.h \ + /usr/local/cuda-5.0/include/vector_types.h ../CuBaseLib/cumatrix.tcc \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/cuda_device_runtime_api.h \ + /usr/local/cuda-5.0/include/cublas.h \ + /usr/local/cuda-5.0/include/cuda_runtime.h \ + /usr/local/cuda-5.0/include/host_config.h \ + /usr/local/cuda-5.0/include/channel_descriptor.h \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/driver_functions.h \ + /usr/local/cuda-5.0/include/vector_functions.h \ + /usr/local/cuda-5.0/include/cublas_api.h \ + /usr/local/cuda-5.0/include/cuComplex.h ../KaldiLib/Timer.h \ + /usr/include/sys/time.h ../CuBaseLib/cucommon.h ../CuBaseLib/cuvector.h \ + ../CuBaseLib/cuvector.tcc ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cudevice.h /usr/include/c++/4.6/map \ + /usr/include/c++/4.6/bits/stl_tree.h /usr/include/c++/4.6/bits/stl_map.h \ + /usr/include/c++/4.6/bits/stl_multimap.h ../CuBaseLib/cumath.h +cuRbmSparse.o: cuRbmSparse.cc /usr/include/c++/4.6/string \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++config.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/os_defines.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-32.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/cpu_defines.h \ + /usr/include/c++/4.6/bits/stringfwd.h \ + /usr/include/c++/4.6/bits/char_traits.h \ + /usr/include/c++/4.6/bits/stl_algobase.h \ + /usr/include/c++/4.6/bits/functexcept.h \ + /usr/include/c++/4.6/bits/exception_defines.h \ + /usr/include/c++/4.6/bits/cpp_type_traits.h \ + /usr/include/c++/4.6/ext/type_traits.h \ + /usr/include/c++/4.6/ext/numeric_traits.h \ + /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ + /usr/include/c++/4.6/bits/concept_check.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.6/bits/stl_iterator.h \ + /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/postypes.h \ + /usr/include/c++/4.6/cwchar /usr/include/wchar.h /usr/include/stdio.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stdarg.h \ + /usr/include/bits/wchar.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stddef.h \ + /usr/include/xlocale.h /usr/include/bits/wchar2.h \ + /usr/include/c++/4.6/bits/allocator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++allocator.h \ + /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ + /usr/include/c++/4.6/exception /usr/include/c++/4.6/bits/localefwd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++locale.h \ + /usr/include/c++/4.6/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ + /usr/include/c++/4.6/cctype /usr/include/ctype.h \ + /usr/include/bits/types.h /usr/include/bits/typesizes.h \ + /usr/include/endian.h /usr/include/bits/endian.h \ + /usr/include/bits/byteswap.h /usr/include/c++/4.6/bits/ostream_insert.h \ + /usr/include/c++/4.6/bits/cxxabi_forced.h \ + /usr/include/c++/4.6/bits/stl_function.h \ + /usr/include/c++/4.6/backward/binders.h \ + /usr/include/c++/4.6/bits/range_access.h \ + /usr/include/c++/4.6/bits/basic_string.h \ + /usr/include/c++/4.6/ext/atomicity.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/time.h \ + /usr/include/bits/sched.h /usr/include/bits/time.h \ + /usr/include/bits/timex.h /usr/include/bits/pthreadtypes.h \ + /usr/include/bits/setjmp.h /usr/include/unistd.h \ + /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ + /usr/include/bits/confname.h /usr/include/getopt.h \ + /usr/include/bits/unistd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/atomic_word.h \ + /usr/include/c++/4.6/initializer_list \ + /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/sstream \ + /usr/include/c++/4.6/istream /usr/include/c++/4.6/ios \ + /usr/include/c++/4.6/bits/ios_base.h \ + /usr/include/c++/4.6/bits/locale_classes.h \ + /usr/include/c++/4.6/bits/locale_classes.tcc \ + /usr/include/c++/4.6/streambuf /usr/include/c++/4.6/bits/streambuf.tcc \ + /usr/include/c++/4.6/bits/basic_ios.h \ + /usr/include/c++/4.6/bits/locale_facets.h /usr/include/c++/4.6/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_base.h \ + /usr/include/c++/4.6/bits/streambuf_iterator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_inline.h \ + /usr/include/c++/4.6/bits/locale_facets.tcc \ + /usr/include/c++/4.6/bits/basic_ios.tcc /usr/include/c++/4.6/ostream \ + /usr/include/c++/4.6/bits/ostream.tcc \ + /usr/include/c++/4.6/bits/istream.tcc \ + /usr/include/c++/4.6/bits/sstream.tcc cuRbmSparse.h cuComponent.h \ + ../KaldiLib/Vector.h /usr/include/c++/4.6/cstddef \ + /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ + /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ + /usr/include/sys/types.h /usr/include/sys/select.h \ + /usr/include/bits/select.h /usr/include/bits/sigset.h \ + /usr/include/bits/select2.h /usr/include/sys/sysmacros.h \ + /usr/include/alloca.h /usr/include/bits/stdlib.h \ + /usr/include/c++/4.6/stdexcept /usr/include/c++/4.6/iostream \ + ../KaldiLib/cblas.h ../KaldiLib/clapack.h ../KaldiLib/cblas.h \ + ../KaldiLib/Common.h /usr/include/string.h /usr/include/bits/string3.h \ + ../KaldiLib/MathAux.h /usr/include/c++/4.6/cmath /usr/include/math.h \ + /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ + /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ + /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ + /usr/include/bits/mathcalls.h /usr/include/bits/mathinline.h \ + ../KaldiLib/Types.h ../KaldiLib/Error.h /usr/include/execinfo.h \ + ../KaldiLib/Vector.tcc /usr/include/c++/4.6/cstring \ + /usr/include/c++/4.6/fstream /usr/include/c++/4.6/bits/codecvt.h \ + /usr/include/c++/4.6/cstdio /usr/include/libio.h \ + /usr/include/_G_config.h /usr/include/bits/stdio_lim.h \ + /usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \ + /usr/include/bits/stdio2.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/basic_file.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++io.h \ + /usr/include/c++/4.6/bits/fstream.tcc /usr/include/c++/4.6/iomanip \ + ../KaldiLib/Matrix.h ../KaldiLib/Matrix.tcc /usr/include/c++/4.6/cfloat \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/float.h \ + /usr/include/c++/4.6/typeinfo /usr/include/c++/4.6/algorithm \ + /usr/include/c++/4.6/utility /usr/include/c++/4.6/bits/stl_relops.h \ + /usr/include/c++/4.6/bits/stl_algo.h \ + /usr/include/c++/4.6/bits/algorithmfwd.h \ + /usr/include/c++/4.6/bits/stl_heap.h \ + /usr/include/c++/4.6/bits/stl_tempbuf.h \ + /usr/include/c++/4.6/bits/stl_construct.h /usr/include/c++/4.6/limits \ + /usr/include/c++/4.6/vector \ + /usr/include/c++/4.6/bits/stl_uninitialized.h \ + /usr/include/c++/4.6/bits/stl_vector.h \ + /usr/include/c++/4.6/bits/stl_bvector.h \ + /usr/include/c++/4.6/bits/vector.tcc ../KaldiLib/Vector.h \ + ../KaldiLib/Matrix.h ../KaldiLib/Error.h ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cukernels.h /usr/local/cuda-5.0/include/vector_types.h \ + /usr/local/cuda-5.0/include/builtin_types.h \ + /usr/local/cuda-5.0/include/device_types.h \ + /usr/local/cuda-5.0/include/host_defines.h \ + /usr/local/cuda-5.0/include/driver_types.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/limits.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/syslimits.h \ + /usr/include/limits.h /usr/include/bits/posix1_lim.h \ + /usr/include/bits/local_lim.h /usr/include/linux/limits.h \ + /usr/include/bits/posix2_lim.h /usr/include/bits/xopen_lim.h \ + /usr/local/cuda-5.0/include/surface_types.h \ + /usr/local/cuda-5.0/include/texture_types.h \ + /usr/local/cuda-5.0/include/vector_types.h ../CuBaseLib/cumatrix.tcc \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/cuda_device_runtime_api.h \ + /usr/local/cuda-5.0/include/cublas.h \ + /usr/local/cuda-5.0/include/cuda_runtime.h \ + /usr/local/cuda-5.0/include/host_config.h \ + /usr/local/cuda-5.0/include/channel_descriptor.h \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/driver_functions.h \ + /usr/local/cuda-5.0/include/vector_functions.h \ + /usr/local/cuda-5.0/include/cublas_api.h \ + /usr/local/cuda-5.0/include/cuComplex.h ../KaldiLib/Timer.h \ + /usr/include/sys/time.h ../CuBaseLib/cucommon.h ../CuBaseLib/cuvector.h \ + ../CuBaseLib/cuvector.tcc ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cudevice.h /usr/include/c++/4.6/map \ + /usr/include/c++/4.6/bits/stl_tree.h /usr/include/c++/4.6/bits/stl_map.h \ + /usr/include/c++/4.6/bits/stl_multimap.h cuRbm.h ../CuBaseLib/cumath.h +cuRecurrent.o: cuRecurrent.cc /usr/include/c++/4.6/string \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++config.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/os_defines.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-32.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/cpu_defines.h \ + /usr/include/c++/4.6/bits/stringfwd.h \ + /usr/include/c++/4.6/bits/char_traits.h \ + /usr/include/c++/4.6/bits/stl_algobase.h \ + /usr/include/c++/4.6/bits/functexcept.h \ + /usr/include/c++/4.6/bits/exception_defines.h \ + /usr/include/c++/4.6/bits/cpp_type_traits.h \ + /usr/include/c++/4.6/ext/type_traits.h \ + /usr/include/c++/4.6/ext/numeric_traits.h \ + /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ + /usr/include/c++/4.6/bits/concept_check.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.6/bits/stl_iterator.h \ + /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/postypes.h \ + /usr/include/c++/4.6/cwchar /usr/include/wchar.h /usr/include/stdio.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stdarg.h \ + /usr/include/bits/wchar.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stddef.h \ + /usr/include/xlocale.h /usr/include/bits/wchar2.h \ + /usr/include/c++/4.6/bits/allocator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++allocator.h \ + /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ + /usr/include/c++/4.6/exception /usr/include/c++/4.6/bits/localefwd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++locale.h \ + /usr/include/c++/4.6/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ + /usr/include/c++/4.6/cctype /usr/include/ctype.h \ + /usr/include/bits/types.h /usr/include/bits/typesizes.h \ + /usr/include/endian.h /usr/include/bits/endian.h \ + /usr/include/bits/byteswap.h /usr/include/c++/4.6/bits/ostream_insert.h \ + /usr/include/c++/4.6/bits/cxxabi_forced.h \ + /usr/include/c++/4.6/bits/stl_function.h \ + /usr/include/c++/4.6/backward/binders.h \ + /usr/include/c++/4.6/bits/range_access.h \ + /usr/include/c++/4.6/bits/basic_string.h \ + /usr/include/c++/4.6/ext/atomicity.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/time.h \ + /usr/include/bits/sched.h /usr/include/bits/time.h \ + /usr/include/bits/timex.h /usr/include/bits/pthreadtypes.h \ + /usr/include/bits/setjmp.h /usr/include/unistd.h \ + /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ + /usr/include/bits/confname.h /usr/include/getopt.h \ + /usr/include/bits/unistd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/atomic_word.h \ + /usr/include/c++/4.6/initializer_list \ + /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/sstream \ + /usr/include/c++/4.6/istream /usr/include/c++/4.6/ios \ + /usr/include/c++/4.6/bits/ios_base.h \ + /usr/include/c++/4.6/bits/locale_classes.h \ + /usr/include/c++/4.6/bits/locale_classes.tcc \ + /usr/include/c++/4.6/streambuf /usr/include/c++/4.6/bits/streambuf.tcc \ + /usr/include/c++/4.6/bits/basic_ios.h \ + /usr/include/c++/4.6/bits/locale_facets.h /usr/include/c++/4.6/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_base.h \ + /usr/include/c++/4.6/bits/streambuf_iterator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_inline.h \ + /usr/include/c++/4.6/bits/locale_facets.tcc \ + /usr/include/c++/4.6/bits/basic_ios.tcc /usr/include/c++/4.6/ostream \ + /usr/include/c++/4.6/bits/ostream.tcc \ + /usr/include/c++/4.6/bits/istream.tcc \ + /usr/include/c++/4.6/bits/sstream.tcc cuRecurrent.h cuComponent.h \ + ../KaldiLib/Vector.h /usr/include/c++/4.6/cstddef \ + /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ + /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ + /usr/include/sys/types.h /usr/include/sys/select.h \ + /usr/include/bits/select.h /usr/include/bits/sigset.h \ + /usr/include/bits/select2.h /usr/include/sys/sysmacros.h \ + /usr/include/alloca.h /usr/include/bits/stdlib.h \ + /usr/include/c++/4.6/stdexcept /usr/include/c++/4.6/iostream \ + ../KaldiLib/cblas.h ../KaldiLib/clapack.h ../KaldiLib/cblas.h \ + ../KaldiLib/Common.h /usr/include/string.h /usr/include/bits/string3.h \ + ../KaldiLib/MathAux.h /usr/include/c++/4.6/cmath /usr/include/math.h \ + /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ + /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ + /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ + /usr/include/bits/mathcalls.h /usr/include/bits/mathinline.h \ + ../KaldiLib/Types.h ../KaldiLib/Error.h /usr/include/execinfo.h \ + ../KaldiLib/Vector.tcc /usr/include/c++/4.6/cstring \ + /usr/include/c++/4.6/fstream /usr/include/c++/4.6/bits/codecvt.h \ + /usr/include/c++/4.6/cstdio /usr/include/libio.h \ + /usr/include/_G_config.h /usr/include/bits/stdio_lim.h \ + /usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \ + /usr/include/bits/stdio2.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/basic_file.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++io.h \ + /usr/include/c++/4.6/bits/fstream.tcc /usr/include/c++/4.6/iomanip \ + ../KaldiLib/Matrix.h ../KaldiLib/Matrix.tcc /usr/include/c++/4.6/cfloat \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/float.h \ + /usr/include/c++/4.6/typeinfo /usr/include/c++/4.6/algorithm \ + /usr/include/c++/4.6/utility /usr/include/c++/4.6/bits/stl_relops.h \ + /usr/include/c++/4.6/bits/stl_algo.h \ + /usr/include/c++/4.6/bits/algorithmfwd.h \ + /usr/include/c++/4.6/bits/stl_heap.h \ + /usr/include/c++/4.6/bits/stl_tempbuf.h \ + /usr/include/c++/4.6/bits/stl_construct.h /usr/include/c++/4.6/limits \ + /usr/include/c++/4.6/vector \ + /usr/include/c++/4.6/bits/stl_uninitialized.h \ + /usr/include/c++/4.6/bits/stl_vector.h \ + /usr/include/c++/4.6/bits/stl_bvector.h \ + /usr/include/c++/4.6/bits/vector.tcc ../KaldiLib/Vector.h \ + ../KaldiLib/Matrix.h ../KaldiLib/Error.h ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cukernels.h /usr/local/cuda-5.0/include/vector_types.h \ + /usr/local/cuda-5.0/include/builtin_types.h \ + /usr/local/cuda-5.0/include/device_types.h \ + /usr/local/cuda-5.0/include/host_defines.h \ + /usr/local/cuda-5.0/include/driver_types.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/limits.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/syslimits.h \ + /usr/include/limits.h /usr/include/bits/posix1_lim.h \ + /usr/include/bits/local_lim.h /usr/include/linux/limits.h \ + /usr/include/bits/posix2_lim.h /usr/include/bits/xopen_lim.h \ + /usr/local/cuda-5.0/include/surface_types.h \ + /usr/local/cuda-5.0/include/texture_types.h \ + /usr/local/cuda-5.0/include/vector_types.h ../CuBaseLib/cumatrix.tcc \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/cuda_device_runtime_api.h \ + /usr/local/cuda-5.0/include/cublas.h \ + /usr/local/cuda-5.0/include/cuda_runtime.h \ + /usr/local/cuda-5.0/include/host_config.h \ + /usr/local/cuda-5.0/include/channel_descriptor.h \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/driver_functions.h \ + /usr/local/cuda-5.0/include/vector_functions.h \ + /usr/local/cuda-5.0/include/cublas_api.h \ + /usr/local/cuda-5.0/include/cuComplex.h ../KaldiLib/Timer.h \ + /usr/include/sys/time.h ../CuBaseLib/cucommon.h ../CuBaseLib/cuvector.h \ + ../CuBaseLib/cuvector.tcc ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cudevice.h /usr/include/c++/4.6/map \ + /usr/include/c++/4.6/bits/stl_tree.h /usr/include/c++/4.6/bits/stl_map.h \ + /usr/include/c++/4.6/bits/stl_multimap.h ../CuBaseLib/cumath.h +cuSharedLinearity.o: cuSharedLinearity.cc cuSharedLinearity.h \ + cuComponent.h ../KaldiLib/Vector.h /usr/include/c++/4.6/cstddef \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++config.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/os_defines.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-32.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/cpu_defines.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stddef.h \ + /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ + /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ + /usr/include/endian.h /usr/include/bits/endian.h \ + /usr/include/bits/byteswap.h /usr/include/xlocale.h \ + /usr/include/sys/types.h /usr/include/bits/types.h \ + /usr/include/bits/typesizes.h /usr/include/time.h \ + /usr/include/sys/select.h /usr/include/bits/select.h \ + /usr/include/bits/sigset.h /usr/include/bits/time.h \ + /usr/include/bits/select2.h /usr/include/sys/sysmacros.h \ + /usr/include/bits/pthreadtypes.h /usr/include/alloca.h \ + /usr/include/bits/stdlib.h /usr/include/c++/4.6/stdexcept \ + /usr/include/c++/4.6/exception /usr/include/c++/4.6/string \ + /usr/include/c++/4.6/bits/stringfwd.h \ + /usr/include/c++/4.6/bits/char_traits.h \ + /usr/include/c++/4.6/bits/stl_algobase.h \ + /usr/include/c++/4.6/bits/functexcept.h \ + /usr/include/c++/4.6/bits/exception_defines.h \ + /usr/include/c++/4.6/bits/cpp_type_traits.h \ + /usr/include/c++/4.6/ext/type_traits.h \ + /usr/include/c++/4.6/ext/numeric_traits.h \ + /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ + /usr/include/c++/4.6/bits/concept_check.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.6/bits/stl_iterator.h \ + /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/postypes.h \ + /usr/include/c++/4.6/cwchar /usr/include/wchar.h /usr/include/stdio.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stdarg.h \ + /usr/include/bits/wchar.h /usr/include/bits/wchar2.h \ + /usr/include/c++/4.6/bits/allocator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++allocator.h \ + /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ + /usr/include/c++/4.6/bits/localefwd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++locale.h \ + /usr/include/c++/4.6/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ + /usr/include/c++/4.6/cctype /usr/include/ctype.h \ + /usr/include/c++/4.6/bits/ostream_insert.h \ + /usr/include/c++/4.6/bits/cxxabi_forced.h \ + /usr/include/c++/4.6/bits/stl_function.h \ + /usr/include/c++/4.6/backward/binders.h \ + /usr/include/c++/4.6/bits/range_access.h \ + /usr/include/c++/4.6/bits/basic_string.h \ + /usr/include/c++/4.6/ext/atomicity.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/bits/sched.h \ + /usr/include/bits/timex.h /usr/include/bits/setjmp.h \ + /usr/include/unistd.h /usr/include/bits/posix_opt.h \ + /usr/include/bits/environments.h /usr/include/bits/confname.h \ + /usr/include/getopt.h /usr/include/bits/unistd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/atomic_word.h \ + /usr/include/c++/4.6/initializer_list \ + /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/iostream \ + /usr/include/c++/4.6/ostream /usr/include/c++/4.6/ios \ + /usr/include/c++/4.6/bits/ios_base.h \ + /usr/include/c++/4.6/bits/locale_classes.h \ + /usr/include/c++/4.6/bits/locale_classes.tcc \ + /usr/include/c++/4.6/streambuf /usr/include/c++/4.6/bits/streambuf.tcc \ + /usr/include/c++/4.6/bits/basic_ios.h \ + /usr/include/c++/4.6/bits/locale_facets.h /usr/include/c++/4.6/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_base.h \ + /usr/include/c++/4.6/bits/streambuf_iterator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_inline.h \ + /usr/include/c++/4.6/bits/locale_facets.tcc \ + /usr/include/c++/4.6/bits/basic_ios.tcc \ + /usr/include/c++/4.6/bits/ostream.tcc /usr/include/c++/4.6/istream \ + /usr/include/c++/4.6/bits/istream.tcc ../KaldiLib/cblas.h \ + ../KaldiLib/clapack.h ../KaldiLib/cblas.h ../KaldiLib/Common.h \ + /usr/include/string.h /usr/include/bits/string3.h \ + /usr/include/c++/4.6/sstream /usr/include/c++/4.6/bits/sstream.tcc \ + ../KaldiLib/MathAux.h /usr/include/c++/4.6/cmath /usr/include/math.h \ + /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ + /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ + /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ + /usr/include/bits/mathcalls.h /usr/include/bits/mathinline.h \ + ../KaldiLib/Types.h ../KaldiLib/Error.h /usr/include/execinfo.h \ + ../KaldiLib/Vector.tcc /usr/include/c++/4.6/cstring \ + /usr/include/c++/4.6/fstream /usr/include/c++/4.6/bits/codecvt.h \ + /usr/include/c++/4.6/cstdio /usr/include/libio.h \ + /usr/include/_G_config.h /usr/include/bits/stdio_lim.h \ + /usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \ + /usr/include/bits/stdio2.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/basic_file.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++io.h \ + /usr/include/c++/4.6/bits/fstream.tcc /usr/include/c++/4.6/iomanip \ + ../KaldiLib/Matrix.h ../KaldiLib/Matrix.tcc /usr/include/c++/4.6/cfloat \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/float.h \ + /usr/include/c++/4.6/typeinfo /usr/include/c++/4.6/algorithm \ + /usr/include/c++/4.6/utility /usr/include/c++/4.6/bits/stl_relops.h \ + /usr/include/c++/4.6/bits/stl_algo.h \ + /usr/include/c++/4.6/bits/algorithmfwd.h \ + /usr/include/c++/4.6/bits/stl_heap.h \ + /usr/include/c++/4.6/bits/stl_tempbuf.h \ + /usr/include/c++/4.6/bits/stl_construct.h /usr/include/c++/4.6/limits \ + /usr/include/c++/4.6/vector \ + /usr/include/c++/4.6/bits/stl_uninitialized.h \ + /usr/include/c++/4.6/bits/stl_vector.h \ + /usr/include/c++/4.6/bits/stl_bvector.h \ + /usr/include/c++/4.6/bits/vector.tcc ../KaldiLib/Vector.h \ + ../KaldiLib/Matrix.h ../KaldiLib/Error.h ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cukernels.h /usr/local/cuda-5.0/include/vector_types.h \ + /usr/local/cuda-5.0/include/builtin_types.h \ + /usr/local/cuda-5.0/include/device_types.h \ + /usr/local/cuda-5.0/include/host_defines.h \ + /usr/local/cuda-5.0/include/driver_types.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/limits.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/syslimits.h \ + /usr/include/limits.h /usr/include/bits/posix1_lim.h \ + /usr/include/bits/local_lim.h /usr/include/linux/limits.h \ + /usr/include/bits/posix2_lim.h /usr/include/bits/xopen_lim.h \ + /usr/local/cuda-5.0/include/surface_types.h \ + /usr/local/cuda-5.0/include/texture_types.h \ + /usr/local/cuda-5.0/include/vector_types.h ../CuBaseLib/cumatrix.tcc \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/cuda_device_runtime_api.h \ + /usr/local/cuda-5.0/include/cublas.h \ + /usr/local/cuda-5.0/include/cuda_runtime.h \ + /usr/local/cuda-5.0/include/host_config.h \ + /usr/local/cuda-5.0/include/channel_descriptor.h \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/driver_functions.h \ + /usr/local/cuda-5.0/include/vector_functions.h \ + /usr/local/cuda-5.0/include/cublas_api.h \ + /usr/local/cuda-5.0/include/cuComplex.h ../KaldiLib/Timer.h \ + /usr/include/sys/time.h ../CuBaseLib/cucommon.h ../CuBaseLib/cuvector.h \ + ../CuBaseLib/cuvector.tcc ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cudevice.h /usr/include/c++/4.6/map \ + /usr/include/c++/4.6/bits/stl_tree.h /usr/include/c++/4.6/bits/stl_map.h \ + /usr/include/c++/4.6/bits/stl_multimap.h ../CuBaseLib/cumath.h +cuSparseLinearity.o: cuSparseLinearity.cc cuSparseLinearity.h \ + cuComponent.h ../KaldiLib/Vector.h /usr/include/c++/4.6/cstddef \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++config.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/os_defines.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-32.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/cpu_defines.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stddef.h \ + /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ + /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ + /usr/include/endian.h /usr/include/bits/endian.h \ + /usr/include/bits/byteswap.h /usr/include/xlocale.h \ + /usr/include/sys/types.h /usr/include/bits/types.h \ + /usr/include/bits/typesizes.h /usr/include/time.h \ + /usr/include/sys/select.h /usr/include/bits/select.h \ + /usr/include/bits/sigset.h /usr/include/bits/time.h \ + /usr/include/bits/select2.h /usr/include/sys/sysmacros.h \ + /usr/include/bits/pthreadtypes.h /usr/include/alloca.h \ + /usr/include/bits/stdlib.h /usr/include/c++/4.6/stdexcept \ + /usr/include/c++/4.6/exception /usr/include/c++/4.6/string \ + /usr/include/c++/4.6/bits/stringfwd.h \ + /usr/include/c++/4.6/bits/char_traits.h \ + /usr/include/c++/4.6/bits/stl_algobase.h \ + /usr/include/c++/4.6/bits/functexcept.h \ + /usr/include/c++/4.6/bits/exception_defines.h \ + /usr/include/c++/4.6/bits/cpp_type_traits.h \ + /usr/include/c++/4.6/ext/type_traits.h \ + /usr/include/c++/4.6/ext/numeric_traits.h \ + /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ + /usr/include/c++/4.6/bits/concept_check.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.6/bits/stl_iterator.h \ + /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/postypes.h \ + /usr/include/c++/4.6/cwchar /usr/include/wchar.h /usr/include/stdio.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stdarg.h \ + /usr/include/bits/wchar.h /usr/include/bits/wchar2.h \ + /usr/include/c++/4.6/bits/allocator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++allocator.h \ + /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ + /usr/include/c++/4.6/bits/localefwd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++locale.h \ + /usr/include/c++/4.6/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ + /usr/include/c++/4.6/cctype /usr/include/ctype.h \ + /usr/include/c++/4.6/bits/ostream_insert.h \ + /usr/include/c++/4.6/bits/cxxabi_forced.h \ + /usr/include/c++/4.6/bits/stl_function.h \ + /usr/include/c++/4.6/backward/binders.h \ + /usr/include/c++/4.6/bits/range_access.h \ + /usr/include/c++/4.6/bits/basic_string.h \ + /usr/include/c++/4.6/ext/atomicity.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/bits/sched.h \ + /usr/include/bits/timex.h /usr/include/bits/setjmp.h \ + /usr/include/unistd.h /usr/include/bits/posix_opt.h \ + /usr/include/bits/environments.h /usr/include/bits/confname.h \ + /usr/include/getopt.h /usr/include/bits/unistd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/atomic_word.h \ + /usr/include/c++/4.6/initializer_list \ + /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/iostream \ + /usr/include/c++/4.6/ostream /usr/include/c++/4.6/ios \ + /usr/include/c++/4.6/bits/ios_base.h \ + /usr/include/c++/4.6/bits/locale_classes.h \ + /usr/include/c++/4.6/bits/locale_classes.tcc \ + /usr/include/c++/4.6/streambuf /usr/include/c++/4.6/bits/streambuf.tcc \ + /usr/include/c++/4.6/bits/basic_ios.h \ + /usr/include/c++/4.6/bits/locale_facets.h /usr/include/c++/4.6/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_base.h \ + /usr/include/c++/4.6/bits/streambuf_iterator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_inline.h \ + /usr/include/c++/4.6/bits/locale_facets.tcc \ + /usr/include/c++/4.6/bits/basic_ios.tcc \ + /usr/include/c++/4.6/bits/ostream.tcc /usr/include/c++/4.6/istream \ + /usr/include/c++/4.6/bits/istream.tcc ../KaldiLib/cblas.h \ + ../KaldiLib/clapack.h ../KaldiLib/cblas.h ../KaldiLib/Common.h \ + /usr/include/string.h /usr/include/bits/string3.h \ + /usr/include/c++/4.6/sstream /usr/include/c++/4.6/bits/sstream.tcc \ + ../KaldiLib/MathAux.h /usr/include/c++/4.6/cmath /usr/include/math.h \ + /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ + /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ + /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ + /usr/include/bits/mathcalls.h /usr/include/bits/mathinline.h \ + ../KaldiLib/Types.h ../KaldiLib/Error.h /usr/include/execinfo.h \ + ../KaldiLib/Vector.tcc /usr/include/c++/4.6/cstring \ + /usr/include/c++/4.6/fstream /usr/include/c++/4.6/bits/codecvt.h \ + /usr/include/c++/4.6/cstdio /usr/include/libio.h \ + /usr/include/_G_config.h /usr/include/bits/stdio_lim.h \ + /usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \ + /usr/include/bits/stdio2.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/basic_file.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++io.h \ + /usr/include/c++/4.6/bits/fstream.tcc /usr/include/c++/4.6/iomanip \ + ../KaldiLib/Matrix.h ../KaldiLib/Matrix.tcc /usr/include/c++/4.6/cfloat \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/float.h \ + /usr/include/c++/4.6/typeinfo /usr/include/c++/4.6/algorithm \ + /usr/include/c++/4.6/utility /usr/include/c++/4.6/bits/stl_relops.h \ + /usr/include/c++/4.6/bits/stl_algo.h \ + /usr/include/c++/4.6/bits/algorithmfwd.h \ + /usr/include/c++/4.6/bits/stl_heap.h \ + /usr/include/c++/4.6/bits/stl_tempbuf.h \ + /usr/include/c++/4.6/bits/stl_construct.h /usr/include/c++/4.6/limits \ + /usr/include/c++/4.6/vector \ + /usr/include/c++/4.6/bits/stl_uninitialized.h \ + /usr/include/c++/4.6/bits/stl_vector.h \ + /usr/include/c++/4.6/bits/stl_bvector.h \ + /usr/include/c++/4.6/bits/vector.tcc ../KaldiLib/Vector.h \ + ../KaldiLib/Matrix.h ../KaldiLib/Error.h ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cukernels.h /usr/local/cuda-5.0/include/vector_types.h \ + /usr/local/cuda-5.0/include/builtin_types.h \ + /usr/local/cuda-5.0/include/device_types.h \ + /usr/local/cuda-5.0/include/host_defines.h \ + /usr/local/cuda-5.0/include/driver_types.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/limits.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/syslimits.h \ + /usr/include/limits.h /usr/include/bits/posix1_lim.h \ + /usr/include/bits/local_lim.h /usr/include/linux/limits.h \ + /usr/include/bits/posix2_lim.h /usr/include/bits/xopen_lim.h \ + /usr/local/cuda-5.0/include/surface_types.h \ + /usr/local/cuda-5.0/include/texture_types.h \ + /usr/local/cuda-5.0/include/vector_types.h ../CuBaseLib/cumatrix.tcc \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/cuda_device_runtime_api.h \ + /usr/local/cuda-5.0/include/cublas.h \ + /usr/local/cuda-5.0/include/cuda_runtime.h \ + /usr/local/cuda-5.0/include/host_config.h \ + /usr/local/cuda-5.0/include/channel_descriptor.h \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/driver_functions.h \ + /usr/local/cuda-5.0/include/vector_functions.h \ + /usr/local/cuda-5.0/include/cublas_api.h \ + /usr/local/cuda-5.0/include/cuComplex.h ../KaldiLib/Timer.h \ + /usr/include/sys/time.h ../CuBaseLib/cucommon.h ../CuBaseLib/cuvector.h \ + ../CuBaseLib/cuvector.tcc ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cudevice.h /usr/include/c++/4.6/map \ + /usr/include/c++/4.6/bits/stl_tree.h /usr/include/c++/4.6/bits/stl_map.h \ + /usr/include/c++/4.6/bits/stl_multimap.h +cuUpdatableBias.o: cuUpdatableBias.cc cuUpdatableBias.h cuComponent.h \ + ../KaldiLib/Vector.h /usr/include/c++/4.6/cstddef \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++config.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/os_defines.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-32.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/cpu_defines.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stddef.h \ + /usr/include/c++/4.6/cstdlib /usr/include/stdlib.h \ + /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ + /usr/include/endian.h /usr/include/bits/endian.h \ + /usr/include/bits/byteswap.h /usr/include/xlocale.h \ + /usr/include/sys/types.h /usr/include/bits/types.h \ + /usr/include/bits/typesizes.h /usr/include/time.h \ + /usr/include/sys/select.h /usr/include/bits/select.h \ + /usr/include/bits/sigset.h /usr/include/bits/time.h \ + /usr/include/bits/select2.h /usr/include/sys/sysmacros.h \ + /usr/include/bits/pthreadtypes.h /usr/include/alloca.h \ + /usr/include/bits/stdlib.h /usr/include/c++/4.6/stdexcept \ + /usr/include/c++/4.6/exception /usr/include/c++/4.6/string \ + /usr/include/c++/4.6/bits/stringfwd.h \ + /usr/include/c++/4.6/bits/char_traits.h \ + /usr/include/c++/4.6/bits/stl_algobase.h \ + /usr/include/c++/4.6/bits/functexcept.h \ + /usr/include/c++/4.6/bits/exception_defines.h \ + /usr/include/c++/4.6/bits/cpp_type_traits.h \ + /usr/include/c++/4.6/ext/type_traits.h \ + /usr/include/c++/4.6/ext/numeric_traits.h \ + /usr/include/c++/4.6/bits/stl_pair.h /usr/include/c++/4.6/bits/move.h \ + /usr/include/c++/4.6/bits/concept_check.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.6/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.6/bits/stl_iterator.h \ + /usr/include/c++/4.6/debug/debug.h /usr/include/c++/4.6/bits/postypes.h \ + /usr/include/c++/4.6/cwchar /usr/include/wchar.h /usr/include/stdio.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/stdarg.h \ + /usr/include/bits/wchar.h /usr/include/bits/wchar2.h \ + /usr/include/c++/4.6/bits/allocator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++allocator.h \ + /usr/include/c++/4.6/ext/new_allocator.h /usr/include/c++/4.6/new \ + /usr/include/c++/4.6/bits/localefwd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++locale.h \ + /usr/include/c++/4.6/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.6/iosfwd \ + /usr/include/c++/4.6/cctype /usr/include/ctype.h \ + /usr/include/c++/4.6/bits/ostream_insert.h \ + /usr/include/c++/4.6/bits/cxxabi_forced.h \ + /usr/include/c++/4.6/bits/stl_function.h \ + /usr/include/c++/4.6/backward/binders.h \ + /usr/include/c++/4.6/bits/range_access.h \ + /usr/include/c++/4.6/bits/basic_string.h \ + /usr/include/c++/4.6/ext/atomicity.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/bits/sched.h \ + /usr/include/bits/timex.h /usr/include/bits/setjmp.h \ + /usr/include/unistd.h /usr/include/bits/posix_opt.h \ + /usr/include/bits/environments.h /usr/include/bits/confname.h \ + /usr/include/getopt.h /usr/include/bits/unistd.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/atomic_word.h \ + /usr/include/c++/4.6/initializer_list \ + /usr/include/c++/4.6/bits/basic_string.tcc /usr/include/c++/4.6/iostream \ + /usr/include/c++/4.6/ostream /usr/include/c++/4.6/ios \ + /usr/include/c++/4.6/bits/ios_base.h \ + /usr/include/c++/4.6/bits/locale_classes.h \ + /usr/include/c++/4.6/bits/locale_classes.tcc \ + /usr/include/c++/4.6/streambuf /usr/include/c++/4.6/bits/streambuf.tcc \ + /usr/include/c++/4.6/bits/basic_ios.h \ + /usr/include/c++/4.6/bits/locale_facets.h /usr/include/c++/4.6/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_base.h \ + /usr/include/c++/4.6/bits/streambuf_iterator.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/ctype_inline.h \ + /usr/include/c++/4.6/bits/locale_facets.tcc \ + /usr/include/c++/4.6/bits/basic_ios.tcc \ + /usr/include/c++/4.6/bits/ostream.tcc /usr/include/c++/4.6/istream \ + /usr/include/c++/4.6/bits/istream.tcc ../KaldiLib/cblas.h \ + ../KaldiLib/clapack.h ../KaldiLib/cblas.h ../KaldiLib/Common.h \ + /usr/include/string.h /usr/include/bits/string3.h \ + /usr/include/c++/4.6/sstream /usr/include/c++/4.6/bits/sstream.tcc \ + ../KaldiLib/MathAux.h /usr/include/c++/4.6/cmath /usr/include/math.h \ + /usr/include/bits/huge_val.h /usr/include/bits/huge_valf.h \ + /usr/include/bits/huge_vall.h /usr/include/bits/inf.h \ + /usr/include/bits/nan.h /usr/include/bits/mathdef.h \ + /usr/include/bits/mathcalls.h /usr/include/bits/mathinline.h \ + ../KaldiLib/Types.h ../KaldiLib/Error.h /usr/include/execinfo.h \ + ../KaldiLib/Vector.tcc /usr/include/c++/4.6/cstring \ + /usr/include/c++/4.6/fstream /usr/include/c++/4.6/bits/codecvt.h \ + /usr/include/c++/4.6/cstdio /usr/include/libio.h \ + /usr/include/_G_config.h /usr/include/bits/stdio_lim.h \ + /usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \ + /usr/include/bits/stdio2.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/basic_file.h \ + /usr/include/c++/4.6/x86_64-linux-gnu/32/bits/c++io.h \ + /usr/include/c++/4.6/bits/fstream.tcc /usr/include/c++/4.6/iomanip \ + ../KaldiLib/Matrix.h ../KaldiLib/Matrix.tcc /usr/include/c++/4.6/cfloat \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include/float.h \ + /usr/include/c++/4.6/typeinfo /usr/include/c++/4.6/algorithm \ + /usr/include/c++/4.6/utility /usr/include/c++/4.6/bits/stl_relops.h \ + /usr/include/c++/4.6/bits/stl_algo.h \ + /usr/include/c++/4.6/bits/algorithmfwd.h \ + /usr/include/c++/4.6/bits/stl_heap.h \ + /usr/include/c++/4.6/bits/stl_tempbuf.h \ + /usr/include/c++/4.6/bits/stl_construct.h /usr/include/c++/4.6/limits \ + /usr/include/c++/4.6/vector \ + /usr/include/c++/4.6/bits/stl_uninitialized.h \ + /usr/include/c++/4.6/bits/stl_vector.h \ + /usr/include/c++/4.6/bits/stl_bvector.h \ + /usr/include/c++/4.6/bits/vector.tcc ../KaldiLib/Vector.h \ + ../KaldiLib/Matrix.h ../KaldiLib/Error.h ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cukernels.h /usr/local/cuda-5.0/include/vector_types.h \ + /usr/local/cuda-5.0/include/builtin_types.h \ + /usr/local/cuda-5.0/include/device_types.h \ + /usr/local/cuda-5.0/include/host_defines.h \ + /usr/local/cuda-5.0/include/driver_types.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/limits.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.6/include-fixed/syslimits.h \ + /usr/include/limits.h /usr/include/bits/posix1_lim.h \ + /usr/include/bits/local_lim.h /usr/include/linux/limits.h \ + /usr/include/bits/posix2_lim.h /usr/include/bits/xopen_lim.h \ + /usr/local/cuda-5.0/include/surface_types.h \ + /usr/local/cuda-5.0/include/texture_types.h \ + /usr/local/cuda-5.0/include/vector_types.h ../CuBaseLib/cumatrix.tcc \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/cuda_device_runtime_api.h \ + /usr/local/cuda-5.0/include/cublas.h \ + /usr/local/cuda-5.0/include/cuda_runtime.h \ + /usr/local/cuda-5.0/include/host_config.h \ + /usr/local/cuda-5.0/include/channel_descriptor.h \ + /usr/local/cuda-5.0/include/cuda_runtime_api.h \ + /usr/local/cuda-5.0/include/driver_functions.h \ + /usr/local/cuda-5.0/include/vector_functions.h \ + /usr/local/cuda-5.0/include/cublas_api.h \ + /usr/local/cuda-5.0/include/cuComplex.h ../KaldiLib/Timer.h \ + /usr/include/sys/time.h ../CuBaseLib/cucommon.h ../CuBaseLib/cuvector.h \ + ../CuBaseLib/cuvector.tcc ../CuBaseLib/cumatrix.h \ + ../CuBaseLib/cudevice.h /usr/include/c++/4.6/map \ + /usr/include/c++/4.6/bits/stl_tree.h /usr/include/c++/4.6/bits/stl_map.h \ + /usr/include/c++/4.6/bits/stl_multimap.h diff --git a/src/CuTNetLib/.svn/entries b/src/CuTNetLib/.svn/entries new file mode 100644 index 0000000..8846e9b --- /dev/null +++ b/src/CuTNetLib/.svn/entries @@ -0,0 +1,946 @@ +10 + +dir +117 +svn+ssh://merlin.fit.vutbr.cz/svn/TNet/trunk/src/CuTNetLib +svn+ssh://merlin.fit.vutbr.cz/svn/TNet + + + +2012-02-07T17:50:53.635354Z +103 +iveselyk + + + + + + + + + + + + + + +bda6da93-004a-4ae9-8e07-715c10848801 + +cuCache.cc +file + + + + +2012-04-02T13:49:14.000000Z +ba7400672c1166f91f51b1d02b0124ab +2011-12-22T15:49:51.623339Z +96 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +5190 + +cuActivation.h +file + + + + +2012-04-02T13:49:14.000000Z +a6e665ef8082542cddeb9cfb344f01d2 +2011-09-26T14:48:24.142069Z +73 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +2417 + +cuComponent.h +file + + + + +2012-04-02T13:49:14.000000Z +410e86f76bb21318f8e348f7287de915 +2012-02-07T17:50:53.635354Z +103 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +8634 + +cuCache.h +file + + + + +2012-04-02T13:49:14.000000Z +2d53b9558d6ce6627a31f4c76399905e +2011-04-04T17:14:16.666438Z +46 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +2021 + +cuCRBEDctFeat.h +file + + + + +2012-04-02T13:49:14.000000Z +02cfc50560d325a7b919e321c854a888 +2011-09-26T14:48:24.142069Z +73 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +7633 + +cuBiasedLinearity.cc +file + + + + +2012-04-02T13:49:14.000000Z +293eda5bda355fdea6417f94c3657a08 +2011-04-04T17:14:16.666438Z +46 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +3082 + +cuRecurrent.cc +file + + + + +2012-04-02T13:49:14.000000Z +45994d12f52bfb36ecc3b40d041073d0 +2011-03-22T21:01:03.678832Z +41 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +5721 + +cuBiasedLinearity.h +file + + + + +2012-04-02T13:49:14.000000Z +df1f7b17849004a0574744a43befabc8 +2011-09-26T14:48:24.142069Z +73 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +1846 + +cuRecurrent.h +file + + + + +2012-04-02T13:49:14.000000Z +f0005036c7deeb217115a673e6fff070 +2011-09-26T14:48:24.142069Z +73 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +2002 + +cuSparseLinearity.cc +file + + + + +2012-04-02T13:49:14.000000Z +e7121409664c29ea00a3abc0b3bd6caa +2011-09-26T13:47:57.076756Z +70 +iveselyk + + + + + + + + + + + + + + + + + + + + + +4896 + +cuObjectiveFunction.cc +file + + + + +2012-04-02T13:49:14.000000Z +fbedbd40c62459261f9f694f8bf62c29 +2011-03-07T10:43:43.160610Z +40 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +2128 + +cuRbm.cc +file + + + + +2012-04-02T13:49:14.000000Z +e849f983f4533fd34b87295e4d93602f +2011-10-11T11:00:50.704096Z +81 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +6654 + +cuSparseLinearity.h +file + + + + +2012-04-02T13:49:14.000000Z +0d38089478e85f485e737a1f5a5e24ad +2011-09-26T14:48:24.142069Z +73 +iveselyk + + + + + + + + + + + + + + + + + + + + + +2570 + +cuObjectiveFunction.h +file + + + + +2012-04-02T13:49:14.000000Z +ba18c448975a869560b72311cefdca58 +2011-04-04T17:14:16.666438Z +46 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +3792 + +cuRbm.h +file + + + + +2012-04-02T13:49:14.000000Z +065ff585dcb58651e1084e90a447290a +2011-12-08T10:59:03.566125Z +94 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +3522 + +cuDiscreteLinearity.cc +file + + + + +2012-04-02T13:49:14.000000Z +d5580b34ae83cdadd40902f705edb990 +2011-12-08T10:59:03.566125Z +94 +iveselyk + + + + + + + + + + + + + + + + + + + + + +4235 + +cuDiscreteLinearity.h +file + + + + +2012-04-02T13:49:14.000000Z +1bc0c4c0ab790c9585a272fb63319041 +2011-12-08T10:59:03.566125Z +94 +iveselyk + + + + + + + + + + + + + + + + + + + + + +1971 + +cuSharedLinearity.cc +file + + + + +2012-04-02T13:49:14.000000Z +ee6608a11be7ded2ba5c60b009e4846a +2011-04-04T17:14:16.666438Z +46 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +5282 + +cuNetwork.cc +file + + + + +2012-04-02T13:49:14.000000Z +6bd052ddb4d464abf3dc8ef5ef17a4b3 +2012-02-07T17:50:53.635354Z +103 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +9320 + +cuBlockArray.cc +file + + + + +2012-04-02T13:49:14.000000Z +641f990dfe9ed2666cf076a43982556e +2012-02-07T17:50:53.635354Z +103 +iveselyk + + + + + + + + + + + + + + + + + + + + + +3240 + +cuSharedLinearity.h +file + + + + +2012-04-02T13:49:14.000000Z +fa4ddac3fab6f14d678184aef599c269 +2011-09-26T14:48:24.142069Z +73 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +1776 + +cuNetwork.h +file + + + + +2012-04-02T13:49:14.000000Z +af7fdb7f8e37a6159ed4802eba57e25b +2011-09-19T11:12:27.685840Z +69 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +5499 + +cuBlockArray.h +file + + + + +2012-04-02T13:49:14.000000Z +0edf09ffe051998c7f2dd04fec19914c +2012-02-07T17:50:53.635354Z +103 +iveselyk + + + + + + + + + + + + + + + + + + + + + +1529 + +cuRbmSparse.cc +file + + + + +2012-04-02T13:49:14.000000Z +626fd92e997ff3c18dbecef8150c5e80 +2011-12-08T10:59:03.566125Z +94 +iveselyk + + + + + + + + + + + + + + + + + + + + + +7681 + +cuRbmSparse.h +file + + + + +2012-04-02T13:49:14.000000Z +a8c1d1d2a7dcf461c4180a1eb34c7991 +2011-12-08T10:59:03.566125Z +94 +iveselyk + + + + + + + + + + + + + + + + + + + + + +3500 + +cuActivation.cc +file + + + + +2012-04-02T13:49:14.000000Z +4c43c28a4c87e7fddad7958a3cfe0e1f +2011-02-24T12:12:08.754106Z +34 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +694 + +Makefile +file + + + + +2012-04-02T13:49:14.000000Z +57c55cf8cb29bdb2590cfb9d6ba02413 +2011-03-24T17:03:17.103393Z +43 +iveselyk +has-props + + + + + + + + + + + + + + + + + + + + +383 + diff --git a/src/CuTNetLib/.svn/prop-base/Makefile.svn-base b/src/CuTNetLib/.svn/prop-base/Makefile.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/Makefile.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuActivation.cc.svn-base b/src/CuTNetLib/.svn/prop-base/cuActivation.cc.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuActivation.cc.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuActivation.h.svn-base b/src/CuTNetLib/.svn/prop-base/cuActivation.h.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuActivation.h.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuBiasedLinearity.cc.svn-base b/src/CuTNetLib/.svn/prop-base/cuBiasedLinearity.cc.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuBiasedLinearity.cc.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuBiasedLinearity.h.svn-base b/src/CuTNetLib/.svn/prop-base/cuBiasedLinearity.h.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuBiasedLinearity.h.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuCRBEDctFeat.h.svn-base b/src/CuTNetLib/.svn/prop-base/cuCRBEDctFeat.h.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuCRBEDctFeat.h.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuCache.cc.svn-base b/src/CuTNetLib/.svn/prop-base/cuCache.cc.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuCache.cc.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuCache.h.svn-base b/src/CuTNetLib/.svn/prop-base/cuCache.h.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuCache.h.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuComponent.h.svn-base b/src/CuTNetLib/.svn/prop-base/cuComponent.h.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuComponent.h.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuNetwork.cc.svn-base b/src/CuTNetLib/.svn/prop-base/cuNetwork.cc.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuNetwork.cc.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuNetwork.h.svn-base b/src/CuTNetLib/.svn/prop-base/cuNetwork.h.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuNetwork.h.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuObjectiveFunction.cc.svn-base b/src/CuTNetLib/.svn/prop-base/cuObjectiveFunction.cc.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuObjectiveFunction.cc.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuObjectiveFunction.h.svn-base b/src/CuTNetLib/.svn/prop-base/cuObjectiveFunction.h.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuObjectiveFunction.h.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuRbm.cc.svn-base b/src/CuTNetLib/.svn/prop-base/cuRbm.cc.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuRbm.cc.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuRbm.h.svn-base b/src/CuTNetLib/.svn/prop-base/cuRbm.h.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuRbm.h.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuRecurrent.cc.svn-base b/src/CuTNetLib/.svn/prop-base/cuRecurrent.cc.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuRecurrent.cc.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuRecurrent.h.svn-base b/src/CuTNetLib/.svn/prop-base/cuRecurrent.h.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuRecurrent.h.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuSharedLinearity.cc.svn-base b/src/CuTNetLib/.svn/prop-base/cuSharedLinearity.cc.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuSharedLinearity.cc.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/prop-base/cuSharedLinearity.h.svn-base b/src/CuTNetLib/.svn/prop-base/cuSharedLinearity.h.svn-base new file mode 100644 index 0000000..eff4929 --- /dev/null +++ b/src/CuTNetLib/.svn/prop-base/cuSharedLinearity.h.svn-base @@ -0,0 +1,5 @@ +K 12 +svn:keywords +V 23 +Date Revision Author Id +END diff --git a/src/CuTNetLib/.svn/text-base/Makefile.svn-base b/src/CuTNetLib/.svn/text-base/Makefile.svn-base new file mode 100644 index 0000000..c7678da --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/Makefile.svn-base @@ -0,0 +1,30 @@ + +include ../tnet.mk + +INCLUDE= -I. -I../ -I../KaldiLib -I../CuBaseLib -I../TNetLib -I$(CUDA_TK_BASE)/include/ + + +all : libCuTNet.a + +libCuTNet.a : $(OBJ) + $(AR) ruv $@ $? + $(RANLIB) $@ + + +%.o : %.cc + $(CXX) -c $< -o $@ $(CFLAGS) $(CXXFLAGS) $(INCLUDE) + + + + +.PHONY: clean depend + +clean : + rm -f *.o *.a + + +depend: + $(CXX) -M $(CXXFLAGS) *.cc $(INCLUDE) > .depend.mk + +-include .depend.mk + diff --git a/src/CuTNetLib/.svn/text-base/cuActivation.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuActivation.cc.svn-base new file mode 100644 index 0000000..bd57ae5 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuActivation.cc.svn-base @@ -0,0 +1,46 @@ + +#include "cuActivation.h" +#include "cumath.h" + + +namespace TNet { + + + void + CuSigmoid:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + CuMath<BaseFloat>::Sigmoid(Y, X); + } + + + void + CuSigmoid:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + CuMath<BaseFloat>::DiffSigmoid(Y, X, mOutput); + } + + + + void + CuSoftmax:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + CuMath<BaseFloat>::Softmax(Y,X); + } + + + + void + CuSoftmax:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + //we assume X is already dE/dSoftmax_input + Y.CopyFrom(X); + } + + + +} //namespace + diff --git a/src/CuTNetLib/.svn/text-base/cuActivation.h.svn-base b/src/CuTNetLib/.svn/text-base/cuActivation.h.svn-base new file mode 100644 index 0000000..9fb2862 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuActivation.h.svn-base @@ -0,0 +1,123 @@ + +#ifndef _CUACT_FUN_I_ +#define _CUACT_FUN_I_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +namespace TNet +{ + + /** + * Common interface for activation functions + */ + class CuActivation : public CuComponent + { + public: + CuActivation(size_t nInputs, size_t nOutputs, CuComponent *pPred); + + protected: + }; + + + /** + * Sigmoid activation function + */ + class CuSigmoid : public CuActivation + { + public: + CuSigmoid(size_t nInputs, size_t nOutputs, CuComponent *pPred); + + ComponentType GetType() const; + const char* GetName() const; + + protected: + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + }; + + + /** + * Softmax activation function + */ + class CuSoftmax : public CuActivation + { + public: + CuSoftmax(size_t nInputs, size_t nOutputs, CuComponent *pPred); + + ComponentType GetType() const; + const char* GetName() const; + + protected: + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + }; + + + ////////////////////////////////////////////////////////////////////////// + // Inline functions + // Activation:: + inline + CuActivation:: + CuActivation(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuComponent(nInputs,nOutputs, pPred) + { + assert(nInputs == nOutputs); + } + + + ////////////////////////////////////////////////////////////////////////// + // Inline functions + // Sigmoid:: + inline + CuSigmoid:: + CuSigmoid(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuActivation(nInputs,nOutputs, pPred) + { } + + inline CuComponent::ComponentType + CuSigmoid:: + GetType() const + { + return CuComponent::SIGMOID; + } + + inline const char* + CuSigmoid:: + GetName() const + { + return "<sigmoid>"; + } + + + + ////////////////////////////////////////////////////////////////////////// + // Inline functions + // Softmax:: + inline + CuSoftmax:: + CuSoftmax(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuActivation(nInputs,nOutputs, pPred) + { } + + inline CuComponent::ComponentType + CuSoftmax:: + GetType() const + { + return CuComponent::SOFTMAX; + } + + inline const char* + CuSoftmax:: + GetName() const + { + return "<softmax>"; + } + + +} //namespace + + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuBiasedLinearity.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuBiasedLinearity.cc.svn-base new file mode 100644 index 0000000..b9ac137 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuBiasedLinearity.cc.svn-base @@ -0,0 +1,123 @@ + + +#include "cuBiasedLinearity.h" + + +namespace TNet +{ + + void + CuBiasedLinearity:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + //Y.SetConst(0.0); + Y.AddScaledRow(1.0,mBias,0.0); + Y.Gemm('N','N', 1.0, X, mLinearity, 1.0); + } + + + void + CuBiasedLinearity:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + //Y.SetConst(0.0); + Y.Gemm('N', 'T', 1.0, X, mLinearity, 0.0); + } + + + void + CuBiasedLinearity:: + Update() + { +#if 0 + //former implementation + BaseFloat N = static_cast<BaseFloat>(GetInput().Rows()); + + mLinearityCorrection.Gemm('T','N',-mLearningRate/N,GetInput(),GetErrorInput(),mMomentum); + mBiasCorrection.AddColSum(-mLearningRate/N,GetErrorInput(),mMomentum); + + //regularization weight decay + mLinearityCorrection.AddScaled(-mLearningRate*mWeightcost,mLinearity,1.0); + + mLinearity.AddScaled(1.0,mLinearityCorrection,1.0); + mBias.AddScaled(1.0,mBiasCorrection,1.0); +#endif + +#if 1 + //new implementation + BaseFloat N = 1; + if(mGradDivFrm) { + N = static_cast<BaseFloat>(GetInput().Rows()); + } + BaseFloat mmt_gain = static_cast<BaseFloat>(1.0/(1.0-mMomentum)); + N *= mmt_gain; + + mLinearityCorrection.Gemm('T','N',1.0,GetInput(),GetErrorInput(),mMomentum); + mBiasCorrection.AddColSum(1.0,GetErrorInput(),mMomentum); + + mLinearity.AddScaled(-mLearningRate/N,mLinearityCorrection,1.0); + mBias.AddScaled(-mLearningRate/N,mBiasCorrection,1.0); + + //regularization weight decay (from actual weights only) + BaseFloat L2_decay = -mLearningRate*mWeightcost*(mGradDivFrm?1.0:GetInput().Rows()); + mLinearity.AddScaled(L2_decay, mLinearity,1.0); +#endif + } + + + void + CuBiasedLinearity:: + ReadFromStream(std::istream& rIn) + { + //matrix is stored transposed as SNet does + BfMatrix transpose; + rIn >> transpose; + mLinearity.CopyFrom(BfMatrix(transpose, TRANS)); + //biases stored normally + BfVector bias; + rIn >> bias; + mBias.CopyFrom(bias); + + if(transpose.Cols()*transpose.Rows() == 0) { + Error("Missing linearity matrix in network file"); + } + if(bias.Dim() == 0) { + Error("Missing bias vector in network file"); + } + if(mLinearity.Cols() != GetNOutputs() || + mLinearity.Rows() != GetNInputs() || + mBias.Dim() != GetNOutputs() + ){ + std::ostringstream os; + os << "Wrong dimensionalities of matrix/vector in network file\n" + << "Inputs:" << GetNInputs() + << "Outputs:" << GetNOutputs() + << "\n" + << "linearityCols:" << mLinearity.Cols() + << "linearityRows:" << mLinearity.Rows() + << "biasDims:" << mBias.Dim() + << "\n"; + Error(os.str()); + } + } + + + void + CuBiasedLinearity:: + WriteToStream(std::ostream& rOut) + { + //matrix is stored transposed as SNet does + BfMatrix tmp; + mLinearity.CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + //biases stored normally + BfVector vec; + mBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + } + + +} //namespace + diff --git a/src/CuTNetLib/.svn/text-base/cuBiasedLinearity.h.svn-base b/src/CuTNetLib/.svn/text-base/cuBiasedLinearity.h.svn-base new file mode 100644 index 0000000..18a75ac --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuBiasedLinearity.h.svn-base @@ -0,0 +1,85 @@ +#ifndef _CUBIASED_LINEARITY_H_ +#define _CUBIASED_LINEARITY_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + + class CuBiasedLinearity : public CuUpdatableComponent + { + public: + + CuBiasedLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuBiasedLinearity(); + + ComponentType GetType() const; + const char* GetName() const; + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + void Update(); + + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + CuMatrix<BaseFloat> mLinearity; ///< Matrix with neuron weights + CuVector<BaseFloat> mBias; ///< Vector with biases + + CuMatrix<BaseFloat> mLinearityCorrection; ///< Matrix for linearity updates + CuVector<BaseFloat> mBiasCorrection; ///< Vector for bias updates + + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuBiasedLinearity:: + inline + CuBiasedLinearity:: + CuBiasedLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred), + mLinearity(nInputs,nOutputs), mBias(nOutputs), + mLinearityCorrection(nInputs,nOutputs), mBiasCorrection(nOutputs) + { + mLinearityCorrection.SetConst(0.0); + mBiasCorrection.SetConst(0.0); + } + + + inline + CuBiasedLinearity:: + ~CuBiasedLinearity() + { } + + inline CuComponent::ComponentType + CuBiasedLinearity:: + GetType() const + { + return CuComponent::BIASED_LINEARITY; + } + + inline const char* + CuBiasedLinearity:: + GetName() const + { + return "<biasedlinearity>"; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuBlockArray.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuBlockArray.cc.svn-base new file mode 100644 index 0000000..461bd37 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuBlockArray.cc.svn-base @@ -0,0 +1,139 @@ + + +#include "cuBlockArray.h" +#include "cuNetwork.h" + + +namespace TNet +{ + + void + CuBlockArray:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + CuMatrix<BaseFloat> colsX; + CuMatrix<BaseFloat> colsY; + + int X_src_ori=0, Y_tgt_ori=0; + for(int i=0; i<mNBlocks; i++) { + //copy column stripe from the input X + int colsX_cnt=mBlocks[i]->GetNInputs(); + colsX.Init(X.Rows(),colsX_cnt); + colsX.CopyCols(colsX_cnt,X_src_ori,X,0); + + //propagate through the block(network) + mBlocks[i]->Propagate(colsX,colsY); + + //copy column stripe to the output Y + int colsY_cnt=mBlocks[i]->GetNOutputs(); + Y.CopyCols(colsY_cnt,0,colsY,Y_tgt_ori); + + //shift the origin coordinates + X_src_ori += colsX_cnt; + Y_tgt_ori += colsY_cnt; + } + + assert(X_src_ori == X.Cols()); + assert(Y_tgt_ori == Y.Cols()); + } + + + void + CuBlockArray:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + KALDI_ERR << "Unimplemented"; + } + + + void + CuBlockArray:: + Update() + { + KALDI_ERR << "Unimplemented"; + } + + + void + CuBlockArray:: + ReadFromStream(std::istream& rIn) + { + if(mBlocks.size() > 0) { + KALDI_ERR << "Cannot read block vector, " + << "aleady filled bt " + << mBlocks.size() + << "elements"; + } + + rIn >> std::ws >> mNBlocks; + if(mNBlocks < 1) { + KALDI_ERR << "Bad number of blocks:" << mNBlocks; + } + + //read all the blocks + std::string tag; + int block_id; + for(int i=0; i<mNBlocks; i++) { + //read tag <block> + rIn >> std::ws >> tag; + //make it lowercase + std::transform(tag.begin(), tag.end(), tag.begin(), tolower); + //check + if(tag!="<block>") { + KALDI_ERR << "<block> keywotd expected"; + } + + //read block number + rIn >> std::ws >> block_id; + if(block_id != i+1) { + KALDI_ERR << "Expected block number:" << i+1 + << " read block number: " << block_id; + } + + //read the nnet + CuNetwork* p_nnet = new CuNetwork; + p_nnet->ReadNetwork(rIn); + if(p_nnet->Layers() == 0) { + KALDI_ERR << "Cannot read empty network to a block"; + } + + //add it to the vector + mBlocks.push_back(p_nnet); + } + + //check the declared dimensionality + int sum_inputs=0, sum_outputs=0; + for(int i=0; i<mNBlocks; i++) { + sum_inputs += mBlocks[i]->GetNInputs(); + sum_outputs += mBlocks[i]->GetNOutputs(); + } + if(sum_inputs != GetNInputs()) { + KALDI_ERR << "Non-matching number of INPUTS! Declared:" + << GetNInputs() + << " summed from blocks" + << sum_inputs; + } + if(sum_outputs != GetNOutputs()) { + KALDI_ERR << "Non-matching number of OUTPUTS! Declared:" + << GetNOutputs() + << " summed from blocks" + << sum_outputs; + } + } + + + void + CuBlockArray:: + WriteToStream(std::ostream& rOut) + { + rOut << " " << mBlocks.size() << " "; + for(int i=0; i<mBlocks.size(); i++) { + rOut << "<block> " << i+1 << "\n"; + mBlocks[i]->WriteNetwork(rOut); + rOut << "<endblock>\n"; + } + } + + +} //namespace + diff --git a/src/CuTNetLib/.svn/text-base/cuBlockArray.h.svn-base b/src/CuTNetLib/.svn/text-base/cuBlockArray.h.svn-base new file mode 100644 index 0000000..aea7922 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuBlockArray.h.svn-base @@ -0,0 +1,83 @@ +#ifndef _CUBLOCK_ARRAY_H_ +#define _CUBLOCK_ARRAY_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + + class CuNetwork; + + class CuBlockArray : public CuUpdatableComponent + { + public: + + CuBlockArray(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuBlockArray(); + + ComponentType GetType() const; + const char* GetName() const; + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + void Update(); + + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + std::vector<CuNetwork*> mBlocks; ///< vector with networks, one network is one block + size_t mNBlocks; + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuBlockArray:: + inline + CuBlockArray:: + CuBlockArray(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred), + mNBlocks(0) + { } + + + inline + CuBlockArray:: + ~CuBlockArray() + { + for(int i=0; i<mBlocks.size(); i++) { + delete mBlocks[i]; + } + mBlocks.clear(); + } + + inline CuComponent::ComponentType + CuBlockArray:: + GetType() const + { + return CuComponent::BLOCK_ARRAY; + } + + inline const char* + CuBlockArray:: + GetName() const + { + return "<blockarray>"; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuCRBEDctFeat.h.svn-base b/src/CuTNetLib/.svn/text-base/cuCRBEDctFeat.h.svn-base new file mode 100644 index 0000000..62c62f2 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuCRBEDctFeat.h.svn-base @@ -0,0 +1,310 @@ +#ifndef _CUCRBEDCTFEATURES_H_ +#define _CUCRBEDCTFEATURES_H_ + + +#include "cuComponent.h" +#include "cumath.h" + + +namespace TNet { + + /** + * Expands the time context of the input features + * in N, out k*N, FrameOffset o_1,o_2,...,o_k + * FrameOffset example 11frames: -5 -4 -3 -2 -1 0 1 2 3 4 5 + */ + class CuExpand : public CuComponent + { + public: + CuExpand(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred) + { } + + ~CuExpand() + { } + + ComponentType GetType() const + { return EXPAND; } + + const char* GetName() const + { return "<expand>"; } + + void ReadFromStream(std::istream& rIn) + { Vector<int> vec; rIn >> vec; mFrameOffset.CopyFrom(vec); } + + void WriteToStream(std::ostream& rOut) + { Vector<int> vec; mFrameOffset.CopyTo(vec); rOut << vec; } + + protected: + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { CuMath<BaseFloat>::Expand(Y,X,mFrameOffset); } + + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Error("__func__ Nonsense"); } + + protected: + CuVector<int> mFrameOffset; + }; + + + + /** + * Rearrange the matrix columns according to the indices in mCopyFromIndices + */ + class CuCopy : public CuComponent + { + public: + CuCopy(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred) + { } + + ~CuCopy() + { } + + ComponentType GetType() const + { return COPY; } + + const char* GetName() const + { return "<copy>"; } + + void ReadFromStream(std::istream& rIn) + { Vector<int> vec; rIn >> vec; vec.Add(-1); mCopyFromIndices.CopyFrom(vec); } + + void WriteToStream(std::ostream& rOut) + { Vector<int> vec; mCopyFromIndices.CopyTo(vec); vec.Add(1); rOut << vec; } + + protected: + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { CuMath<BaseFloat>::Rearrange(Y,X,mCopyFromIndices); } + + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Error("__func__ Nonsense"); } + + protected: + CuVector<int> mCopyFromIndices; + }; + + class CuTranspose : public CuComponent + { + public: + CuTranspose(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred), mContext(0) + { } + + ~CuTranspose() + { } + + ComponentType GetType() const + { return TRANSPOSE; } + + const char* GetName() const + { return "<transpose>"; } + + void ReadFromStream(std::istream& rIn) + { + rIn >> std::ws >> mContext; + + if(GetNInputs() != GetNOutputs()) { + Error("Input dim must be same as output dim"); + } + if(GetNInputs() % mContext != 0) { + Error("Number of inputs must be divisible by context length"); + } + + Vector<int> vec(GetNInputs()); + int channels = GetNInputs() / mContext; + for(int i=0, ch=0; ch<channels; ch++) { + for(int idx=ch; idx < (int)GetNInputs(); idx+=channels, i++) { + assert(i < (int)GetNInputs()); + vec[i] = idx; + } + } + + mCopyFromIndices.CopyFrom(vec); + } + + void WriteToStream(std::ostream& rOut) + { rOut << " " << mContext << "\n"; } + + protected: + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { CuMath<BaseFloat>::Rearrange(Y,X,mCopyFromIndices); } + + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Error("__func__ Nonsense"); } + + protected: + int mContext; + CuVector<int> mCopyFromIndices; + }; + + + /** + * CuBlockLinearity is used for the blockwise multiplication by + * DCT transform loaded from disk + */ + class CuBlockLinearity : public CuComponent + { + public: + CuBlockLinearity(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred) + { } + + ~CuBlockLinearity() + { } + + + ComponentType GetType() const + { return CuComponent::BLOCK_LINEARITY; } + + const char* GetName() const + { return "<blocklinearity>"; } + + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { CuMath<BaseFloat>::BlockLinearity(Y,X,mBlockLinearity); } + + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Error("__func__ Not implemented"); } + + + void ReadFromStream(std::istream& rIn) + { + Matrix<BaseFloat> mat; + rIn >> mat; + Matrix<BaseFloat> trans(mat,TRANS); + mBlockLinearity.CopyFrom(trans); + + if((GetNOutputs() % mBlockLinearity.Cols() != 0) || + (GetNInputs() % mBlockLinearity.Rows() != 0) || + ((GetNOutputs() / mBlockLinearity.Cols()) != + (GetNInputs() / mBlockLinearity.Rows()))) + { + Error("BlockLinearity matrix dimensions must divide IO dims"); + } + } + + void WriteToStream(std::ostream& rOut) + { + Matrix<BaseFloat> mat; + mBlockLinearity.CopyTo(mat); + Matrix<BaseFloat> trans(mat,TRANS); + rOut << trans; + } + + private: + CuMatrix<BaseFloat> mBlockLinearity; + }; + + + + class CuBias : public CuComponent + { + public: + CuBias(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred) + { } + + ~CuBias() + { } + + + ComponentType GetType() const + { return CuComponent::BIAS; } + + const char* GetName() const + { return "<bias>"; } + + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Y.CopyFrom(X); Y.AddScaledRow(1.0, mBias, 1.0); } + + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Y.CopyFrom(X); } + + + void ReadFromStream(std::istream& rIn) + { Vector<BaseFloat> vec; rIn >> vec; mBias.CopyFrom(vec); } + + void WriteToStream(std::ostream& rOut) + { Vector<BaseFloat> vec; mBias.CopyTo(vec); rOut << vec; } + + private: + CuVector<BaseFloat> mBias; + }; + + + + class CuWindow : public CuComponent + { + public: + CuWindow(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs, nOutputs, pPred) + { } + + ~CuWindow() + { } + + + ComponentType GetType() const + { return CuComponent::WINDOW; } + + const char* GetName() const + { return "<window>"; } + + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Y.CopyFrom(X); Y.ScaleCols(mWindow); } + + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Error("__func__ Not implemented"); } + + + void ReadFromStream(std::istream& rIn) + { Vector<BaseFloat> vec; rIn >> vec; mWindow.CopyFrom(vec); } + + void WriteToStream(std::ostream& rOut) + { Vector<BaseFloat> vec; mWindow.CopyTo(vec); rOut << vec; } + + private: + CuVector<BaseFloat> mWindow; + }; + + class CuLog : public CuComponent + { + public: + CuLog(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs, nOutputs, pPred) + { } + + ~CuLog() + { } + + + ComponentType GetType() const + { return CuComponent::LOG; } + + const char* GetName() const + { return "<log>"; } + + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Y.CopyFrom(X); Y.ApplyLog(); } + + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Error("__func__ Not implemented"); } + + + void ReadFromStream(std::istream& rIn) + { } + + void WriteToStream(std::ostream& rOut) + { } + + }; + +} + + +#endif + diff --git a/src/CuTNetLib/.svn/text-base/cuCache.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuCache.cc.svn-base new file mode 100644 index 0000000..f96b3b1 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuCache.cc.svn-base @@ -0,0 +1,203 @@ + + +#include "cuCache.h" +#include "cumath.h" + + + +namespace TNet { + + CuCache:: + CuCache() + : mState(EMPTY), mIntakePos(0), mExhaustPos(0), mDiscarded(0), + mRandomized(false), mTrace(0) + { } + + CuCache:: + ~CuCache() + { } + + void + CuCache:: + Init(size_t cachesize, size_t bunchsize) + { + if((cachesize % bunchsize) != 0) { + Error("Non divisible cachesize by bunchsize"); + } + + mCachesize = cachesize; + mBunchsize = bunchsize; + + mState = EMPTY; + + mIntakePos = 0; + mExhaustPos = 0; + + mRandomized = false; + + } + + void + CuCache:: + AddData(const CuMatrix<BaseFloat>& rFeatures, const CuMatrix<BaseFloat>& rDesired) + { + assert(rFeatures.Rows() == rDesired.Rows()); + + //lazy buffers allocation + if(mFeatures.Rows() != mCachesize) { + mFeatures.Init(mCachesize,rFeatures.Cols()); + mDesired.Init(mCachesize,rDesired.Cols()); + } + + //warn if segment longer than half-cache + if(rFeatures.Rows() > mCachesize/2) { + std::ostringstream os; + os << "Too long segment and small feature cache! " + << " cachesize: " << mCachesize + << " segmentsize: " << rFeatures.Rows(); + Warning(os.str()); + } + + //change state + if(mState == EMPTY) { + if(mTrace&3) std::cout << "/" << std::flush; + mState = INTAKE; mIntakePos = 0; + + //check for leftover from previous segment + int leftover = mFeaturesLeftover.Rows(); + //check if leftover is not bigger than cachesize + if(leftover > mCachesize) { + std::ostringstream os; + os << "Too small feature cache: " << mCachesize + << ", truncating: " + << leftover - mCachesize << " frames from previous segment leftover"; + //Error(os.str()); + Warning(os.str()); + leftover = mCachesize; + } + //prefill cache with leftover + if(leftover > 0) { + mFeatures.CopyRows(leftover,0,mFeaturesLeftover,0); + mDesired.CopyRows(leftover,0,mDesiredLeftover,0); + mFeaturesLeftover.Destroy(); + mDesiredLeftover.Destroy(); + mIntakePos += leftover; + } + } + + assert(mState == INTAKE); + assert(rFeatures.Rows() == rDesired.Rows()); + if(mTrace&2) std::cout << "F" << std::flush; + + int cache_space = mCachesize - mIntakePos; + int feature_length = rFeatures.Rows(); + int fill_rows = (cache_space<feature_length)? cache_space : feature_length; + int leftover = feature_length - fill_rows; + + assert(cache_space > 0); + + //copy the data to cache + mFeatures.CopyRows(fill_rows,0,rFeatures,mIntakePos); + mDesired.CopyRows(fill_rows,0,rDesired,mIntakePos); + + //copy leftovers + if(leftover > 0) { + mFeaturesLeftover.Init(leftover,mFeatures.Cols()); + mDesiredLeftover.Init(leftover,mDesired.Cols()); + mFeaturesLeftover.CopyRows(leftover,fill_rows,rFeatures,0); + mDesiredLeftover.CopyRows(leftover,fill_rows,rDesired,0); + } + + //update cursor + mIntakePos += fill_rows; + + //change state + if(mIntakePos == mCachesize) { + if(mTrace&3) std::cout << "\\" << std::flush; + mState = FULL; + } + } + + + + void + CuCache:: + Randomize() + { + assert(mState == FULL || mState == INTAKE); + + if(mTrace&3) std::cout << "R" << std::flush; + + //lazy initialization of hte output buffers + mFeaturesRandom.Init(mCachesize,mFeatures.Cols()); + mDesiredRandom.Init(mCachesize,mDesired.Cols()); + + //generate random series of integers + Vector<int> randmask(mIntakePos); + for(unsigned int i=0; i<mIntakePos; i++) { + randmask[i]=i; + } + int* ptr = randmask.pData(); + std::random_shuffle(ptr, ptr+mIntakePos, GenerateRandom); + + CuVector<int> cu_randmask; + cu_randmask.CopyFrom(randmask); + + //randomize + CuMath<BaseFloat>::Randomize(mFeaturesRandom,mFeatures,cu_randmask); + CuMath<BaseFloat>::Randomize(mDesiredRandom,mDesired,cu_randmask); + + mRandomized = true; + + } + + void + CuCache:: + GetBunch(CuMatrix<BaseFloat>& rFeatures, CuMatrix<BaseFloat>& rDesired) + { + if(mState == EMPTY) { + Error("GetBunch on empty cache!!!"); + } + + //change state if full... + if(mState == FULL) { + if(mTrace&3) std::cout << "\\" << std::flush; + mState = EXHAUST; mExhaustPos = 0; + } + + //final cache is not completely filled + if(mState == INTAKE) //&& mpFeatures->EndOfList() + { + if(mTrace&3) std::cout << "\\-LAST\n" << std::flush; + mState = EXHAUST; mExhaustPos = 0; + } + + assert(mState == EXHAUST); + + //init the output + rFeatures.Init(mBunchsize,mFeatures.Cols()); + rDesired.Init(mBunchsize,mDesired.Cols()); + + //copy the output + if(mRandomized) { + rFeatures.CopyRows(mBunchsize,mExhaustPos,mFeaturesRandom,0); + rDesired.CopyRows(mBunchsize,mExhaustPos,mDesiredRandom,0); + } else { + rFeatures.CopyRows(mBunchsize,mExhaustPos,mFeatures,0); + rDesired.CopyRows(mBunchsize,mExhaustPos,mDesired,0); + } + + //update cursor + mExhaustPos += mBunchsize; + + //change state to EMPTY + if(mExhaustPos > mIntakePos-mBunchsize) { + //we don't have more complete bunches... + mDiscarded += mIntakePos - mExhaustPos; + + mState = EMPTY; + } + } + + +} diff --git a/src/CuTNetLib/.svn/text-base/cuCache.h.svn-base b/src/CuTNetLib/.svn/text-base/cuCache.h.svn-base new file mode 100644 index 0000000..cef2dd9 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuCache.h.svn-base @@ -0,0 +1,74 @@ +#ifndef _CUCACHE_H_ +#define _CUCACHE_H_ + +#include "cumatrix.h" + +namespace TNet { + + + /** + * The feature-target pair cache + */ + class CuCache { + typedef enum { EMPTY, INTAKE, FULL, EXHAUST } State; + public: + CuCache(); + ~CuCache(); + + /// Initialize the cache + void Init(size_t cachesize, size_t bunchsize); + + /// Add data to cache, returns number of added vectors + void AddData(const CuMatrix<BaseFloat>& rFeatures, const CuMatrix<BaseFloat>& rDesired); + /// Randomizes the cache + void Randomize(); + /// Get the bunch of training data + void GetBunch(CuMatrix<BaseFloat>& rFeatures, CuMatrix<BaseFloat>& rDesired); + + + /// Returns true if the cache was completely filled + bool Full() + { return (mState == FULL); } + + /// Returns true if the cache is empty + bool Empty() + { return (mState == EMPTY || mIntakePos < mBunchsize); } + + /// Number of discarded frames + int Discarded() + { return mDiscarded; } + + /// Set the trace message level + void Trace(int trace) + { mTrace = trace; } + + private: + + static long int GenerateRandom(int max) + { return lrand48() % max; } + + State mState; ///< Current state of the cache + + size_t mIntakePos; ///< Number of intaken vectors by AddData + size_t mExhaustPos; ///< Number of exhausted vectors by GetBunch + + size_t mCachesize; ///< Size of cache + size_t mBunchsize; ///< Size of bunch + int mDiscarded; ///< Number of discarded frames + + CuMatrix<BaseFloat> mFeatures; ///< Feature cache + CuMatrix<BaseFloat> mFeaturesRandom; ///< Feature cache + CuMatrix<BaseFloat> mFeaturesLeftover; ///< Feature cache + + CuMatrix<BaseFloat> mDesired; ///< Desired vector cache + CuMatrix<BaseFloat> mDesiredRandom; ///< Desired vector cache + CuMatrix<BaseFloat> mDesiredLeftover; ///< Desired vector cache + + bool mRandomized; + + int mTrace; + }; + +} + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuComponent.h.svn-base b/src/CuTNetLib/.svn/text-base/cuComponent.h.svn-base new file mode 100644 index 0000000..332c156 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuComponent.h.svn-base @@ -0,0 +1,384 @@ +#ifndef _CUNETWORK_COMPONENT_I_H +#define _CUNETWORK_COMPONENT_I_H + + +#include "Vector.h" +#include "Matrix.h" +#include "Error.h" + +#include "cumatrix.h" + +#include <iostream> +#include <stdexcept> + + +namespace TNet { + + + /** + * Basic element of the network, + * it is a box with defined inputs and outputs, + * and functions to refresh outputs + * + * it is able to compute tranformation function (forward pass) + * and jacobian function (backward pass), + * which is to be implemented in descendents + */ + class CuComponent + { + public: + /// Types of the net components + typedef enum { + UPDATABLE_COMPONENT = 0x0100, + BIASED_LINEARITY, + DISCRETE_LINEARITY, + SHARED_LINEARITY, + SPARSE_LINEARITY, + RBM, + RBM_SPARSE, + RECURRENT, + + ACT_FUN = 0x0200, + SOFTMAX, + SIGMOID, + + OTHER = 0x0400, + EXPAND, + COPY, + TRANSPOSE, + BLOCK_LINEARITY, + WINDOW, + BIAS, + LOG, + + BLOCK_ARRAY, + } ComponentType; + + ////////////////////////////////////////////////////////////// + // Constructor & Destructor + public: + CuComponent(size_t nInputs, size_t nOutputs, CuComponent *pPred); + virtual ~CuComponent(); + + ////////////////////////////////////////////////////////////// + // Interface specification (public) + public: + /// Get Type Identification of the component + virtual ComponentType GetType() const = 0; + /// Get Type Label of the component + virtual const char* GetName() const = 0; + /// + virtual bool IsUpdatable() const + { return false; } + + /// Get size of input vectors + size_t GetNInputs() const; + /// Get size of output vectors + size_t GetNOutputs() const; + + /// IO Data getters + const CuMatrix<BaseFloat>& GetInput() const; + const CuMatrix<BaseFloat>& GetOutput() const; + const CuMatrix<BaseFloat>& GetErrorInput() const; + const CuMatrix<BaseFloat>& GetErrorOutput() const; + + /// Set input vector (bind with the preceding NetworkComponent) + void SetInput(const CuMatrix<BaseFloat>& rInput); + /// Set error input vector (bind with the following NetworkComponent) + void SetErrorInput(const CuMatrix<BaseFloat>& rErrorInput); + + /// Perform forward pass propagateion Input->Output + void Propagate(); + /// Perform backward pass propagateion ErrorInput->ErrorOutput + void Backpropagate(); + + /// Reads the component parameters from stream + virtual void ReadFromStream(std::istream& rIn) { } + /// Writes the components parameters to stream + virtual void WriteToStream(std::ostream& rOut) { } + + + /////////////////////////////////////////////////////////////// + // Nonpublic member functions used to update data outputs + protected: + /// Forward pass transformation (to be implemented by descendents...) + virtual void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) = 0; + /// Backward pass transformation (to be implemented by descendents...) + virtual void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) = 0; + + + /////////////////////////////////////////////////////////////// + // data members + protected: + + size_t mNInputs; ///< Size of input vectors + size_t mNOutputs; ///< Size of output vectors + + const CuMatrix<BaseFloat>* mpInput; ///< inputs are NOT OWNED by component + const CuMatrix<BaseFloat>* mpErrorInput;///< inputs are NOT OWNED by component + + CuMatrix<BaseFloat> mOutput; ///< outputs are OWNED by component + CuMatrix<BaseFloat> mErrorOutput; ///< outputs are OWNED by component + + }; + + + /** + * Class UpdatableComponent is a box which has some + * parameters adjustable by learning + * + * you can set the learning rate, lock the params, + * and learn from each data observation + */ + class CuUpdatableComponent : public CuComponent + { + ////////////////////////////////////////////////////////////// + // Constructor & Destructor + public: + CuUpdatableComponent(size_t nInputs, size_t nOutputs, CuComponent *pPred); + virtual ~CuUpdatableComponent(); + + + ////////////////////////////////////////////////////////////// + // Interface specification (public) + public: + /// + virtual bool IsUpdatable() const + { return true; } + + /// get gradient and update the parameters in one step + virtual void Update() = 0; + + /// Sets the learning rate of gradient descent + void LearnRate(BaseFloat rate); + /// Gets the learning rate of gradient descent + BaseFloat LearnRate(); + + void Momentum(BaseFloat mmt); + BaseFloat Momentum(); + + void Weightcost(BaseFloat cost); + BaseFloat Weightcost(); + + void GradDivFrm(bool div); + bool GradDivFrm(); + + protected: + BaseFloat mLearningRate; + BaseFloat mMomentum; + BaseFloat mWeightcost; + bool mGradDivFrm; + + }; + + + + + ////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuComponent:: + inline + CuComponent:: + CuComponent(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : mNInputs(nInputs), mNOutputs(nOutputs), + mpInput(NULL), mpErrorInput(NULL), + mOutput(), mErrorOutput() + { + /* DOUBLE LINK the Components */ + if (pPred != NULL) { + SetInput(pPred->GetOutput()); + pPred->SetErrorInput(GetErrorOutput()); + } + } + + + inline + CuComponent:: + ~CuComponent() + { + ; + } + + inline void + CuComponent:: + Propagate() + { + //initialize output buffer + mOutput.Init(GetInput().Rows(),GetNOutputs()); + //do the dimensionality test + if(GetNInputs() != GetInput().Cols()) { + KALDI_ERR << "Non-matching INPUT dim!!! Network dim: " << GetNInputs() + << " Data dim: " << GetInput().Cols(); + } + //run transform + PropagateFnc(GetInput(),mOutput); + } + + + inline void + CuComponent:: + Backpropagate() + { + //re-initialize the output buffer + mErrorOutput.Init(GetErrorInput().Rows(),GetNInputs()); + + //do the dimensionality test + assert(GetErrorInput().Cols() == mNOutputs); + assert(mErrorOutput.Cols() == mNInputs); + assert(mErrorOutput.Rows() == GetErrorInput().Rows()); + + //transform + BackpropagateFnc(GetErrorInput(),mErrorOutput); + } + + + inline void + CuComponent:: + SetInput(const CuMatrix<BaseFloat>& rInput) + { + mpInput = &rInput; + } + + + inline void + CuComponent:: + SetErrorInput(const CuMatrix<BaseFloat>& rErrorInput) + { + mpErrorInput = &rErrorInput; + } + + + inline const CuMatrix<BaseFloat>& + CuComponent:: + GetInput() const + { + if (NULL == mpInput) Error("mpInput is NULL"); + return *mpInput; + } + + inline const CuMatrix<BaseFloat>& + CuComponent:: + GetOutput() const + { + return mOutput; + } + + inline const CuMatrix<BaseFloat>& + CuComponent:: + GetErrorInput() const + { + if (NULL == mpErrorInput) Error("mpErrorInput is NULL"); + return *mpErrorInput; + } + + inline const CuMatrix<BaseFloat>& + CuComponent:: + GetErrorOutput() const + { + return mErrorOutput; + } + + inline size_t + CuComponent:: + GetNInputs() const + { + return mNInputs; + } + + inline size_t + CuComponent:: + GetNOutputs() const + { + return mNOutputs; + } + + + + ////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // UpdatableComponent:: + + inline + CuUpdatableComponent:: + CuUpdatableComponent(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuComponent(nInputs, nOutputs, pPred), + mLearningRate(0.0), mMomentum(0), mWeightcost(0), mGradDivFrm(true) + { + ; + } + + + inline + CuUpdatableComponent:: + ~CuUpdatableComponent() + { + ; + } + + + inline void + CuUpdatableComponent:: + LearnRate(BaseFloat rate) + { + mLearningRate = rate; + } + + + inline BaseFloat + CuUpdatableComponent:: + LearnRate() + { + return mLearningRate; + } + + + inline void + CuUpdatableComponent:: + Momentum(BaseFloat mmt) + { + mMomentum = mmt; + } + + + inline BaseFloat + CuUpdatableComponent:: + Momentum() + { + return mMomentum; + } + + + inline void + CuUpdatableComponent:: + Weightcost(BaseFloat cost) + { + mWeightcost = cost; + } + + + inline BaseFloat + CuUpdatableComponent:: + Weightcost() + { + return mWeightcost; + } + + + inline void + CuUpdatableComponent:: + GradDivFrm(bool div) + { + mGradDivFrm = div; + } + + inline bool + CuUpdatableComponent:: + GradDivFrm() + { + return mGradDivFrm; + } + +} // namespace TNet + + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuDiscreteLinearity.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuDiscreteLinearity.cc.svn-base new file mode 100644 index 0000000..befde24 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuDiscreteLinearity.cc.svn-base @@ -0,0 +1,160 @@ + + +#include "cuDiscreteLinearity.h" +#include "cumath.h" + +namespace TNet +{ + + void + CuDiscreteLinearity:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + //Y.SetConst(0.0); + + //precopy bias + Y.AddScaledRow(1.0,mBias,0.0); + + //mulitply with the matrices + int offset_in=0, offset_out=0; + for (int i=0; i<mNBlocks; i++) { + CuMath<BaseFloat>::OffsetGemm('N','N', 1.0, X, mLinearity[i], 1.0, Y, + offset_in, 0, offset_out); + offset_in += mLinearity[i].Rows(); + offset_out += mLinearity[i].Cols(); + } + } + + + void + CuDiscreteLinearity:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + //Y.SetConst(0.0); + + int offset_in=0, offset_out=0; + for(int i=0; i<mNBlocks; i++) { + CuMath<BaseFloat>::OffsetGemm('N', 'T', 1.0, X, mLinearity[i], 0.0, Y, + offset_in, 0, offset_out); + offset_in += mLinearity[i].Cols(); + offset_out += mLinearity[i].Rows(); + } + } + + + void + CuDiscreteLinearity:: + Update() + { + //new implementation + BaseFloat N = 1; + if(mGradDivFrm) { + N = static_cast<BaseFloat>(GetInput().Rows()); + } + BaseFloat mmt_gain = static_cast<BaseFloat>(1.0/(1.0-mMomentum)); + N *= mmt_gain; //compensate higher gradient estimates due to momentum + + //get gradients of discrete linearities + int offset_in=0, offset_out=0; + for(int i=0; i<mNBlocks; i++) { + CuMath<BaseFloat>::OffsetGemm('T','N',1.0, + GetInput(),GetErrorInput(), + mMomentum, mLinearityCorrection[i], + offset_in,offset_out,0); + offset_in += mLinearity[i].Rows(); + offset_out += mLinearity[i].Cols(); + } + for(int i=0; i<mNBlocks; i++) { + //perform update + mLinearity[i].AddScaled(-mLearningRate/N,mLinearityCorrection[i],1.0); + //regularization weight decay + mLinearity[i].AddScaled(-mLearningRate*mWeightcost,mLinearity[i],1.0); + } + + //get gradient of bias + mBiasCorrection.AddColSum(1.0,GetErrorInput(),mMomentum); + //update biases + mBias.AddScaled(-mLearningRate/N,mBiasCorrection,1.0); + } + + + void + CuDiscreteLinearity:: + ReadFromStream(std::istream& rIn) + { + rIn >> std::ws >> mNBlocks; + if(mNBlocks < 1) { + KALDI_ERR << "Bad number of blocks:" << mNBlocks; + } + + mLinearity.resize(mNBlocks); + mLinearityCorrection.resize(mNBlocks); + + int in_dim = 0, out_dim = 0; + for(int i=0; i<mNBlocks; i++) { + //matrix is stored transposed as SNet does + BfMatrix transpose; + rIn >> transpose; + mLinearity[i].CopyFrom(BfMatrix(transpose, TRANS)); + + if(transpose.Cols()*transpose.Rows() == 0) { + Error("Missing linearity matrix in network file"); + } + //allocate training buffers + mLinearityCorrection[i].Init(mLinearity[i].Rows(),mLinearity[i].Cols()); + mLinearityCorrection[i].SetConst(0.0); + + in_dim += transpose.Cols(); + out_dim += transpose.Rows(); + } + + //biases stored normally + BfVector bias; + rIn >> bias; + mBias.CopyFrom(bias); + if(bias.Dim() == 0) { + Error("Missing bias vector in network file"); + } + mBiasCorrection.Init(mBias.Dim()); + mBiasCorrection.SetConst(0.0); + + if(out_dim != GetNOutputs() || + in_dim != GetNInputs() || + mBias.Dim() != GetNOutputs() + ){ + std::ostringstream os; + os << "Wrong dimensionalities of matrix/vector in network file\n" + << "Inputs:" << GetNInputs() + << "Outputs:" << GetNOutputs() + << "\n" + << "linearityCols:" << in_dim + << "linearityRows:" << out_dim + << "biasDims:" << mBias.Dim() + << "\n"; + Error(os.str()); + } + } + + + void + CuDiscreteLinearity:: + WriteToStream(std::ostream& rOut) + { + rOut << mNBlocks << "\n"; + for(int i=0; i< mNBlocks; i++) { + //matrix is stored transposed as SNet does + BfMatrix tmp; + mLinearity[i].CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + } + //biases stored normally + BfVector vec; + mBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + } + + +} //namespace + diff --git a/src/CuTNetLib/.svn/text-base/cuDiscreteLinearity.h.svn-base b/src/CuTNetLib/.svn/text-base/cuDiscreteLinearity.h.svn-base new file mode 100644 index 0000000..06c8d74 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuDiscreteLinearity.h.svn-base @@ -0,0 +1,90 @@ +#ifndef _CUDISCRETE_LINEARITY_H_ +#define _CUDISCRETE_LINEARITY_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +#include "Matrix.h" +#include "Vector.h" + +#include <vector> + + +namespace TNet { + + class CuDiscreteLinearity : public CuUpdatableComponent + { + public: + + CuDiscreteLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuDiscreteLinearity(); + + ComponentType GetType() const; + const char* GetName() const; + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + void Update(); + + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + std::vector<CuMatrix<BaseFloat> > mLinearity; ///< Matrix with neuron weights + CuVector<BaseFloat> mBias; ///< Vector with biases + + std::vector<CuMatrix<BaseFloat> > mLinearityCorrection; ///< Matrix for linearity updates + CuVector<BaseFloat> mBiasCorrection; ///< Vector for bias updates + + size_t mNBlocks; + + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuDiscreteLinearity:: + inline + CuDiscreteLinearity:: + CuDiscreteLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred), + //mLinearity(nInputs,nOutputs), mBias(nOutputs), + //mLinearityCorrection(nInputs,nOutputs), mBiasCorrection(nOutputs) + mNBlocks(0) + { + //mLinearityCorrection.SetConst(0.0); + //mBiasCorrection.SetConst(0.0); + } + + + inline + CuDiscreteLinearity:: + ~CuDiscreteLinearity() + { } + + inline CuComponent::ComponentType + CuDiscreteLinearity:: + GetType() const + { + return CuComponent::DISCRETE_LINEARITY; + } + + inline const char* + CuDiscreteLinearity:: + GetName() const + { + return "<discretelinearity>"; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuNetwork.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuNetwork.cc.svn-base new file mode 100644 index 0000000..e245699 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuNetwork.cc.svn-base @@ -0,0 +1,380 @@ + +#include <algorithm> +//#include <locale> +#include <cctype> +#include <list> +#include <sstream> + +#include "cuNetwork.h" + +#include "cuDiscreteLinearity.h" +#include "cuSharedLinearity.h" +#include "cuSparseLinearity.h" +#include "cuRbm.h" +#include "cuRbmSparse.h" +#include "cuRecurrent.h" +#include "cuBlockArray.h" + +namespace TNet { + + + + + void + CuNetwork:: + ReadNetwork(const char* pSrc) + { + std::ifstream in(pSrc); + if(!in.good()) { + Error(std::string("Error, cannot read model: ")+pSrc); + } + ReadNetwork(in); + in.close(); + } + + + + void + CuNetwork:: + WriteNetwork(const char* pDst) + { + std::ofstream out(pDst); + if(!out.good()) { + Error(std::string("Error, cannot write model: ")+pDst); + } + WriteNetwork(out); + out.close(); + } + + + + void + CuNetwork:: + ReadNetwork(std::istream& rIn) + { + //get the network elements from a factory + CuComponent *pComp; + while(NULL != (pComp = ComponentFactory(rIn))) { + mNetComponents.push_back(pComp); + } + } + + + + void + CuNetwork:: + WriteNetwork(std::ostream& rOut) + { + //dump all the componetns + LayeredType::iterator it; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + ComponentDumper(rOut, **it); + } + } + + + void + CuNetwork:: + SetLearnRate(BaseFloat learnRate, const char* pLearnRateFactors) + { + //parse the learn rate factors: "0.1:0.5:0.6:1.0" to std::list + std::list<BaseFloat> lr_factors; + if(NULL != pLearnRateFactors) { + //replace ':' by ' ' + std::string str(pLearnRateFactors); + size_t pos = 0; + while((pos = str.find(':',pos)) != std::string::npos) str[pos] = ' '; + while((pos = str.find(',',pos)) != std::string::npos) str[pos] = ' '; + + //parse to std::list + std::istringstream is(str); + is >> std::skipws; + BaseFloat f; + while(!is.eof()) { + if(!(is >> f).fail()) { lr_factors.push_back(f); } + else break; + } + } + + //initialize rate factors iterator + BaseFloat scale = 1.0f; + + //store global learning rate + mGlobLearnRate = learnRate; + mpLearnRateFactors = pLearnRateFactors; + + //give scaled learning rate to components + LayeredType::iterator it; + bool stopper_given = false; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + if((*it)->IsUpdatable()) { + //get next scale factor + if(NULL != pLearnRateFactors) { + if(!(lr_factors.size() > 0)) { + Error("Too few learninig rate scale factors"); + } + scale = lr_factors.front(); + lr_factors.pop_front(); + } + //set scaled learning rate to the component + dynamic_cast<CuUpdatableComponent*>(*it)->LearnRate(learnRate*scale); + //set the stopper component for backpropagation + if(!stopper_given && (learnRate*scale > 0.0)) { + mpPropagErrorStopper = *it; stopper_given = true; + } + } + } + if(lr_factors.size() > 0) { + Error("Too much learninig rate scale factors"); + } + } + + + BaseFloat + CuNetwork:: + GetLearnRate() + { + return mGlobLearnRate; + } + + + void + CuNetwork:: + PrintLearnRate() + { + assert(mNetComponents.size() > 0); + std::cout << "Learning rate: global " << mGlobLearnRate; + std::cout << " components' "; + for(size_t i=0; i<mNetComponents.size(); i++) { + if(mNetComponents[i]->IsUpdatable()) { + std::cout << " " << dynamic_cast<CuUpdatableComponent*>(mNetComponents[i])->LearnRate(); + } + } + std::cout << "\n" << std::flush; + } + + + + void + CuNetwork:: + SetMomentum(BaseFloat momentum) + { + LayeredType::iterator it; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + if((*it)->IsUpdatable()) { + dynamic_cast<CuUpdatableComponent*>(*it)->Momentum(momentum); + } + } + } + + void + CuNetwork:: + SetWeightcost(BaseFloat weightcost) + { + LayeredType::iterator it; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + if((*it)->IsUpdatable()) { + dynamic_cast<CuUpdatableComponent*>(*it)->Weightcost(weightcost); + } + } + } + + void + CuNetwork:: + SetL1(BaseFloat l1) + { + LayeredType::iterator it; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + if((*it)->GetType() == CuComponent::SPARSE_LINEARITY) { + dynamic_cast<CuSparseLinearity*>(*it)->L1(l1); + } + } + } + + void + CuNetwork:: + SetGradDivFrm(bool div) + { + LayeredType::iterator it; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + if((*it)->IsUpdatable()) { + dynamic_cast<CuUpdatableComponent*>(*it)->GradDivFrm(div); + } + } + } + + + CuComponent* + CuNetwork:: + ComponentFactory(std::istream& rIn) + { + rIn >> std::ws; + if(rIn.eof()) return NULL; + + CuComponent* pRet=NULL; + CuComponent* pPred=NULL; + + std::string componentTag; + size_t nInputs, nOutputs; + + rIn >> std::ws; + rIn >> componentTag; + if(componentTag == "") return NULL; //nothing left in the file + + //make it lowercase + std::transform(componentTag.begin(), componentTag.end(), + componentTag.begin(), tolower); + + if(componentTag[0] != '<' || componentTag[componentTag.size()-1] != '>') { + Error(std::string("Invalid component tag:")+componentTag); + } + + //the 'endblock' tag terminates the network + if(componentTag == "<endblock>") return NULL; + + rIn >> std::ws; + rIn >> nOutputs; + rIn >> std::ws; + rIn >> nInputs; + assert(nInputs > 0 && nOutputs > 0); + + //make coupling with predecessor + if(mNetComponents.size() != 0) { + pPred = mNetComponents.back(); + } + + //array with list of component tags + static const std::string TAGS[] = { + "<biasedlinearity>", + "<discretelinearity>", + "<sharedlinearity>", + "<sparselinearity>", + "<rbm>", + "<rbmsparse>", + "<recurrent>", + + "<softmax>", + "<sigmoid>", + + "<expand>", + "<copy>", + "<transpose>", + "<blocklinearity>", + "<bias>", + "<window>", + "<log>", + + "<blockarray>", + }; + + static const int n_tags = sizeof(TAGS) / sizeof(TAGS[0]); + int i; + for(i=0; i<n_tags; i++) { + if(componentTag == TAGS[i]) break; + } + + //switch according to position in array TAGS + switch(i) { + case 0: pRet = new CuBiasedLinearity(nInputs,nOutputs,pPred); break; + case 1: pRet = new CuDiscreteLinearity(nInputs,nOutputs,pPred); break; + case 2: pRet = new CuSharedLinearity(nInputs,nOutputs,pPred); break; + case 3: pRet = new CuSparseLinearity(nInputs,nOutputs,pPred); break; + case 4: pRet = new CuRbm(nInputs,nOutputs,pPred); break; + case 5: pRet = new CuRbmSparse(nInputs,nOutputs,pPred); break; + case 6: pRet = new CuRecurrent(nInputs,nOutputs,pPred); break; + + case 7: pRet = new CuSoftmax(nInputs,nOutputs,pPred); break; + case 8: pRet = new CuSigmoid(nInputs,nOutputs,pPred); break; + + case 9: pRet = new CuExpand(nInputs,nOutputs,pPred); break; + case 10: pRet = new CuCopy(nInputs,nOutputs,pPred); break; + case 11: pRet = new CuTranspose(nInputs,nOutputs,pPred); break; + case 12: pRet = new CuBlockLinearity(nInputs,nOutputs,pPred); break; + case 13: pRet = new CuBias(nInputs,nOutputs,pPred); break; + case 14: pRet = new CuWindow(nInputs,nOutputs,pPred); break; + case 15: pRet = new CuLog(nInputs,nOutputs,pPred); break; + + case 16: pRet = new CuBlockArray(nInputs,nOutputs,pPred); break; + + default: Error(std::string("Unknown Component tag:")+componentTag); + } + + //read components content + pRet->ReadFromStream(rIn); + + //return + return pRet; + } + + + void + CuNetwork:: + ComponentDumper(std::ostream& rOut, CuComponent& rComp) + { + //use tags of all the components; or the identification codes + //array with list of component tags + static const CuComponent::ComponentType TYPES[] = { + CuComponent::BIASED_LINEARITY, + CuComponent::DISCRETE_LINEARITY, + CuComponent::SHARED_LINEARITY, + CuComponent::SPARSE_LINEARITY, + CuComponent::RBM, + CuComponent::RBM_SPARSE, + CuComponent::RECURRENT, + + CuComponent::SIGMOID, + CuComponent::SOFTMAX, + + CuComponent::EXPAND, + CuComponent::COPY, + CuComponent::TRANSPOSE, + CuComponent::BLOCK_LINEARITY, + CuComponent::BIAS, + CuComponent::WINDOW, + CuComponent::LOG, + + CuComponent::BLOCK_ARRAY, + }; + static const std::string TAGS[] = { + "<biasedlinearity>", + "<discretelinearity>", + "<sharedlinearity>", + "<sparselinearity>", + "<rbm>", + "<rbmsparse>", + "<recurrent>", + + "<sigmoid>", + "<softmax>", + + "<expand>", + "<copy>", + "<transpose>", + "<blocklinearity>", + "<bias>", + "<window>", + "<log>", + + "<blockarray>", + }; + static const int MAX = sizeof TYPES / sizeof TYPES[0]; + + int i; + for(i=0; i<MAX; ++i) { + if(TYPES[i] == rComp.GetType()) break; + } + if(i == MAX) Error("Unknown ComponentType"); + + //dump the component tag + rOut << TAGS[i] << " " + << rComp.GetNOutputs() << " " + << rComp.GetNInputs() << std::endl; + + //write components content + rComp.WriteToStream(rOut); + } + + + +} //namespace + diff --git a/src/CuTNetLib/.svn/text-base/cuNetwork.h.svn-base b/src/CuTNetLib/.svn/text-base/cuNetwork.h.svn-base new file mode 100644 index 0000000..0453376 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuNetwork.h.svn-base @@ -0,0 +1,220 @@ +#ifndef _CUNETWORK_H_ +#define _CUNETWORK_H_ + +#include "cuComponent.h" + +#include "cuBiasedLinearity.h" +//#include "cuBlockLinearity.h" +//#include "cuBias.h" +//#include "cuWindow.h" + +#include "cuActivation.h" + +#include "cuCRBEDctFeat.h" + +#include "Vector.h" + +#include <vector> + + +namespace TNet { + + class CuNetwork + { + ////////////////////////////////////// + // Typedefs + typedef std::vector<CuComponent*> LayeredType; + + ////////////////////////////////////// + // Disable copy construction, assignment and default constructor + private: + CuNetwork(CuNetwork&); + CuNetwork& operator=(CuNetwork&); + + public: + CuNetwork() { } + CuNetwork(std::istream& rIn); + ~CuNetwork(); + + void AddLayer(CuComponent* layer); + + int Layers() + { return mNetComponents.size(); } + + CuComponent& Layer(int i) + { return *mNetComponents[i]; } + + /// forward the data to the output + void Propagate(const CuMatrix<BaseFloat>& in, CuMatrix<BaseFloat>& out); + + /// backpropagate the error while updating weights + void Backpropagate(const CuMatrix<BaseFloat>& globerr); + + void ReadNetwork(const char* pSrc); ///< read the network from file + void WriteNetwork(const char* pDst); ///< write network to file + + void ReadNetwork(std::istream& rIn); ///< read the network from stream + void WriteNetwork(std::ostream& rOut); ///< write network to stream + + size_t GetNInputs() const; ///< Dimensionality of the input features + size_t GetNOutputs() const; ///< Dimensionality of the desired vectors + + /// set the learning rate + void SetLearnRate(BaseFloat learnRate, const char* pLearnRateFactors = NULL); + BaseFloat GetLearnRate(); ///< get the learning rate value + void PrintLearnRate(); ///< log the learning rate values + + void SetMomentum(BaseFloat momentum); + void SetWeightcost(BaseFloat weightcost); + void SetL1(BaseFloat l1); + + void SetGradDivFrm(bool div); + + + private: + /// Creates a component by reading from stream + CuComponent* ComponentFactory(std::istream& In); + /// Dumps component into a stream + void ComponentDumper(std::ostream& rOut, CuComponent& rComp); + + + + private: + LayeredType mNetComponents; ///< container with the network layers + CuComponent* mpPropagErrorStopper; + BaseFloat mGlobLearnRate; ///< The global (unscaled) learn rate of the network + const char* mpLearnRateFactors; ///< The global (unscaled) learn rate of the network + + + //friend class NetworkGenerator; //<< For generating networks... + + }; + + + ////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuNetwork:: + inline + CuNetwork:: + CuNetwork(std::istream& rSource) + : mpPropagErrorStopper(NULL), mGlobLearnRate(0.0), mpLearnRateFactors(NULL) + { + ReadNetwork(rSource); + } + + + inline + CuNetwork:: + ~CuNetwork() + { + //delete all the components + LayeredType::iterator it; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + delete *it; + *it = NULL; + } + mNetComponents.resize(0); + } + + + inline void + CuNetwork:: + AddLayer(CuComponent* layer) + { + if(mNetComponents.size() > 0) { + if(GetNOutputs() != layer->GetNInputs()) { + Error("Nonmatching dims"); + } + layer->SetInput(mNetComponents.back()->GetOutput()); + mNetComponents.back()->SetErrorInput(layer->GetErrorOutput()); + } + mNetComponents.push_back(layer); + } + + + inline void + CuNetwork:: + Propagate(const CuMatrix<BaseFloat>& in, CuMatrix<BaseFloat>& out) + { + //empty network => copy input + if(mNetComponents.size() == 0) { + out.CopyFrom(in); + return; + } + + //check dims + if(in.Cols() != GetNInputs()) { + std::ostringstream os; + os << "Nonmatching dims" + << " data dim is: " << in.Cols() + << " network needs: " << GetNInputs(); + Error(os.str()); + } + mNetComponents.front()->SetInput(in); + + //propagate + LayeredType::iterator it; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + (*it)->Propagate(); + } + + //copy the output + out.CopyFrom(mNetComponents.back()->GetOutput()); + } + + + + + inline void + CuNetwork:: + Backpropagate(const CuMatrix<BaseFloat>& globerr) + { + mNetComponents.back()->SetErrorInput(globerr); + + // back-propagation + LayeredType::reverse_iterator it; + for(it=mNetComponents.rbegin(); it!=mNetComponents.rend(); ++it) { + //stopper component does not propagate error (no updatable predecessors) + if(*it != mpPropagErrorStopper) { + //compute errors for preceding network components + (*it)->Backpropagate(); + } + //update weights if updatable component + if((*it)->IsUpdatable()) { + CuUpdatableComponent& rComp = dynamic_cast<CuUpdatableComponent&>(**it); + if(rComp.LearnRate() > 0.0f) { + rComp.Update(); + } + } + //stop backprop if no updatable components precede current component + if(mpPropagErrorStopper == *it) break; + } + } + + + inline size_t + CuNetwork:: + GetNInputs() const + { + if(!mNetComponents.size() > 0) return 0; + return mNetComponents.front()->GetNInputs(); + } + + + inline size_t + CuNetwork:: + GetNOutputs() const + { + if(!mNetComponents.size() > 0) return 0; + return mNetComponents.back()->GetNOutputs(); + } + + + + + +} //namespace + +#endif + + diff --git a/src/CuTNetLib/.svn/text-base/cuObjectiveFunction.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuObjectiveFunction.cc.svn-base new file mode 100644 index 0000000..e2b0a1d --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuObjectiveFunction.cc.svn-base @@ -0,0 +1,87 @@ + +#include "cuObjectiveFunction.h" + +#include "Error.h" +#include "cumath.h" + + +namespace TNet +{ + + + + CuObjectiveFunction* + CuObjectiveFunction:: + Factory(ObjFunType type) { + CuObjectiveFunction* ret = NULL; + switch(type) { + case MEAN_SQUARE_ERROR: ret = new CuMeanSquareError; break; + case CROSS_ENTROPY: ret = new CuCrossEntropy; break; + default: Error("Unknown ObjFun type"); + } + return ret; + } + + + void + CuMeanSquareError:: + Evaluate(const CuMatrix<BaseFloat>& rNetOutput, const CuMatrix<BaseFloat>& rDesired, CuMatrix<BaseFloat>& rNetError) + { + //get the global error + rNetError.CopyFrom(rNetOutput); + rNetError.AddScaled(-1.0,rDesired,1.0); + + //calculate the MSE + mAuxMat.CopyFrom(rNetError); + mAuxMat.MulElem(mAuxMat); + + mAuxVec.Init(mAuxMat.Cols()); + mAuxVec.AddColSum(1.0,mAuxMat,0.0); + mAuxVec.CopyTo(mAuxVecHost); + + mError += mAuxVecHost.Sum(); + + //count the frames + mFrames += rNetError.Rows(); + } + + void + CuCrossEntropy:: + Evaluate(const CuMatrix<BaseFloat>& rNetOutput, const CuMatrix<BaseFloat>& rDesired, CuMatrix<BaseFloat>& rNetError) + { + if(rDesired.Cols() != rNetOutput.Cols()) { + std::ostringstream os; + os << "Non-matching dimensions of network output with training targets!!!" + << " Netoutput:" << rNetOutput.Cols() + << " Targets:" << rDesired.Cols(); + Error(os.str()); + } + + //get the global error + //dXent/dSoftmax_in = y-d + rNetError.CopyFrom(rNetOutput); + rNetError.AddScaled(-1.0,rDesired,1.0); + + //check classification + mClassifyVec.Init(rNetOutput.Rows()); + CuMath<BaseFloat>::CheckClass(rNetOutput,rDesired,mClassifyVec); + mClassifyVec.CopyTo(mClassifyVecHost); + mCorrect += mClassifyVecHost.Sum(); + + //calculate Xent + mAuxMat.CopyFrom(rNetOutput); + mAuxMat.LogElem(); + mAuxMat.MulElem(rDesired); + + mAuxVec.Init(mAuxMat.Cols()); + mAuxVec.AddColSum(-1.0,mAuxMat,0.0); + mAuxVec.CopyTo(mAuxVecHost); + + mError += mAuxVecHost.Sum(); + + //count the frames + mFrames += rNetError.Rows(); + } + + +} // namespace TNet diff --git a/src/CuTNetLib/.svn/text-base/cuObjectiveFunction.h.svn-base b/src/CuTNetLib/.svn/text-base/cuObjectiveFunction.h.svn-base new file mode 100644 index 0000000..6b425e8 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuObjectiveFunction.h.svn-base @@ -0,0 +1,166 @@ +#ifndef _CUOBJ_FUN_I_ +#define _CUOBJ_FUN_I_ + +#include <cassert> +#include <limits> +#include <cmath> +#include <sstream> + +#include "Vector.h" +#include "cuvector.h" +#include "cumatrix.h" + +namespace TNet +{ + + + /** + * General interface for objective functions + */ + class CuObjectiveFunction + { + public: + /// Enum with objective function types + typedef enum { + OBJ_FUN_I = 0x0300, + MEAN_SQUARE_ERROR, + CROSS_ENTROPY, + } ObjFunType; + + /// Factory for creating objective function instances + static CuObjectiveFunction* Factory(ObjFunType type); + + ////////////////////////////////////////////////////////////// + // Interface specification + public: + CuObjectiveFunction() + { } + + virtual ~CuObjectiveFunction() + { } + + virtual ObjFunType GetTypeId() = 0; + virtual const char* GetTypeLabel() = 0; + + /// evaluates the data, calculate global error + virtual void Evaluate(const CuMatrix<BaseFloat>& rNetOutput, const CuMatrix<BaseFloat>& rDesired, CuMatrix<BaseFloat>& rNetError) = 0; + + ///get the average per frame error + virtual double GetError() = 0; + ///the number of processed frames + virtual size_t GetFrames() = 0; + ///report the error to std::cout + virtual std::string Report() = 0; + }; + + + + + /** + * Means square error, useful for autoencoders, RBMs et al. + */ + class CuMeanSquareError : public CuObjectiveFunction + { + public: + CuMeanSquareError() + : mError(0), mFrames(0) + { } + + virtual ~CuMeanSquareError() + { } + + ObjFunType GetTypeId() + { return CuObjectiveFunction::MEAN_SQUARE_ERROR; } + + const char* GetTypeLabel() + { return "<mean_square_error>"; } + + void Evaluate(const CuMatrix<BaseFloat>& rNetOutput, const CuMatrix<BaseFloat>& rDesired, CuMatrix<BaseFloat>& rNetError); + + double GetError() + { return mError; } + + size_t GetFrames() + { return mFrames; } + + std::string Report() + { + std::ostringstream ss; + ss << "Mse:" << mError << " frames:" << mFrames + << " err/frm:" << mError/mFrames << "\n"; + return ss.str(); + } + + private: + double mError; + size_t mFrames; + + CuMatrix<BaseFloat> mAuxMat; + CuVector<BaseFloat> mAuxVec; + Vector<BaseFloat> mAuxVecHost; + + }; + + + /** + * Cross entropy, it assumes desired vectors as output values + */ + class CuCrossEntropy : public CuObjectiveFunction + { + public: + CuCrossEntropy() + : mError(0), mFrames(0), mCorrect(0) + { } + + ~CuCrossEntropy() + { } + + ObjFunType GetTypeId() + { return CuObjectiveFunction::CROSS_ENTROPY; } + + const char* GetTypeLabel() + { return "<cross_entropy>"; } + + void Evaluate(const CuMatrix<BaseFloat>& rNetOutput, const CuMatrix<BaseFloat>& rDesired, CuMatrix<BaseFloat>& rNetError); + + double GetError() + { return mError; } + + size_t GetFrames() + { return mFrames; } + + std::string Report() + { + std::ostringstream ss; + //for compatibility with SNet + //ss << " correct: >> " << 100.0*mCorrect/mFrames << "% <<\n"; + + //current new format... + ss << "Xent:" << mError << " frames:" << mFrames + << " err/frm:" << mError/mFrames + << " correct[" << 100.0*mCorrect/mFrames << "%]" + << "\n"; + return ss.str(); + } + + private: + double mError; + size_t mFrames; + size_t mCorrect; + + CuMatrix<BaseFloat> mAuxMat; + CuVector<BaseFloat> mAuxVec; + Vector<BaseFloat> mAuxVecHost; + + CuVector<int> mClassifyVec; + Vector<int> mClassifyVecHost; + }; + + + + + +} //namespace TNet + + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuRbm.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuRbm.cc.svn-base new file mode 100644 index 0000000..3d0699d --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuRbm.cc.svn-base @@ -0,0 +1,244 @@ + +#include <string> +#include <sstream> + +#include "cuRbm.h" + +#include "cumath.h" + + +namespace TNet +{ + + void + CuRbm:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + Y.SetConst(0.0); + Y.AddScaledRow(1.0,mHidBias,0.0); + Y.Gemm('N','N', 1.0, X, mVisHid, 1.0); + if(mHidType == BERNOULLI) { + CuMath<BaseFloat>::Sigmoid(Y,Y); + } + } + + + void + CuRbm:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + if(mHidType == BERNOULLI) { + mBackpropErrBuf.Init(X.Rows(),X.Cols()); + CuMath<BaseFloat>::DiffSigmoid(mBackpropErrBuf,X,GetOutput()); + } else { + mBackpropErrBuf.CopyFrom(X); + } + + Y.SetConst(0.0); + Y.Gemm('N', 'T', 1.0, mBackpropErrBuf, mVisHid, 0.0); + } + + + void + CuRbm:: + Update() + { + //THIS IS DONE TWICE BECAUSE OF THE BACKPROP STOPPER!!! + if(mHidType == BERNOULLI) { + mBackpropErrBuf.Init(GetErrorInput().Rows(),GetErrorInput().Cols()); + CuMath<BaseFloat>::DiffSigmoid(mBackpropErrBuf,GetErrorInput(),GetOutput()); + } else { + mBackpropErrBuf.CopyFrom(GetErrorInput()); + } + +/* + std::cout << " " << GetInput().Rows() + << " " << GetInput().Cols() + << " " << mBackpropErrBuf.Rows() + << " " << mBackpropErrBuf.Cols() + << " " << mVisHidCorrection.Rows() + << " " << mVisHidCorrection.Cols() + ; +*/ + +#if 0 + //former implementation + BaseFloat N = static_cast<BaseFloat>(GetInput().Rows()); + + mVisHidCorrection.Gemm('T','N',-mLearningRate/N,GetInput(),mBackpropErrBuf,mMomentum); + mHidBiasCorrection.AddColSum(-mLearningRate/N,mBackpropErrBuf,mMomentum); + + //regularization weight decay + mVisHidCorrection.AddScaled(-mLearningRate*mWeightcost,mVisHid,1.0); + + mVisHid.AddScaled(1.0,mVisHidCorrection,1.0); + mHidBias.AddScaled(1.0,mHidBiasCorrection,1.0); +#endif + +#if 1 + //new implementation + BaseFloat N = 1; + if(mGradDivFrm) { + N = static_cast<BaseFloat>(GetInput().Rows()); + } + BaseFloat mmt_gain = static_cast<BaseFloat>(1.0/(1.0-mMomentum)); + N *= mmt_gain; + + mVisHidCorrection.Gemm('T','N',1.0,GetInput(),mBackpropErrBuf,mMomentum); + mHidBiasCorrection.AddColSum(1.0,mBackpropErrBuf,mMomentum); + + mVisHid.AddScaled(-mLearningRate/N,mVisHidCorrection,1.0); + mHidBias.AddScaled(-mLearningRate/N,mHidBiasCorrection,1.0); + + //regularization weight decay (from actual weights only) + mVisHid.AddScaled(-mLearningRate*mWeightcost,mVisHid,1.0); +#endif + + } + + + + void + CuRbm:: + Propagate(const CuMatrix<BaseFloat>& visProbs, CuMatrix<BaseFloat>& hidProbs) + { + if(visProbs.Cols() != GetNInputs()) { + std::ostringstream os; + os << " Nonmatching input dim, needs:" << GetNInputs() + << " got:" << visProbs.Cols() << "\n"; + Error(os.str()); + } + + hidProbs.Init(visProbs.Rows(),GetNOutputs()); + + PropagateFnc(visProbs, hidProbs); + } + + void + CuRbm:: + Reconstruct(const CuMatrix<BaseFloat>& hidState, CuMatrix<BaseFloat>& visProbs) + { + visProbs.Init(hidState.Rows(),mNInputs); + visProbs.SetConst(0.0); + visProbs.AddScaledRow(1.0,mVisBias,0.0); + visProbs.Gemm('N','T', 1.0, hidState, mVisHid, 1.0); + if(mVisType == BERNOULLI) { + CuMath<BaseFloat>::Sigmoid(visProbs,visProbs); + } + } + + + void + CuRbm:: + RbmUpdate(const CuMatrix<BaseFloat>& pos_vis, const CuMatrix<BaseFloat>& pos_hid, const CuMatrix<BaseFloat>& neg_vis, const CuMatrix<BaseFloat>& neg_hid) + { + assert(pos_vis.Rows() == pos_hid.Rows() && + pos_vis.Rows() == neg_vis.Rows() && + pos_vis.Rows() == neg_hid.Rows() && + pos_vis.Cols() == neg_vis.Cols() && + pos_hid.Cols() == neg_hid.Cols() && + pos_vis.Cols() == mNInputs && + pos_hid.Cols() == mNOutputs); + + // UPDATE vishid matrix + // + // vishidinc = momentum*vishidinc + ... + // epsilonw*( (posprods-negprods)/numcases - weightcost*vishid); + // + // vishidinc[t] = -(epsilonw/numcases)*negprods + momentum*vishidinc[t-1] + // +(epsilonw/numcases)*posprods + // -(epsilonw*weightcost)*vishid[t-1] + // + BaseFloat N = static_cast<BaseFloat>(pos_vis.Rows()); + mVisHidCorrection.Gemm('T','N',-mLearningRate/N,neg_vis,neg_hid,mMomentum); + mVisHidCorrection.Gemm('T','N',+mLearningRate/N,pos_vis,pos_hid,1.0); + mVisHidCorrection.AddScaled(-mLearningRate*mWeightcost,mVisHid,1.0); + mVisHid.AddScaled(1.0,mVisHidCorrection,1.0); + + // UPDATE visbias vector + // + // visbiasinc = momentum*visbiasinc + (epsilonvb/numcases)*(posvisact-negvisact); + // + mVisBiasCorrection.AddColSum(-mLearningRate/N,neg_vis,mMomentum); + mVisBiasCorrection.AddColSum(+mLearningRate/N,pos_vis,1.0); + mVisBias.AddScaled(1.0,mVisBiasCorrection,1.0); + + // UPDATE hidbias vector + // + // hidbiasinc = momentum*hidbiasinc + (epsilonhb/numcases)*(poshidact-neghidact); + // + mHidBiasCorrection.AddColSum(-mLearningRate/N,neg_hid,mMomentum); + mHidBiasCorrection.AddColSum(+mLearningRate/N,pos_hid,1.0); + mHidBias.AddScaled(1.0/*0.0*/,mHidBiasCorrection,1.0); + + } + + + void + CuRbm:: + ReadFromStream(std::istream& rIn) + { + //type of the units + std::string str; + + rIn >> std::ws >> str; + if(0 == str.compare("bern")) { + mVisType = BERNOULLI; + } else if(0 == str.compare("gauss")) { + mVisType = GAUSSIAN; + } else Error(std::string("Invalid unit type: ")+str); + + rIn >> std::ws >> str; + if(0 == str.compare("bern")) { + mHidType = BERNOULLI; + } else if(0 == str.compare("gauss")) { + mHidType = GAUSSIAN; + } else Error(std::string("Invalid unit type: ")+str); + + + //matrix is stored transposed as SNet does + BfMatrix transpose; + rIn >> transpose; + mVisHid.CopyFrom(BfMatrix(transpose, TRANS)); + //biases stored normally + BfVector bias; + rIn >> bias; + mVisBias.CopyFrom(bias); + rIn >> bias; + mHidBias.CopyFrom(bias); + } + + + void + CuRbm:: + WriteToStream(std::ostream& rOut) + { + //store unit type info + if(mVisType == BERNOULLI) { + rOut << " bern "; + } else { + rOut << " gauss "; + } + if(mHidType == BERNOULLI) { + rOut << " bern\n"; + } else { + rOut << " gauss\n"; + } + + //matrix is stored transposed as SNet does + BfMatrix tmp; + mVisHid.CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + //biases stored normally + BfVector vec; + mVisBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + mHidBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + } + + +} //namespace diff --git a/src/CuTNetLib/.svn/text-base/cuRbm.h.svn-base b/src/CuTNetLib/.svn/text-base/cuRbm.h.svn-base new file mode 100644 index 0000000..c1e984b --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuRbm.h.svn-base @@ -0,0 +1,146 @@ +#ifndef _CU_RBM_H_ +#define _CU_RBM_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + + class CuRbmBase : public CuUpdatableComponent + { + public: + typedef enum { + BERNOULLI, + GAUSSIAN + } RbmUnitType; + + CuRbmBase(size_t nInputs, size_t nOutputs, CuComponent *pPred) : + CuUpdatableComponent(nInputs, nOutputs, pPred) + { } + + + virtual void Propagate( + const CuMatrix<BaseFloat>& visProbs, + CuMatrix<BaseFloat>& hidProbs + ) = 0; + virtual void Reconstruct( + const CuMatrix<BaseFloat>& hidState, + CuMatrix<BaseFloat>& visProbs + ) = 0; + virtual void RbmUpdate( + const CuMatrix<BaseFloat>& pos_vis, + const CuMatrix<BaseFloat>& pos_hid, + const CuMatrix<BaseFloat>& neg_vis, + const CuMatrix<BaseFloat>& neg_hid + ) = 0; + + virtual RbmUnitType VisType() = 0; + virtual RbmUnitType HidType() = 0; + }; + + + class CuRbm : public CuRbmBase + { + public: + + CuRbm(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuRbm(); + + ComponentType GetType() const; + const char* GetName() const; + + //CuUpdatableComponent API + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + void Update(); + + //RBM training API + void Propagate(const CuMatrix<BaseFloat>& visProbs, CuMatrix<BaseFloat>& hidProbs); + void Reconstruct(const CuMatrix<BaseFloat>& hidState, CuMatrix<BaseFloat>& visProbs); + void RbmUpdate(const CuMatrix<BaseFloat>& pos_vis, const CuMatrix<BaseFloat>& pos_hid, const CuMatrix<BaseFloat>& neg_vis, const CuMatrix<BaseFloat>& neg_hid); + + RbmUnitType VisType() + { return mVisType; } + + RbmUnitType HidType() + { return mHidType; } + + //static void BinarizeProbs(const CuMatrix<BaseFloat>& probs, CuMatrix<BaseFloat>& states); + + //I/O + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + CuMatrix<BaseFloat> mVisHid; ///< Matrix with neuron weights + CuVector<BaseFloat> mVisBias; ///< Vector with biases + CuVector<BaseFloat> mHidBias; ///< Vector with biases + + CuMatrix<BaseFloat> mVisHidCorrection; ///< Matrix for linearity updates + CuVector<BaseFloat> mVisBiasCorrection; ///< Vector for bias updates + CuVector<BaseFloat> mHidBiasCorrection; ///< Vector for bias updates + + CuMatrix<BaseFloat> mBackpropErrBuf; + + RbmUnitType mVisType; + RbmUnitType mHidType; + + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuRbm:: + inline + CuRbm:: + CuRbm(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuRbmBase(nInputs, nOutputs, pPred), + mVisHid(nInputs,nOutputs), + mVisBias(nInputs), mHidBias(nOutputs), + mVisHidCorrection(nInputs,nOutputs), + mVisBiasCorrection(nInputs), mHidBiasCorrection(nOutputs), + mBackpropErrBuf(), + mVisType(BERNOULLI), + mHidType(BERNOULLI) + { + mVisHidCorrection.SetConst(0.0); + mVisBiasCorrection.SetConst(0.0); + mHidBiasCorrection.SetConst(0.0); + } + + + inline + CuRbm:: + ~CuRbm() + { } + + inline CuComponent::ComponentType + CuRbm:: + GetType() const + { + return CuComponent::RBM; + } + + inline const char* + CuRbm:: + GetName() const + { + return "<rbm>"; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuRbmSparse.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuRbmSparse.cc.svn-base new file mode 100644 index 0000000..e0b7352 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuRbmSparse.cc.svn-base @@ -0,0 +1,269 @@ + +#include <string> +#include <sstream> + +#include "cuRbmSparse.h" + +#include "cumath.h" + + +namespace TNet +{ + + void + CuRbmSparse:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + Y.SetConst(0.0); + Y.AddScaledRow(1.0,mHidBias,0.0); + Y.Gemm('N','N', 1.0, X, mVisHid, 1.0); + if(mHidType == BERNOULLI) { + CuMath<BaseFloat>::Sigmoid(Y,Y); + } + } + + + void + CuRbmSparse:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + if(mHidType == BERNOULLI) { + mBackpropErrBuf.Init(X.Rows(),X.Cols()); + CuMath<BaseFloat>::DiffSigmoid(mBackpropErrBuf,X,GetOutput()); + } else { + mBackpropErrBuf.CopyFrom(X); + } + + Y.SetConst(0.0); + Y.Gemm('N', 'T', 1.0, mBackpropErrBuf, mVisHid, 0.0); + } + + + void + CuRbmSparse:: + Update() + { + //THIS IS DONE TWICE BECAUSE OF THE BACKPROP STOPPER!!! + if(mHidType == BERNOULLI) { + mBackpropErrBuf.Init(GetErrorInput().Rows(),GetErrorInput().Cols()); + CuMath<BaseFloat>::DiffSigmoid(mBackpropErrBuf,GetErrorInput(),GetOutput()); + } else { + mBackpropErrBuf.CopyFrom(GetErrorInput()); + } + +/* + std::cout << " " << GetInput().Rows() + << " " << GetInput().Cols() + << " " << mBackpropErrBuf.Rows() + << " " << mBackpropErrBuf.Cols() + << " " << mVisHidCorrection.Rows() + << " " << mVisHidCorrection.Cols() + ; +*/ + +#if 0 + //former implementation + BaseFloat N = static_cast<BaseFloat>(GetInput().Rows()); + + mVisHidCorrection.Gemm('T','N',-mLearningRate/N,GetInput(),mBackpropErrBuf,mMomentum); + mHidBiasCorrection.AddColSum(-mLearningRate/N,mBackpropErrBuf,mMomentum); + + //regularization weight decay + mVisHidCorrection.AddScaled(-mLearningRate*mWeightcost,mVisHid,1.0); + + mVisHid.AddScaled(1.0,mVisHidCorrection,1.0); + mHidBias.AddScaled(1.0,mHidBiasCorrection,1.0); +#endif + +#if 1 + //new implementation + BaseFloat N = 1; + if(mGradDivFrm) { + N = static_cast<BaseFloat>(GetInput().Rows()); + } + BaseFloat mmt_gain = static_cast<BaseFloat>(1.0/(1.0-mMomentum)); + N *= mmt_gain; + + mVisHidCorrection.Gemm('T','N',1.0,GetInput(),mBackpropErrBuf,mMomentum); + mHidBiasCorrection.AddColSum(1.0,mBackpropErrBuf,mMomentum); + + mVisHid.AddScaled(-mLearningRate/N,mVisHidCorrection,1.0); + mHidBias.AddScaled(-mLearningRate/N,mHidBiasCorrection,1.0); + + //regularization weight decay (from actual weights only) + mVisHid.AddScaled(-mLearningRate*mWeightcost,mVisHid,1.0); +#endif + + } + + + + void + CuRbmSparse:: + Propagate(const CuMatrix<BaseFloat>& visProbs, CuMatrix<BaseFloat>& hidProbs) + { + if(visProbs.Cols() != GetNInputs()) { + std::ostringstream os; + os << " Nonmatching input dim, needs:" << GetNInputs() + << " got:" << visProbs.Cols() << "\n"; + Error(os.str()); + } + + hidProbs.Init(visProbs.Rows(),GetNOutputs()); + + PropagateFnc(visProbs, hidProbs); + } + + void + CuRbmSparse:: + Reconstruct(const CuMatrix<BaseFloat>& hidState, CuMatrix<BaseFloat>& visProbs) + { + visProbs.Init(hidState.Rows(),mNInputs); + visProbs.SetConst(0.0); + visProbs.AddScaledRow(1.0,mVisBias,0.0); + visProbs.Gemm('N','T', 1.0, hidState, mVisHid, 1.0); + if(mVisType == BERNOULLI) { + CuMath<BaseFloat>::Sigmoid(visProbs,visProbs); + } + } + + + void + CuRbmSparse:: + RbmUpdate(const CuMatrix<BaseFloat>& pos_vis, const CuMatrix<BaseFloat>& pos_hid, const CuMatrix<BaseFloat>& neg_vis, const CuMatrix<BaseFloat>& neg_hid) + { + assert(pos_vis.Rows() == pos_hid.Rows() && + pos_vis.Rows() == neg_vis.Rows() && + pos_vis.Rows() == neg_hid.Rows() && + pos_vis.Cols() == neg_vis.Cols() && + pos_hid.Cols() == neg_hid.Cols() && + pos_vis.Cols() == mNInputs && + pos_hid.Cols() == mNOutputs); + + //:SPARSITY: + if(mHidType==BERNOULLI) { + //get expected node activity from current batch + mSparsityQCurrent.AddColSum(1.0/pos_hid.Rows(),pos_hid,0.0); + //get smoothed expected node activity + mSparsityQ.AddScaled(1.0-mLambda,mSparsityQCurrent,mLambda); + //subtract the prior: (q-p) + mSparsityQCurrent.SetConst(-mSparsityPrior); + mSparsityQCurrent.AddScaled(1.0,mSparsityQ,1.0); + //get mean pos_vis + mVisMean.AddColSum(1.0/pos_vis.Rows(),pos_vis,0.0); + } + + // UPDATE vishid matrix + // + // vishidinc = momentum*vishidinc + ... + // epsilonw*( (posprods-negprods)/numcases - weightcost*vishid) + // -sparsitycost*mean_posvis'*(q-p); + // + // vishidinc[t] = -(epsilonw/numcases)*negprods + momentum*vishidinc[t-1] + // +(epsilonw/numcases)*posprods + // -(epsilonw*weightcost)*vishid[t-1] + // + BaseFloat N = static_cast<BaseFloat>(pos_vis.Rows()); + mVisHidCorrection.Gemm('T','N',-mLearningRate/N,neg_vis,neg_hid,mMomentum); + mVisHidCorrection.Gemm('T','N',+mLearningRate/N,pos_vis,pos_hid,1.0); + mVisHidCorrection.AddScaled(-mLearningRate*mWeightcost,mVisHid,1.0);//L2 + if(mHidType==BERNOULLI) { + mVisHidCorrection.BlasGer(-mSparsityCost,mVisMean,mSparsityQCurrent);//sparsity + } + mVisHid.AddScaled(1.0,mVisHidCorrection,1.0); + + // UPDATE visbias vector + // + // visbiasinc = momentum*visbiasinc + (epsilonvb/numcases)*(posvisact-negvisact); + // + mVisBiasCorrection.AddColSum(-mLearningRate/N,neg_vis,mMomentum); + mVisBiasCorrection.AddColSum(+mLearningRate/N,pos_vis,1.0); + mVisBias.AddScaled(1.0,mVisBiasCorrection,1.0); + + // UPDATE hidbias vector + // + // hidbiasinc = momentum*hidbiasinc + (epsilonhb/numcases)*(poshidact-neghidact); + // + mHidBiasCorrection.AddColSum(-mLearningRate/N,neg_hid,mMomentum); + mHidBiasCorrection.AddColSum(+mLearningRate/N,pos_hid,1.0); + if(mHidType==BERNOULLI) { + mHidBiasCorrection.AddScaled(-mSparsityCost,mSparsityQCurrent,1.0);//sparsity + } + mHidBias.AddScaled(1.0/*0.0*/,mHidBiasCorrection,1.0); + + } + + + void + CuRbmSparse:: + ReadFromStream(std::istream& rIn) + { + //type of the units + std::string str; + + rIn >> std::ws >> str; + if(0 == str.compare("bern")) { + mVisType = BERNOULLI; + } else if(0 == str.compare("gauss")) { + mVisType = GAUSSIAN; + } else Error(std::string("Invalid unit type: ")+str); + + rIn >> std::ws >> str; + if(0 == str.compare("bern")) { + mHidType = BERNOULLI; + } else if(0 == str.compare("gauss")) { + mHidType = GAUSSIAN; + } else Error(std::string("Invalid unit type: ")+str); + + + //matrix is stored transposed as SNet does + BfMatrix transpose; + rIn >> transpose; + mVisHid.CopyFrom(BfMatrix(transpose, TRANS)); + //biases stored normally + BfVector bias; + rIn >> bias; + mVisBias.CopyFrom(bias); + rIn >> bias; + mHidBias.CopyFrom(bias); + + rIn >> std::ws >> mSparsityCost; + std::cout << "RBM::mSparsityCost=" << mSparsityCost; + } + + + void + CuRbmSparse:: + WriteToStream(std::ostream& rOut) + { + //store unit type info + if(mVisType == BERNOULLI) { + rOut << " bern "; + } else { + rOut << " gauss "; + } + if(mHidType == BERNOULLI) { + rOut << " bern\n"; + } else { + rOut << " gauss\n"; + } + + //matrix is stored transposed as SNet does + BfMatrix tmp; + mVisHid.CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + //biases stored normally + BfVector vec; + mVisBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + mHidBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + //store the sparsity cost + rOut << mSparsityCost << std::endl; + } + + +} //namespace diff --git a/src/CuTNetLib/.svn/text-base/cuRbmSparse.h.svn-base b/src/CuTNetLib/.svn/text-base/cuRbmSparse.h.svn-base new file mode 100644 index 0000000..9d7e304 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuRbmSparse.h.svn-base @@ -0,0 +1,134 @@ +#ifndef _CU_RBM_SPARSE_H_ +#define _CU_RBM_SPARSE_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" +#include "cuRbm.h" + + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + + class CuRbmSparse : public CuRbmBase + { + public: + + CuRbmSparse(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuRbmSparse(); + + ComponentType GetType() const; + const char* GetName() const; + + //CuUpdatableComponent API + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + void Update(); + + //RBM training API + void Propagate(const CuMatrix<BaseFloat>& visProbs, CuMatrix<BaseFloat>& hidProbs); + void Reconstruct(const CuMatrix<BaseFloat>& hidState, CuMatrix<BaseFloat>& visProbs); + void RbmUpdate(const CuMatrix<BaseFloat>& pos_vis, const CuMatrix<BaseFloat>& pos_hid, const CuMatrix<BaseFloat>& neg_vis, const CuMatrix<BaseFloat>& neg_hid); + + RbmUnitType VisType() + { return mVisType; } + + RbmUnitType HidType() + { return mHidType; } + + //static void BinarizeProbs(const CuMatrix<BaseFloat>& probs, CuMatrix<BaseFloat>& states); + + //I/O + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + CuMatrix<BaseFloat> mVisHid; ///< Matrix with neuron weights + CuVector<BaseFloat> mVisBias; ///< Vector with biases + CuVector<BaseFloat> mHidBias; ///< Vector with biases + + CuMatrix<BaseFloat> mVisHidCorrection; ///< Matrix for linearity updates + CuVector<BaseFloat> mVisBiasCorrection; ///< Vector for bias updates + CuVector<BaseFloat> mHidBiasCorrection; ///< Vector for bias updates + + CuMatrix<BaseFloat> mBackpropErrBuf; + + RbmUnitType mVisType; + RbmUnitType mHidType; + + ////// sparsity + BaseFloat mSparsityPrior; ///< sparsity target (unit activity prior) + BaseFloat mLambda; ///< exponential decay factor for q (observed probability of unit to be active) + BaseFloat mSparsityCost; ///< sparsity cost coef. + + CuVector<BaseFloat> mSparsityQ; + CuVector<BaseFloat> mSparsityQCurrent; + CuVector<BaseFloat> mVisMean; ///< buffer for mean visible + + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuRbmSparse:: + inline + CuRbmSparse:: + CuRbmSparse(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuRbmBase(nInputs, nOutputs, pPred), + mVisHid(nInputs,nOutputs), + mVisBias(nInputs), mHidBias(nOutputs), + mVisHidCorrection(nInputs,nOutputs), + mVisBiasCorrection(nInputs), mHidBiasCorrection(nOutputs), + mBackpropErrBuf(), + mVisType(BERNOULLI), + mHidType(BERNOULLI), + + mSparsityPrior(0.0001), + mLambda(0.95), + mSparsityCost(1e-7), + mSparsityQ(nOutputs), + mSparsityQCurrent(nOutputs), + mVisMean(nInputs) + { + mVisHidCorrection.SetConst(0.0); + mVisBiasCorrection.SetConst(0.0); + mHidBiasCorrection.SetConst(0.0); + + mSparsityQ.SetConst(mSparsityPrior); + mSparsityQCurrent.SetConst(0.0); + mVisMean.SetConst(0.0); + } + + + inline + CuRbmSparse:: + ~CuRbmSparse() + { } + + inline CuComponent::ComponentType + CuRbmSparse:: + GetType() const + { + return CuComponent::RBM_SPARSE; + } + + inline const char* + CuRbmSparse:: + GetName() const + { + return "<rbmsparse>"; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuRecurrent.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuRecurrent.cc.svn-base new file mode 100644 index 0000000..428df2c --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuRecurrent.cc.svn-base @@ -0,0 +1,191 @@ + +#include <string> +#include <sstream> + +#include "cuRecurrent.h" + +#include "cumath.h" +#include "cuda_runtime.h" + + +namespace TNet +{ + + void + CuRecurrent:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + assert(X.Rows() == 1); + assert(Y.Rows() == 1); + if(mInputHistory.Rows() == 0) { + Error("Bptt order was not set"); + } + + //pushback the history + CuMatrix<BaseFloat> tmp(mInputHistory.Rows()-1,mInputHistory.Cols()); + tmp.CopyRows(tmp.Rows(),0,mInputHistory,0); + mInputHistory.CopyRows(tmp.Rows(),0,tmp,1); + + //compose the input vector to 0th row, use input X and previous Y + cudaMemcpy(mInputHistory.pCUData(), X.pCUData(), + sizeof(BaseFloat)*X.Cols(), cudaMemcpyDeviceToDevice); + cudaMemcpy(mInputHistory.pCUData()+X.Cols(), Y.pCUData(), + sizeof(BaseFloat)*Y.Cols(), cudaMemcpyDeviceToDevice); + + //extract first row + //CuMatrix<BaseFloat> first_row(1,mInputHistory.Cols()); + //first_row.CopyRows(1,0,mInputHistory,0); + + //calculate the output + Y.AddScaledRow(1.0,mBias,0.0); + //take 0th vector of history, propagate + CuMath<BaseFloat>::OffsetGemv('T',1.0,mLinearity,mInputHistory.pCUData(),mInputHistory.Cols(),1.0,Y.pCUData(),Y.Cols(),0); + //Y.Gemm('N','N', 1.0, first_row, mLinearity, 1.0); + CuMath<BaseFloat>::Sigmoid(Y,Y); + + /* + std::cout << "-------------------------------------" << std::endl; + X.Print(); + Y.Print(); + mInputHistory.Print(); + */ + + } + + + void + CuRecurrent:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + assert(Y.Rows() == 1); + assert(X.Rows() == 1); + + //apply diff sigmoid + CuMatrix<BaseFloat> diff_sigm(1,X.Cols()); + CuMath<BaseFloat>::DiffSigmoid(diff_sigm,X,GetOutput()); + + //:TODO: inefficent to calculate all the input errors!!! + // we need only part of them! + // + //backward-multiply by weights + /* + CuMatrix<BaseFloat> err_prev(1,mLinearity.Rows()); + err_prev.Gemm('N', 'T', 1.0, diff_sigm, mLinearity, 0.0); + + //copy out the interval + cudaMemcpy(Y.pCUData(),err_prev.pCUData(), + sizeof(BaseFloat)*Y.Cols(),cudaMemcpyDeviceToDevice); + */ + + //backward-multiply by weights + CuMath<BaseFloat>::OffsetGemv('N',1.0,mLinearity,diff_sigm.pCUData(),diff_sigm.Cols(),1.0,Y.pCUData(),Y.Cols(),0); + + } + + + void + CuRecurrent:: + Update() + { + // + //correction from PRESENT input x error pair + // + //apply diff sigmoid + CuMatrix<BaseFloat> diff_sigm(1,GetOutput().Cols()); + CuMath<BaseFloat>::DiffSigmoid(diff_sigm,GetErrorInput(),GetOutput()); + + //get 0th row of history (present time) + CuMatrix<BaseFloat> history_row(1,mInputHistory.Cols()); + history_row.CopyRows(1,0,mInputHistory,0); + + //calculate update + //mLinearityCorrection.Gemm('T','N',-mLearningRate,history_row,diff_sigm,mMomentum); + mLinearityCorrection.SetConst(0.0); //:TODO: should be scale/momentum + CuMath<BaseFloat>::BlasGer(-mLearningRate,history_row.pCUData(),history_row.Cols(),diff_sigm.pCUData(),diff_sigm.Cols(),mLinearityCorrection); + + mBiasCorrection.AddColSum(-mLearningRate,diff_sigm,mMomentum); + + // + //BPTT (backprop through time) + // + CuMatrix<BaseFloat> err_prev(1,mLinearity.Rows()); + CuMatrix<BaseFloat> err_prev_part(1,diff_sigm.Cols()); + CuMatrix<BaseFloat> history_output(1,GetOutput().Cols()); + for(int i=1; i<=mBpttOrder; i++) { + //:TODO: inefficent to calculate all the input errors!!! + // we need only part of them! + // + /* + //get previous error + err_prev.Gemm('N','T',1.0,diff_sigm,mLinearity,0.0); + //select interval + cudaMemcpy(err_prev_part.pCUData(),err_prev.pCUData()+GetNInputs(), + sizeof(BaseFloat)*err_prev_part.Cols(),cudaMemcpyDeviceToDevice); + */ + + //backward-multiply by weights + CuMath<BaseFloat>::OffsetGemv('N',1.0,mLinearity,diff_sigm.pCUData(),diff_sigm.Cols(),0.0,err_prev_part.pCUData(),err_prev_part.Cols(),GetInput().Cols()); + + //apply diff sigmoid with activations of HISTORY frame!!! + cudaMemcpy(history_output.pCUData(), mInputHistory.pCURowData(i-1)+GetInput().Cols(), + sizeof(BaseFloat)*history_output.Cols(), cudaMemcpyDeviceToDevice); + CuMath<BaseFloat>::DiffSigmoid(diff_sigm,err_prev_part,history_output); + + //get history row + history_row.CopyRows(1,i,mInputHistory,0); + + //accu the update + //mLinearityCorrection.Gemm('T','N',-mLearningRate,history_row,diff_sigm,1.0); + CuMath<BaseFloat>::BlasGer(-mLearningRate,history_row.pCUData(),history_row.Cols(),diff_sigm.pCUData(),diff_sigm.Cols(),mLinearityCorrection); + mBiasCorrection.AddColSum(-mLearningRate,diff_sigm,1.0); + } + + // + //update the weights + // + //regularization weight decay + mLinearityCorrection.AddScaled(-mLearningRate*mWeightcost,mLinearity,1.0); + + //perform update + mLinearity.AddScaled(1.0,mLinearityCorrection,1.0); + mBias.AddScaled(1.0,mBiasCorrection,1.0); + + } + + + + + void + CuRecurrent:: + ReadFromStream(std::istream& rIn) + { + //matrix is stored transposed as SNet does + BfMatrix transpose; + rIn >> transpose; + mLinearity.CopyFrom(BfMatrix(transpose, TRANS)); + //biases stored normally + BfVector bias; + rIn >> bias; + mBias.CopyFrom(bias); + } + + + void + CuRecurrent:: + WriteToStream(std::ostream& rOut) + { + //matrix is stored transposed as SNet does + BfMatrix tmp; + mLinearity.CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + //biases stored normally + BfVector vec; + mBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + } + + +} //namespace + diff --git a/src/CuTNetLib/.svn/text-base/cuRecurrent.h.svn-base b/src/CuTNetLib/.svn/text-base/cuRecurrent.h.svn-base new file mode 100644 index 0000000..e487b27 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuRecurrent.h.svn-base @@ -0,0 +1,101 @@ +#ifndef _CU_RECURRENT_H_ +#define _CU_RECURRENT_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + + class CuRecurrent : public CuUpdatableComponent + { + public: + + CuRecurrent(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuRecurrent(); + + ComponentType GetType() const; + const char* GetName() const; + + //CuUpdatableComponent API + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + void Update(); + + //Recurrent training API + void BpttOrder(int ord) { + mBpttOrder = ord; + mInputHistory.Init(ord+1,GetNInputs()+GetNOutputs()); + } + void ClearHistory() { + mInputHistory.SetConst(0.0); + if(mOutput.MSize() > 0) { + mOutput.SetConst(0.0); + } + } + + //I/O + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + CuMatrix<BaseFloat> mLinearity; + CuVector<BaseFloat> mBias; + + CuMatrix<BaseFloat> mLinearityCorrection; + CuVector<BaseFloat> mBiasCorrection; + + CuMatrix<BaseFloat> mInputHistory; + + int mBpttOrder; + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuRecurrent:: + inline + CuRecurrent:: + CuRecurrent(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred), + mLinearity(nInputs+nOutputs,nOutputs), + mBias(nOutputs), + mLinearityCorrection(nInputs+nOutputs,nOutputs), + mBiasCorrection(nOutputs) + { } + + + inline + CuRecurrent:: + ~CuRecurrent() + { } + + inline CuComponent::ComponentType + CuRecurrent:: + GetType() const + { + return CuComponent::RECURRENT; + } + + inline const char* + CuRecurrent:: + GetName() const + { + return "<recurrent>"; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuSharedLinearity.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuSharedLinearity.cc.svn-base new file mode 100644 index 0000000..8d5ec09 --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuSharedLinearity.cc.svn-base @@ -0,0 +1,179 @@ + + +#include "cuSharedLinearity.h" +#include "cumath.h" + + +namespace TNet +{ + + void + CuSharedLinearity:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + CuMath<BaseFloat>::VecExpand(mBias,mBiasExpand); /// [ 1 2 3 ] -> [ 1 2 3 1 2 3 ... ] + Y.AddScaledRow(1.0,mBiasExpand,0.0); + + //mBiasExpand.Print(); + + for(int i=0; i<mNInstances; i++) { + CuMath<BaseFloat>::OffsetGemm('N','N', 1.0, X, mLinearity, 1.0, Y, + i*mLinearity.Rows(), 0, i*mLinearity.Cols()); + } + //std::cout << CuDevice::Instantiate().GetFreeMemory(); + //GetInput().Print(); + //GetOutput().Print(); + } + + + void + CuSharedLinearity:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + for(int i=0; i<mNInstances; i++) { + CuMath<BaseFloat>::OffsetGemm('N', 'T', 1.0, X, mLinearity, 0.0, Y, + i*mLinearity.Cols(), 0, i*mLinearity.Rows()); + } + } + + + void + CuSharedLinearity:: + Update() + { +#if 0 + //former implementation + BaseFloat N = static_cast<BaseFloat>(GetInput().Rows()); + + for(int i=0; i<mNInstances; i++) { + CuMath<BaseFloat>::OffsetGemm('T','N',-mLearningRate/(N*mNInstances), + GetInput(),GetErrorInput(), + ((i==0)?mMomentum:1.0f), mLinearityCorrection, + i*mLinearity.Rows(),i*mLinearity.Cols(),0); + } + mBiasCorrectionExpand.AddColSum(1.0,GetErrorInput(),0.0); + CuMath<BaseFloat>::VecAddColSum(-mLearningRate/(N*mNInstances),mBiasCorrectionExpand,mMomentum,mBiasCorrection); + + + //regularization weight decay + mLinearityCorrection.AddScaled(-mLearningRate*mWeightcost,mLinearity,1.0); + + mLinearity.AddScaled(1.0,mLinearityCorrection,1.0); + mBias.AddScaled(1.0,mBiasCorrection,1.0); +#endif + +#if 1 + //new implementation + BaseFloat N = 1; + if(mGradDivFrm) { + N = static_cast<BaseFloat>(GetInput().Rows()); + } + BaseFloat mmt_gain = static_cast<BaseFloat>(1.0/(1.0-mMomentum)); + N *= mmt_gain; //compensate higher gradient estimates due to momentum + + //compensate augmented dyn. range of gradient caused by multiple instances + N *= static_cast<BaseFloat>(mNInstances); + + //get gradient of shared linearity + for(int i=0; i<mNInstances; i++) { + CuMath<BaseFloat>::OffsetGemm('T','N',1.0, + GetInput(),GetErrorInput(), + ((i==0)?mMomentum:1.0f), mLinearityCorrection, + i*mLinearity.Rows(),i*mLinearity.Cols(),0); + } + //get gradient of shared bias + mBiasCorrectionExpand.AddColSum(1.0,GetErrorInput(),0.0); + CuMath<BaseFloat>::VecAddColSum(1.0,mBiasCorrectionExpand,mMomentum,mBiasCorrection); + + //perform update + mLinearity.AddScaled(-mLearningRate/N,mLinearityCorrection,1.0); + mBias.AddScaled(-mLearningRate/N,mBiasCorrection,1.0); + + //regularization weight decay + mLinearity.AddScaled(-mLearningRate*mWeightcost,mLinearity,1.0); +#endif + + } + + + void + CuSharedLinearity:: + ReadFromStream(std::istream& rIn) + { + //number of instances of shared weights in layer + rIn >> std::ws >> mNInstances; + if(mNInstances < 1) { + std::ostringstream os; + os << "Bad number of instances:" << mNInstances; + Error(os.str()); + } + if(GetNInputs() % mNInstances != 0 || GetNOutputs() % mNInstances != 0) { + std::ostringstream os; + os << "Number of Inputs/Outputs must be divisible by number of instances" + << " Inputs:" << GetNInputs() + << " Outputs" << GetNOutputs() + << " Intances:" << mNInstances; + Error(os.str()); + } + + //matrix is stored transposed as SNet does + BfMatrix transpose; + rIn >> transpose; + mLinearity.CopyFrom(BfMatrix(transpose, TRANS)); + //biases stored normally + BfVector bias; + rIn >> bias; + mBias.CopyFrom(bias); + + if(transpose.Cols()*transpose.Rows() == 0) { + Error("Missing linearity matrix in network file"); + } + if(bias.Dim() == 0) { + Error("Missing bias vector in network file"); + } + + + if(mLinearity.Cols() != GetNOutputs() / mNInstances || + mLinearity.Rows() != GetNInputs() / mNInstances || + mBias.Dim() != GetNOutputs() / mNInstances + ){ + std::ostringstream os; + os << "Wrong dimensionalities of matrix/vector in network file\n" + << "Inputs:" << GetNInputs() + << "Outputs:" << GetNOutputs() + << "\n" + << "linearityCols:" << mLinearity.Cols() + << "linearityRows:" << mLinearity.Rows() + << "biasDims:" << mBias.Dim() + << "\n"; + Error(os.str()); + } + + mLinearityCorrection.Init(mLinearity.Rows(),mLinearity.Cols()); + mBiasCorrection.Init(mBias.Dim()); + + mBiasExpand.Init(mBias.Dim()*mNInstances); + mBiasCorrectionExpand.Init(mBias.Dim()*mNInstances); + } + + + void + CuSharedLinearity:: + WriteToStream(std::ostream& rOut) + { + rOut << mNInstances << std::endl; + + //matrix is stored transposed as SNet does + BfMatrix tmp; + mLinearity.CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + //biases stored normally + BfVector vec; + mBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + } + + +} //namespace diff --git a/src/CuTNetLib/.svn/text-base/cuSharedLinearity.h.svn-base b/src/CuTNetLib/.svn/text-base/cuSharedLinearity.h.svn-base new file mode 100644 index 0000000..4aa022a --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuSharedLinearity.h.svn-base @@ -0,0 +1,85 @@ +#ifndef _CUSHARED_LINEARITY_H_ +#define _CUSHARED_LINEARITY_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + + class CuSharedLinearity : public CuUpdatableComponent + { + public: + + CuSharedLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuSharedLinearity(); + + ComponentType GetType() const; + const char* GetName() const; + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + void Update(); + + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + CuMatrix<BaseFloat> mLinearity; ///< Matrix with neuron weights + CuVector<BaseFloat> mBias; ///< Vector with biases + + CuMatrix<BaseFloat> mLinearityCorrection; ///< Matrix for linearity updates + CuVector<BaseFloat> mBiasCorrection; ///< Vector for bias updates + + int mNInstances; + CuVector<BaseFloat> mBiasExpand; + CuVector<BaseFloat> mBiasCorrectionExpand; + + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuSharedLinearity:: + inline + CuSharedLinearity:: + CuSharedLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred), + mNInstances(0) + { } + + + inline + CuSharedLinearity:: + ~CuSharedLinearity() + { } + + inline CuComponent::ComponentType + CuSharedLinearity:: + GetType() const + { + return CuComponent::SHARED_LINEARITY; + } + + inline const char* + CuSharedLinearity:: + GetName() const + { + return "<sharedlinearity>"; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/.svn/text-base/cuSparseLinearity.cc.svn-base b/src/CuTNetLib/.svn/text-base/cuSparseLinearity.cc.svn-base new file mode 100644 index 0000000..2f1159b --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuSparseLinearity.cc.svn-base @@ -0,0 +1,190 @@ + + +#include "cuSparseLinearity.h" +#include <cmath> +#include <cstdlib> + + +namespace TNet +{ + + void + CuSparseLinearity:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + Y.AddScaledRow(1.0,mBias,0.0); + Y.Gemm('N','N', 1.0, X, mLinearity, 1.0); + } + + + void + CuSparseLinearity:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + Y.Gemm('N', 'T', 1.0, X, mLinearity, 0.0); + } + + + void + CuSparseLinearity:: + Update() + { + BaseFloat N = 1; + if(mGradDivFrm) { + N = static_cast<BaseFloat>(GetInput().Rows()); + } + BaseFloat mmt_gain = static_cast<BaseFloat>(1.0/(1.0-mMomentum)); + N *= mmt_gain; + + mLinearityCorrection.Gemm('T','N',1.0,GetInput(),GetErrorInput(),mMomentum); + mBiasCorrection.AddColSum(1.0,GetErrorInput(),mMomentum); + + mLinearity.AddScaled(-mLearningRate/N,mLinearityCorrection,1.0); + mBias.AddScaled(-mLearningRate/N,mBiasCorrection,1.0); + + mLinearityCorrectionAccu.AddScaled(1.0,mLinearityCorrection,1.0); + mLinearity.ApplyMask(mSparsityMask); + + //L1 regularization lassoo... + //each update? everty 1000th update? + if(mL1Const > 0) { + BaseFloat L1_const = mLearningRate*mL1Const*(mGradDivFrm?1.0:GetInput().Rows()); + mLinearity.ApplyL1(L1_const); + } + + //L2 regularization weight decay (from actual weights only) + if(mWeightcost > 0) { + BaseFloat L2_decay = -mLearningRate*mWeightcost*(mGradDivFrm?1.0:GetInput().Rows()); + mLinearity.AddScaled(L2_decay, mLinearity,1.0); + } + + mNFrames += GetInput().Rows(); + + } + + + void + CuSparseLinearity:: + UpdateMask() + { + //move data to host + Matrix<BaseFloat> linearity, linearity_correction_accu; + Matrix<BaseFloat> sparsity_mask; + + mLinearity.CopyTo(linearity); + mLinearityCorrectionAccu.CopyTo(linearity_correction_accu); + mSparsityMask.CopyTo(sparsity_mask); + + //decide on new sparsity mask + for(size_t r=0; r<sparsity_mask.Rows(); r++) { + for(size_t c=0; c<sparsity_mask.Cols(); c++) { + if(sparsity_mask(r,c) == 1.0f) { //weight active + if(fabs(linearity(r,c)) < mSparsifyWeightThreshold) { + sparsity_mask(r,c) = 0;//deactivate + linearity(r,c) = 0; + } + } else { //weight inactive + if(abs(linearity_correction_accu(r,c))/(BaseFloat)mNFrames > mUnsparsifyAccu) { + sparsity_mask(r,c) = 1;//activate + } + } + } + } + + //move data to the device + mLinearity.CopyFrom(linearity); + mSparsityMask.CopyFrom(sparsity_mask); + } + + + void + CuSparseLinearity:: + ReadFromStream(std::istream& rIn) + { + //matrix is stored transposed as SNet does + BfMatrix transpose; + rIn >> transpose; + mLinearity.CopyFrom(BfMatrix(transpose, TRANS)); + //biases stored normally + BfVector bias; + rIn >> bias; + mBias.CopyFrom(bias); + + //sparsity mask + rIn >> std::ws; + Matrix<BaseFloat> mask_transp; + if(rIn.peek() == 'm') {//load from file + rIn >> mask_transp; + } else {//or set all elements active + mask_transp.Init(transpose.Rows(),transpose.Cols()); + int items=transpose.Rows()*transpose.Stride(); + BaseFloat* p = mask_transp.pData(); + for(int i=0; i<items; i++) {//set all elements to one + *p++ = 1; + } + } + mSparsityMask.CopyFrom(BfMatrix(mask_transp,TRANS)); + + //dummy matrix with acumulated gradients + rIn >> std::ws; + if(rIn.peek() == 'm') {//load from file + BfMatrix dummy; + rIn >> dummy; + } + + if(transpose.Cols()*transpose.Rows() == 0) { + Error("Missing linearity matrix in network file"); + } + if(bias.Dim() == 0) { + Error("Missing bias vector in network file"); + } + if(mLinearity.Cols() != GetNOutputs() || + mLinearity.Rows() != GetNInputs() || + mBias.Dim() != GetNOutputs() + ){ + std::ostringstream os; + os << "Wrong dimensionalities of matrix/vector in network file\n" + << "Inputs:" << GetNInputs() + << "Outputs:" << GetNOutputs() + << "\n" + << "linearityCols:" << mLinearity.Cols() + << "linearityRows:" << mLinearity.Rows() + << "biasDims:" << mBias.Dim() + << "\n"; + Error(os.str()); + } + + assert(mLinearity.Rows() == mSparsityMask.Rows()); + assert(mLinearity.Cols() == mSparsityMask.Cols()); + + } + + + void + CuSparseLinearity:: + WriteToStream(std::ostream& rOut) + { + UpdateMask(); + + //matrix is stored transposed as SNet does + BfMatrix tmp; + mLinearity.CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + //biases stored normally + BfVector vec; + mBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + //store mask + mSparsityMask.CopyTo(tmp); + rOut << BfMatrix(tmp,TRANS); + //store accu + mLinearityCorrectionAccu.CopyTo(tmp); + rOut << BfMatrix(tmp,TRANS); + + } + + +} //namespace + diff --git a/src/CuTNetLib/.svn/text-base/cuSparseLinearity.h.svn-base b/src/CuTNetLib/.svn/text-base/cuSparseLinearity.h.svn-base new file mode 100644 index 0000000..c2b6d6f --- /dev/null +++ b/src/CuTNetLib/.svn/text-base/cuSparseLinearity.h.svn-base @@ -0,0 +1,104 @@ +#ifndef _CUSPARSE_LINEARITY_H_ +#define _CUSPARSE_LINEARITY_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + + class CuSparseLinearity : public CuUpdatableComponent + { + public: + + CuSparseLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuSparseLinearity(); + + ComponentType GetType() const; + const char* GetName() const; + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + void Update(); + void UpdateMask(); + + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + void L1(BaseFloat l1) { + mL1Const = l1; + } + + protected: + CuMatrix<BaseFloat> mLinearity; ///< Matrix with neuron weights + CuVector<BaseFloat> mBias; ///< Vector with biases + CuMatrix<BaseFloat> mSparsityMask; ///< Mask which selects active weights + + CuMatrix<BaseFloat> mLinearityCorrection; ///< Matrix for linearity updates + CuVector<BaseFloat> mBiasCorrection; ///< Vector for bias updates + + CuMatrix<BaseFloat> mLinearityCorrectionAccu; ///< Accumulator for linearity updates + + BaseFloat mL1Const; ///< L1 regularization constant + + size_t mNFrames; ///< Number of accumulated frames + BaseFloat mSparsifyWeightThreshold; ///< Cutoff + BaseFloat mUnsparsifyAccu; ///< Threshold to unsparsify the Cutoff + + + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuSparseLinearity:: + inline + CuSparseLinearity:: + CuSparseLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred), + mLinearity(nInputs,nOutputs), mBias(nOutputs), mSparsityMask(nInputs,nOutputs), + mLinearityCorrection(nInputs,nOutputs), mBiasCorrection(nOutputs), + mLinearityCorrectionAccu(nInputs,nOutputs), + mNFrames(0), mSparsifyWeightThreshold(1.0e-3), + mUnsparsifyAccu(1e20f) + { + mLinearityCorrection.SetConst(0.0f); + mBiasCorrection.SetConst(0.0f); + mLinearityCorrectionAccu.SetConst(0.0f); + } + + + inline + CuSparseLinearity:: + ~CuSparseLinearity() + { } + + inline CuComponent::ComponentType + CuSparseLinearity:: + GetType() const + { + return CuComponent::SPARSE_LINEARITY; + } + + inline const char* + CuSparseLinearity:: + GetName() const + { + return "<sparselinearity>"; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/Makefile b/src/CuTNetLib/Makefile new file mode 100644 index 0000000..c7678da --- /dev/null +++ b/src/CuTNetLib/Makefile @@ -0,0 +1,30 @@ + +include ../tnet.mk + +INCLUDE= -I. -I../ -I../KaldiLib -I../CuBaseLib -I../TNetLib -I$(CUDA_TK_BASE)/include/ + + +all : libCuTNet.a + +libCuTNet.a : $(OBJ) + $(AR) ruv $@ $? + $(RANLIB) $@ + + +%.o : %.cc + $(CXX) -c $< -o $@ $(CFLAGS) $(CXXFLAGS) $(INCLUDE) + + + + +.PHONY: clean depend + +clean : + rm -f *.o *.a + + +depend: + $(CXX) -M $(CXXFLAGS) *.cc $(INCLUDE) > .depend.mk + +-include .depend.mk + diff --git a/src/CuTNetLib/cuActivation.cc b/src/CuTNetLib/cuActivation.cc new file mode 100644 index 0000000..bd57ae5 --- /dev/null +++ b/src/CuTNetLib/cuActivation.cc @@ -0,0 +1,46 @@ + +#include "cuActivation.h" +#include "cumath.h" + + +namespace TNet { + + + void + CuSigmoid:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + CuMath<BaseFloat>::Sigmoid(Y, X); + } + + + void + CuSigmoid:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + CuMath<BaseFloat>::DiffSigmoid(Y, X, mOutput); + } + + + + void + CuSoftmax:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + CuMath<BaseFloat>::Softmax(Y,X); + } + + + + void + CuSoftmax:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + //we assume X is already dE/dSoftmax_input + Y.CopyFrom(X); + } + + + +} //namespace + diff --git a/src/CuTNetLib/cuActivation.h b/src/CuTNetLib/cuActivation.h new file mode 100644 index 0000000..c66640c --- /dev/null +++ b/src/CuTNetLib/cuActivation.h @@ -0,0 +1,132 @@ + +#ifndef _CUACT_FUN_I_ +#define _CUACT_FUN_I_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +namespace TNet +{ + + /** + * \brief Common interface for activation functions + */ + class CuActivation : public CuComponent + { + public: + CuActivation(size_t nInputs, size_t nOutputs, CuComponent *pPred); + + protected: + }; + + + /** + * \brief CuSigmoid activation function + * + * \ingroup CuNNActivation + * Implements forward pass: \f[ Y_i=\frac{1}{1+e^{-X_i}} \f] + * Error propagation: \f[ E_i=Y_i(1-Y_i)e_i \f] + */ + class CuSigmoid : public CuActivation + { + public: + CuSigmoid(size_t nInputs, size_t nOutputs, CuComponent *pPred); + + ComponentType GetType() const; + const char* GetName() const; + + protected: + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + }; + + + /** + * \brief CuSoftmax activation function + * + * \ingroup CuNNActivation + * Implements forward pass: \f[ Y_i=\frac{1}{Z} e^{X_i} \f] + * where \f$ Z=\Sigma_{i=0}^{i=N-1} e^{X_i} \f$ + * Error Propagation: \f[ E_i=Y_i - \Sigma_{j=0}^{j=N-1} Y_i Y_j e_j \f] + */ + class CuSoftmax : public CuActivation + { + public: + CuSoftmax(size_t nInputs, size_t nOutputs, CuComponent *pPred); + + ComponentType GetType() const; + const char* GetName() const; + + protected: + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + }; + + + ////////////////////////////////////////////////////////////////////////// + // Inline functions + // Activation:: + inline + CuActivation:: + CuActivation(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuComponent(nInputs,nOutputs, pPred) + { + assert(nInputs == nOutputs); + } + + + ////////////////////////////////////////////////////////////////////////// + // Inline functions + // Sigmoid:: + inline + CuSigmoid:: + CuSigmoid(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuActivation(nInputs,nOutputs, pPred) + { } + + inline CuComponent::ComponentType + CuSigmoid:: + GetType() const + { + return CuComponent::SIGMOID; + } + + inline const char* + CuSigmoid:: + GetName() const + { + return "<sigmoid>"; + } + + + + ////////////////////////////////////////////////////////////////////////// + // Inline functions + // Softmax:: + inline + CuSoftmax:: + CuSoftmax(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuActivation(nInputs,nOutputs, pPred) + { } + + inline CuComponent::ComponentType + CuSoftmax:: + GetType() const + { + return CuComponent::SOFTMAX; + } + + inline const char* + CuSoftmax:: + GetName() const + { + return "<softmax>"; + } + + +} //namespace + + +#endif diff --git a/src/CuTNetLib/cuBiasedLinearity.cc b/src/CuTNetLib/cuBiasedLinearity.cc new file mode 100644 index 0000000..b9ac137 --- /dev/null +++ b/src/CuTNetLib/cuBiasedLinearity.cc @@ -0,0 +1,123 @@ + + +#include "cuBiasedLinearity.h" + + +namespace TNet +{ + + void + CuBiasedLinearity:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + //Y.SetConst(0.0); + Y.AddScaledRow(1.0,mBias,0.0); + Y.Gemm('N','N', 1.0, X, mLinearity, 1.0); + } + + + void + CuBiasedLinearity:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + //Y.SetConst(0.0); + Y.Gemm('N', 'T', 1.0, X, mLinearity, 0.0); + } + + + void + CuBiasedLinearity:: + Update() + { +#if 0 + //former implementation + BaseFloat N = static_cast<BaseFloat>(GetInput().Rows()); + + mLinearityCorrection.Gemm('T','N',-mLearningRate/N,GetInput(),GetErrorInput(),mMomentum); + mBiasCorrection.AddColSum(-mLearningRate/N,GetErrorInput(),mMomentum); + + //regularization weight decay + mLinearityCorrection.AddScaled(-mLearningRate*mWeightcost,mLinearity,1.0); + + mLinearity.AddScaled(1.0,mLinearityCorrection,1.0); + mBias.AddScaled(1.0,mBiasCorrection,1.0); +#endif + +#if 1 + //new implementation + BaseFloat N = 1; + if(mGradDivFrm) { + N = static_cast<BaseFloat>(GetInput().Rows()); + } + BaseFloat mmt_gain = static_cast<BaseFloat>(1.0/(1.0-mMomentum)); + N *= mmt_gain; + + mLinearityCorrection.Gemm('T','N',1.0,GetInput(),GetErrorInput(),mMomentum); + mBiasCorrection.AddColSum(1.0,GetErrorInput(),mMomentum); + + mLinearity.AddScaled(-mLearningRate/N,mLinearityCorrection,1.0); + mBias.AddScaled(-mLearningRate/N,mBiasCorrection,1.0); + + //regularization weight decay (from actual weights only) + BaseFloat L2_decay = -mLearningRate*mWeightcost*(mGradDivFrm?1.0:GetInput().Rows()); + mLinearity.AddScaled(L2_decay, mLinearity,1.0); +#endif + } + + + void + CuBiasedLinearity:: + ReadFromStream(std::istream& rIn) + { + //matrix is stored transposed as SNet does + BfMatrix transpose; + rIn >> transpose; + mLinearity.CopyFrom(BfMatrix(transpose, TRANS)); + //biases stored normally + BfVector bias; + rIn >> bias; + mBias.CopyFrom(bias); + + if(transpose.Cols()*transpose.Rows() == 0) { + Error("Missing linearity matrix in network file"); + } + if(bias.Dim() == 0) { + Error("Missing bias vector in network file"); + } + if(mLinearity.Cols() != GetNOutputs() || + mLinearity.Rows() != GetNInputs() || + mBias.Dim() != GetNOutputs() + ){ + std::ostringstream os; + os << "Wrong dimensionalities of matrix/vector in network file\n" + << "Inputs:" << GetNInputs() + << "Outputs:" << GetNOutputs() + << "\n" + << "linearityCols:" << mLinearity.Cols() + << "linearityRows:" << mLinearity.Rows() + << "biasDims:" << mBias.Dim() + << "\n"; + Error(os.str()); + } + } + + + void + CuBiasedLinearity:: + WriteToStream(std::ostream& rOut) + { + //matrix is stored transposed as SNet does + BfMatrix tmp; + mLinearity.CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + //biases stored normally + BfVector vec; + mBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + } + + +} //namespace + diff --git a/src/CuTNetLib/cuBiasedLinearity.h b/src/CuTNetLib/cuBiasedLinearity.h new file mode 100644 index 0000000..d3c3b78 --- /dev/null +++ b/src/CuTNetLib/cuBiasedLinearity.h @@ -0,0 +1,98 @@ +#ifndef _CUBIASED_LINEARITY_H_ +#define _CUBIASED_LINEARITY_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + /** + * \brief CuBiasedLinearity summation function + * + * \ingroup CuNNUpdatable + * Implements forward pass: \f[ Y_j=\Sigma_{i=0}^{i=N-1}w_ij X_i +{\beta}_j \f] + * Error propagation: \f[ E_i = \Sigma_{i=0}^{i=N-1} w_ij e_j \f] + * + * Weight adjustion: \f[ W_{ij} = (1-D)(w_{ij} - \alpha(1-\mu)x_i e_j - \mu \Delta) \f] + * and fot bias: \f[ {\Beta}_i = {\beta}_i - \alpha(1-\mu)e_i - \mu \Delta \f] + * where + * - D for weight decay => penalizing large weight + * - \f$ \alpha \f$ for learning rate + * - \f$ \mu \f$ for momentum => avoiding oscillation + */ + class CuBiasedLinearity : public CuUpdatableComponent + { + public: + + CuBiasedLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuBiasedLinearity(); + + ComponentType GetType() const; + const char* GetName() const; + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + void Update(); + + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + CuMatrix<BaseFloat> mLinearity; ///< Matrix with neuron weights + CuVector<BaseFloat> mBias; ///< Vector with biases + + CuMatrix<BaseFloat> mLinearityCorrection; ///< Matrix for linearity updates + CuVector<BaseFloat> mBiasCorrection; ///< Vector for bias updates + + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuBiasedLinearity:: + inline + CuBiasedLinearity:: + CuBiasedLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred), + mLinearity(nInputs,nOutputs), mBias(nOutputs), + mLinearityCorrection(nInputs,nOutputs), mBiasCorrection(nOutputs) + { + mLinearityCorrection.SetConst(0.0); + mBiasCorrection.SetConst(0.0); + } + + + inline + CuBiasedLinearity:: + ~CuBiasedLinearity() + { } + + inline CuComponent::ComponentType + CuBiasedLinearity:: + GetType() const + { + return CuComponent::BIASED_LINEARITY; + } + + inline const char* + CuBiasedLinearity:: + GetName() const + { + return "<biasedlinearity>"; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/cuBlockArray.cc b/src/CuTNetLib/cuBlockArray.cc new file mode 100644 index 0000000..890a752 --- /dev/null +++ b/src/CuTNetLib/cuBlockArray.cc @@ -0,0 +1,138 @@ + + +#include "cuBlockArray.h" +#include "cuNetwork.h" + + +namespace TNet +{ + + void + CuBlockArray:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + CuMatrix<BaseFloat> colsX; + CuMatrix<BaseFloat> colsY; + + int X_src_ori=0, Y_tgt_ori=0; + for(int i=0; i<mNBlocks; i++) { + //copy column stripe from the input X + int colsX_cnt=mBlocks[i]->GetNInputs(); + colsX.Init(X.Rows(),colsX_cnt); + colsX.CopyCols(colsX_cnt,X_src_ori,X,0); + + //propagate through the block(network) + mBlocks[i]->Propagate(colsX,colsY); + + //copy column stripe to the output Y + int colsY_cnt=mBlocks[i]->GetNOutputs(); + Y.CopyCols(colsY_cnt,0,colsY,Y_tgt_ori); + + //shift the origin coordinates + X_src_ori += colsX_cnt; + Y_tgt_ori += colsY_cnt; + } + + assert(X_src_ori == X.Cols()); + assert(Y_tgt_ori == Y.Cols()); + } + + /// @todo CuBlockArray::BackpropagateFnc not implemented + void + CuBlockArray:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + KALDI_ERR << "Unimplemented"; + } + + void + CuBlockArray:: + Update() + { + KALDI_ERR << "Unimplemented"; + } + + + void + CuBlockArray:: + ReadFromStream(std::istream& rIn) + { + if(mBlocks.size() > 0) { + KALDI_ERR << "Cannot read block vector, " + << "aleady filled bt " + << mBlocks.size() + << "elements"; + } + + rIn >> std::ws >> mNBlocks; + if(mNBlocks < 1) { + KALDI_ERR << "Bad number of blocks:" << mNBlocks; + } + + //read all the blocks + std::string tag; + int block_id; + for(int i=0; i<mNBlocks; i++) { + //read tag <block> + rIn >> std::ws >> tag; + //make it lowercase + std::transform(tag.begin(), tag.end(), tag.begin(), tolower); + //check + if(tag!="<block>") { + KALDI_ERR << "<block> keywotd expected"; + } + + //read block number + rIn >> std::ws >> block_id; + if(block_id != i+1) { + KALDI_ERR << "Expected block number:" << i+1 + << " read block number: " << block_id; + } + + //read the nnet + CuNetwork* p_nnet = new CuNetwork; + p_nnet->ReadNetwork(rIn); + if(p_nnet->Layers() == 0) { + KALDI_ERR << "Cannot read empty network to a block"; + } + + //add it to the vector + mBlocks.push_back(p_nnet); + } + + //check the declared dimensionality + int sum_inputs=0, sum_outputs=0; + for(int i=0; i<mNBlocks; i++) { + sum_inputs += mBlocks[i]->GetNInputs(); + sum_outputs += mBlocks[i]->GetNOutputs(); + } + if(sum_inputs != GetNInputs()) { + KALDI_ERR << "Non-matching number of INPUTS! Declared:" + << GetNInputs() + << " summed from blocks" + << sum_inputs; + } + if(sum_outputs != GetNOutputs()) { + KALDI_ERR << "Non-matching number of OUTPUTS! Declared:" + << GetNOutputs() + << " summed from blocks" + << sum_outputs; + } + } + + + void + CuBlockArray:: + WriteToStream(std::ostream& rOut) + { + rOut << " " << mBlocks.size() << " "; + for(int i=0; i<mBlocks.size(); i++) { + rOut << "<block> " << i+1 << "\n"; + mBlocks[i]->WriteNetwork(rOut); + rOut << "<endblock>\n"; + } + } + + +} //namespace + diff --git a/src/CuTNetLib/cuBlockArray.h b/src/CuTNetLib/cuBlockArray.h new file mode 100644 index 0000000..5454538 --- /dev/null +++ b/src/CuTNetLib/cuBlockArray.h @@ -0,0 +1,90 @@ +#ifndef _CUBLOCK_ARRAY_H_ +#define _CUBLOCK_ARRAY_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + + class CuNetwork; + /** + * \brief Updatable compenent consists of several networks + * + * \ingroup CuNNUpdatable + * Each network is individually propagated and backpropagated with non-overlaping input and output. + * + * Enabling multipath topological structure within the network. + */ + class CuBlockArray : public CuUpdatableComponent + { + public: + + CuBlockArray(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuBlockArray(); + + ComponentType GetType() const; + const char* GetName() const; + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + void Update(); + + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + std::vector<CuNetwork*> mBlocks; ///< vector with networks, one network is one block + size_t mNBlocks; + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuBlockArray:: + inline + CuBlockArray:: + CuBlockArray(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred), + mNBlocks(0) + { } + + + inline + CuBlockArray:: + ~CuBlockArray() + { + for(int i=0; i<mBlocks.size(); i++) { + delete mBlocks[i]; + } + mBlocks.clear(); + } + + inline CuComponent::ComponentType + CuBlockArray:: + GetType() const + { + return CuComponent::BLOCK_ARRAY; + } + + inline const char* + CuBlockArray:: + GetName() const + { + return "<blockarray>"; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/cuCRBEDctFeat.h b/src/CuTNetLib/cuCRBEDctFeat.h new file mode 100644 index 0000000..e74a5b6 --- /dev/null +++ b/src/CuTNetLib/cuCRBEDctFeat.h @@ -0,0 +1,340 @@ +#ifndef _CUCRBEDCTFEATURES_H_ +#define _CUCRBEDCTFEATURES_H_ + + +#include "cuComponent.h" +#include "cumath.h" + + +namespace TNet { + + /** + * \brief Input features time context expand + * + * \ingroup CuNNMisc + * Expands the time context of the input features according to FrameOffset + * + * in N, out k*N, FrameOffset o_1,o_2,...,o_k + * FrameOffset example 11frames: -5 -4 -3 -2 -1 0 1 2 3 4 5 + */ + class CuExpand : public CuComponent + { + public: + CuExpand(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred) + { } + + ~CuExpand() + { } + + ComponentType GetType() const + { return EXPAND; } + + const char* GetName() const + { return "<expand>"; } + + void ReadFromStream(std::istream& rIn) + { Vector<int> vec; rIn >> vec; mFrameOffset.CopyFrom(vec); } + + void WriteToStream(std::ostream& rOut) + { Vector<int> vec; mFrameOffset.CopyTo(vec); rOut << vec; } + + protected: + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { CuMath<BaseFloat>::Expand(Y,X,mFrameOffset); } + + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Error("__func__ Nonsense"); } + + protected: + CuVector<int> mFrameOffset; + }; + + + + /** + * \brief Columns' value shuffling + * + * \ingroup CuNNMisc + * Rearrange the matrix columns according to the indices in CopyFromIndices + */ + class CuCopy : public CuComponent + { + public: + CuCopy(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred) + { } + + ~CuCopy() + { } + + ComponentType GetType() const + { return COPY; } + + const char* GetName() const + { return "<copy>"; } + + void ReadFromStream(std::istream& rIn) + { Vector<int> vec; rIn >> vec; vec.Add(-1); mCopyFromIndices.CopyFrom(vec); } + + void WriteToStream(std::ostream& rOut) + { Vector<int> vec; mCopyFromIndices.CopyTo(vec); vec.Add(1); rOut << vec; } + + protected: + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { CuMath<BaseFloat>::Rearrange(Y,X,mCopyFromIndices); } + + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Error("__func__ Nonsense"); } + + protected: + CuVector<int> mCopyFromIndices; + }; + + /** + * \brief Perform Feature transpose + * + * \ingroup CuNNMisc + */ + class CuTranspose : public CuComponent + { + public: + CuTranspose(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred), mContext(0) + { } + + ~CuTranspose() + { } + + ComponentType GetType() const + { return TRANSPOSE; } + + const char* GetName() const + { return "<transpose>"; } + + void ReadFromStream(std::istream& rIn) + { + rIn >> std::ws >> mContext; + + if(GetNInputs() != GetNOutputs()) { + Error("Input dim must be same as output dim"); + } + if(GetNInputs() % mContext != 0) { + Error("Number of inputs must be divisible by context length"); + } + + Vector<int> vec(GetNInputs()); + int channels = GetNInputs() / mContext; + for(int i=0, ch=0; ch<channels; ch++) { + for(int idx=ch; idx < (int)GetNInputs(); idx+=channels, i++) { + assert(i < (int)GetNInputs()); + vec[i] = idx; + } + } + + mCopyFromIndices.CopyFrom(vec); + } + + void WriteToStream(std::ostream& rOut) + { rOut << " " << mContext << "\n"; } + + protected: + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { CuMath<BaseFloat>::Rearrange(Y,X,mCopyFromIndices); } + + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Error("__func__ Nonsense"); } + + protected: + int mContext; + CuVector<int> mCopyFromIndices; + }; + + + /** + * \brief Used for the blockwise multiplication by + * DCT transform loaded from disk + * + * \ingroup CuNNMisc + */ + class CuBlockLinearity : public CuComponent + { + public: + CuBlockLinearity(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred) + { } + + ~CuBlockLinearity() + { } + + + ComponentType GetType() const + { return CuComponent::BLOCK_LINEARITY; } + + const char* GetName() const + { return "<blocklinearity>"; } + + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { CuMath<BaseFloat>::BlockLinearity(Y,X,mBlockLinearity); } + + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Error("__func__ Not implemented"); } + + + void ReadFromStream(std::istream& rIn) + { + Matrix<BaseFloat> mat; + rIn >> mat; + Matrix<BaseFloat> trans(mat,TRANS); + mBlockLinearity.CopyFrom(trans); + + if((GetNOutputs() % mBlockLinearity.Cols() != 0) || + (GetNInputs() % mBlockLinearity.Rows() != 0) || + ((GetNOutputs() / mBlockLinearity.Cols()) != + (GetNInputs() / mBlockLinearity.Rows()))) + { + Error("BlockLinearity matrix dimensions must divide IO dims"); + } + } + + void WriteToStream(std::ostream& rOut) + { + Matrix<BaseFloat> mat; + mBlockLinearity.CopyTo(mat); + Matrix<BaseFloat> trans(mat,TRANS); + rOut << trans; + } + + private: + CuMatrix<BaseFloat> mBlockLinearity; + }; + + + /** + * \brief Bias layer + * + * \ingroup CuNNMisc + * Implements: \f$ \vec{Y}=\vec{X}+\vec{\beta} \f$ + */ + class CuBias : public CuComponent + { + public: + CuBias(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred) + { } + + ~CuBias() + { } + + + ComponentType GetType() const + { return CuComponent::BIAS; } + + const char* GetName() const + { return "<bias>"; } + + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Y.CopyFrom(X); Y.AddScaledRow(1.0, mBias, 1.0); } + + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Y.CopyFrom(X); } + + + void ReadFromStream(std::istream& rIn) + { Vector<BaseFloat> vec; rIn >> vec; mBias.CopyFrom(vec); } + + void WriteToStream(std::ostream& rOut) + { Vector<BaseFloat> vec; mBias.CopyTo(vec); rOut << vec; } + + private: + CuVector<BaseFloat> mBias; + }; + + + /** + * \brief Column Scaling + * + * \ingroup CuNNMisc + * Scaling the inputs with coefficients in scaling CuVector mWindow + */ + class CuWindow : public CuComponent + { + public: + CuWindow(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs, nOutputs, pPred) + { } + + ~CuWindow() + { } + + + ComponentType GetType() const + { return CuComponent::WINDOW; } + + const char* GetName() const + { return "<window>"; } + + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Y.CopyFrom(X); Y.ScaleCols(mWindow); } + + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Error("__func__ Not implemented"); } + + + void ReadFromStream(std::istream& rIn) + { Vector<BaseFloat> vec; rIn >> vec; mWindow.CopyFrom(vec); } + + void WriteToStream(std::ostream& rOut) + { Vector<BaseFloat> vec; mWindow.CopyTo(vec); rOut << vec; } + + private: + CuVector<BaseFloat> mWindow; ///< Scaling factors + }; + + /** + * \brief Perform log transorm + * + * \ingroup CuNNMisc + * Calculate: \f[ \vec{Y}=\ln \vec{X} \f] + */ + class CuLog : public CuComponent + { + public: + CuLog(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs, nOutputs, pPred) + { } + + ~CuLog() + { } + + + ComponentType GetType() const + { return CuComponent::LOG; } + + const char* GetName() const + { return "<log>"; } + + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Y.CopyFrom(X); Y.ApplyLog(); } + + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Error("__func__ Not implemented"); } + + + void ReadFromStream(std::istream& rIn) + { } + + void WriteToStream(std::ostream& rOut) + { } + + }; + +} + + +#endif + diff --git a/src/CuTNetLib/cuCache.cc b/src/CuTNetLib/cuCache.cc new file mode 100644 index 0000000..f96b3b1 --- /dev/null +++ b/src/CuTNetLib/cuCache.cc @@ -0,0 +1,203 @@ + + +#include "cuCache.h" +#include "cumath.h" + + + +namespace TNet { + + CuCache:: + CuCache() + : mState(EMPTY), mIntakePos(0), mExhaustPos(0), mDiscarded(0), + mRandomized(false), mTrace(0) + { } + + CuCache:: + ~CuCache() + { } + + void + CuCache:: + Init(size_t cachesize, size_t bunchsize) + { + if((cachesize % bunchsize) != 0) { + Error("Non divisible cachesize by bunchsize"); + } + + mCachesize = cachesize; + mBunchsize = bunchsize; + + mState = EMPTY; + + mIntakePos = 0; + mExhaustPos = 0; + + mRandomized = false; + + } + + void + CuCache:: + AddData(const CuMatrix<BaseFloat>& rFeatures, const CuMatrix<BaseFloat>& rDesired) + { + assert(rFeatures.Rows() == rDesired.Rows()); + + //lazy buffers allocation + if(mFeatures.Rows() != mCachesize) { + mFeatures.Init(mCachesize,rFeatures.Cols()); + mDesired.Init(mCachesize,rDesired.Cols()); + } + + //warn if segment longer than half-cache + if(rFeatures.Rows() > mCachesize/2) { + std::ostringstream os; + os << "Too long segment and small feature cache! " + << " cachesize: " << mCachesize + << " segmentsize: " << rFeatures.Rows(); + Warning(os.str()); + } + + //change state + if(mState == EMPTY) { + if(mTrace&3) std::cout << "/" << std::flush; + mState = INTAKE; mIntakePos = 0; + + //check for leftover from previous segment + int leftover = mFeaturesLeftover.Rows(); + //check if leftover is not bigger than cachesize + if(leftover > mCachesize) { + std::ostringstream os; + os << "Too small feature cache: " << mCachesize + << ", truncating: " + << leftover - mCachesize << " frames from previous segment leftover"; + //Error(os.str()); + Warning(os.str()); + leftover = mCachesize; + } + //prefill cache with leftover + if(leftover > 0) { + mFeatures.CopyRows(leftover,0,mFeaturesLeftover,0); + mDesired.CopyRows(leftover,0,mDesiredLeftover,0); + mFeaturesLeftover.Destroy(); + mDesiredLeftover.Destroy(); + mIntakePos += leftover; + } + } + + assert(mState == INTAKE); + assert(rFeatures.Rows() == rDesired.Rows()); + if(mTrace&2) std::cout << "F" << std::flush; + + int cache_space = mCachesize - mIntakePos; + int feature_length = rFeatures.Rows(); + int fill_rows = (cache_space<feature_length)? cache_space : feature_length; + int leftover = feature_length - fill_rows; + + assert(cache_space > 0); + + //copy the data to cache + mFeatures.CopyRows(fill_rows,0,rFeatures,mIntakePos); + mDesired.CopyRows(fill_rows,0,rDesired,mIntakePos); + + //copy leftovers + if(leftover > 0) { + mFeaturesLeftover.Init(leftover,mFeatures.Cols()); + mDesiredLeftover.Init(leftover,mDesired.Cols()); + mFeaturesLeftover.CopyRows(leftover,fill_rows,rFeatures,0); + mDesiredLeftover.CopyRows(leftover,fill_rows,rDesired,0); + } + + //update cursor + mIntakePos += fill_rows; + + //change state + if(mIntakePos == mCachesize) { + if(mTrace&3) std::cout << "\\" << std::flush; + mState = FULL; + } + } + + + + void + CuCache:: + Randomize() + { + assert(mState == FULL || mState == INTAKE); + + if(mTrace&3) std::cout << "R" << std::flush; + + //lazy initialization of hte output buffers + mFeaturesRandom.Init(mCachesize,mFeatures.Cols()); + mDesiredRandom.Init(mCachesize,mDesired.Cols()); + + //generate random series of integers + Vector<int> randmask(mIntakePos); + for(unsigned int i=0; i<mIntakePos; i++) { + randmask[i]=i; + } + int* ptr = randmask.pData(); + std::random_shuffle(ptr, ptr+mIntakePos, GenerateRandom); + + CuVector<int> cu_randmask; + cu_randmask.CopyFrom(randmask); + + //randomize + CuMath<BaseFloat>::Randomize(mFeaturesRandom,mFeatures,cu_randmask); + CuMath<BaseFloat>::Randomize(mDesiredRandom,mDesired,cu_randmask); + + mRandomized = true; + + } + + void + CuCache:: + GetBunch(CuMatrix<BaseFloat>& rFeatures, CuMatrix<BaseFloat>& rDesired) + { + if(mState == EMPTY) { + Error("GetBunch on empty cache!!!"); + } + + //change state if full... + if(mState == FULL) { + if(mTrace&3) std::cout << "\\" << std::flush; + mState = EXHAUST; mExhaustPos = 0; + } + + //final cache is not completely filled + if(mState == INTAKE) //&& mpFeatures->EndOfList() + { + if(mTrace&3) std::cout << "\\-LAST\n" << std::flush; + mState = EXHAUST; mExhaustPos = 0; + } + + assert(mState == EXHAUST); + + //init the output + rFeatures.Init(mBunchsize,mFeatures.Cols()); + rDesired.Init(mBunchsize,mDesired.Cols()); + + //copy the output + if(mRandomized) { + rFeatures.CopyRows(mBunchsize,mExhaustPos,mFeaturesRandom,0); + rDesired.CopyRows(mBunchsize,mExhaustPos,mDesiredRandom,0); + } else { + rFeatures.CopyRows(mBunchsize,mExhaustPos,mFeatures,0); + rDesired.CopyRows(mBunchsize,mExhaustPos,mDesired,0); + } + + //update cursor + mExhaustPos += mBunchsize; + + //change state to EMPTY + if(mExhaustPos > mIntakePos-mBunchsize) { + //we don't have more complete bunches... + mDiscarded += mIntakePos - mExhaustPos; + + mState = EMPTY; + } + } + + +} diff --git a/src/CuTNetLib/cuCache.h b/src/CuTNetLib/cuCache.h new file mode 100644 index 0000000..42d9b4d --- /dev/null +++ b/src/CuTNetLib/cuCache.h @@ -0,0 +1,94 @@ +#ifndef _CUCACHE_H_ +#define _CUCACHE_H_ + +#include "cumatrix.h" + +namespace TNet { + + + /** + * \brief The feature-target pair cache + * + * \ingroup CuNNComp + * Preloads mCachesize features and labels to GRAM + * + * During every iterations read mBunchsize data to networks. + * + * When Cache is to be filled with more data that it can hold + * extras are stored in LeftOver, and when data is filled again + * LeftOvers are moved to the Cache. + * + * Note: + * - Cache Size must be divisible by Bunch Size to ensure proper functionality + * - Once extracted data. Cache must be depleted to begin filling or every time extraction start at location Zero. + * - Cache must be filled to begin extraction of Data or we can't start filling and harder to avoid discarding data. + * - @todo Why not implement CuCache as a Stack instead of a Queue? + * . + */ + class CuCache { + typedef enum { EMPTY, INTAKE, FULL, EXHAUST } State; + public: + CuCache(); + ~CuCache(); + + /// Initialize the cache + void Init(size_t cachesize, size_t bunchsize); + + /// Add data to cache, returns number of added vectors + /// \param[in] rFeatures CuNN Input features data + /// \param[in] rDesired CuNN features data label + void AddData(const CuMatrix<BaseFloat>& rFeatures, const CuMatrix<BaseFloat>& rDesired); + /// Randomizes the cache + void Randomize(); + /// Get the bunch of training data + /// \param[out] rFeatures Bunchsize of CuNN Input features data + /// \param[out] rDesired Bunchsize of CuNN features data label + void GetBunch(CuMatrix<BaseFloat>& rFeatures, CuMatrix<BaseFloat>& rDesired); + + + /// Returns true if the cache was completely filled + bool Full() + { return (mState == FULL); } + + /// Returns true if the cache is empty + bool Empty() + { return (mState == EMPTY || mIntakePos < mBunchsize); } + + /// Number of discarded frames + int Discarded() + { return mDiscarded; } + + /// Set the trace message level + void Trace(int trace) + { mTrace = trace; } + + private: + + static long int GenerateRandom(int max) + { return lrand48() % max; } + + State mState; ///< Current state of the cache + + size_t mIntakePos; ///< Number of intaken vectors by AddData + size_t mExhaustPos; ///< Number of exhausted vectors by GetBunch + + size_t mCachesize; ///< Size of cache + size_t mBunchsize; ///< Size of bunch + int mDiscarded; ///< Number of discarded frames + + CuMatrix<BaseFloat> mFeatures; ///< Feature cache + CuMatrix<BaseFloat> mFeaturesRandom; ///< Feature cache + CuMatrix<BaseFloat> mFeaturesLeftover; ///< Feature cache + + CuMatrix<BaseFloat> mDesired; ///< Desired vector cache + CuMatrix<BaseFloat> mDesiredRandom; ///< Desired vector cache + CuMatrix<BaseFloat> mDesiredLeftover; ///< Desired vector cache + + bool mRandomized; + + int mTrace; + }; + +} + +#endif diff --git a/src/CuTNetLib/cuCompDisc.cc b/src/CuTNetLib/cuCompDisc.cc new file mode 100644 index 0000000..2336a86 --- /dev/null +++ b/src/CuTNetLib/cuCompDisc.cc @@ -0,0 +1,178 @@ + + +#include "cuCompDisc.h" +#include "cuNetwork.h" + +#include "Error.h" + + +namespace TNet +{ + + void + CuDiscrete:: + Propagate() + { + for (int i=0;i<inID.size(); i++) + mBlocks[inID[i].block]->SetInput(GetInput(i),inID[i].pos); + for (int i=0; i<mBlocks.size(); i++) + mBlocks[i]->Propagate(); + } + + void + CuDiscrete:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + Error("Not applicable"); + } + + void + CuDiscrete:: + Backpropagate() + { + for (int i=0;i<outID.size(); i++) + mBlocks[outID[i].block]->SetErrorInput(GetOutput(i),outID[i].pos); + for(int i=0; i<mBlocks.size(); i++) + mBlocks[i]->Backpropagate(); + } + + void + CuDiscrete:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + Error("Not applicable"); + } + + void + CuDiscrete:: + Update() + { + for(int i=0; i<mBlocks.size(); i++) + if ( mBlocks[i]->IsUpdatable() ) + { + CuUpdatableComponent& rComp = dynamic_cast<CuUpdatableComponent&>(*mBlocks[i]); + rComp.Update(); + } + } + + + void + CuDiscrete:: + ReadFromStream(std::istream& rIn) + { + int i; + for(i=0; i<mBlocks.size(); i++) { + delete mBlocks[i]; + } + mBlocks.clear(); + inID.clear(); + outID.clear(); + CuComponent* comp; + i=0; + while ( NULL != (comp=CuNetwork::ComponentReader(rIn,NULL)) ) + { + mBlocks.push_back(comp); + for (int j=0;j<(comp->GetInSect());++j) + inID.push_back(posID(i,j)); + for (int j=0;j<(comp->GetOutSect());++j) + outID.push_back(posID(i,j)); + ++i; + } + } + + + void + CuDiscrete:: + WriteToStream(std::ostream& rOut) + { + for(int i=0; i<mBlocks.size(); i++) + CuNetwork::ComponentDumper(rOut,*mBlocks[i]); + rOut << "<endblock>\n"; + } + + void + CuCompound:: + PropagateF(CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + int iLoc=0,oLoc=0; + CuMatrix<BaseFloat> In; + CuMatrix<BaseFloat> Out; + for(int i=0; i<mBlocks.size(); i++) + { + In.Init(X,iLoc,mBlocks[i]->GetNInputs()); + Out.Init(Y,oLoc,mBlocks[i]->GetNOutputs()); + mBlocks[i]->PropagateF(In,Out); + iLoc+=mBlocks[i]->GetNInputs(); + oLoc+=mBlocks[i]->GetNOutputs(); + } + } + + void + CuCompound:: + BackpropagateF(CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + int iLoc=0,oLoc=0; + CuMatrix<BaseFloat> In; + CuMatrix<BaseFloat> Out; + for(int i=0; i<mBlocks.size(); i++) + { + In.Init(X,iLoc,mBlocks[i]->GetNOutputs()); + Out.Init(Y,oLoc,mBlocks[i]->GetNInputs()); + mBlocks[i]->BackpropagateF(In,Out); + iLoc+=mBlocks[i]->GetNOutputs(); + oLoc+=mBlocks[i]->GetNInputs(); + } + } + + void + CuCompound:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + Error("Not applicable"); + } + + void + CuCompound:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + Error("Not applicable"); + } + + void + CuCompound:: + Update() + { + for(int i=0; i<mBlocks.size(); i++) + if ( mBlocks[i]->IsUpdatable() ) + { + CuUpdatableComponent& rComp = dynamic_cast<CuUpdatableComponent&>(*mBlocks[i]); + rComp.Update(); + } + } + + + void + CuCompound:: + ReadFromStream(std::istream& rIn) + { + for(int i=0; i<mBlocks.size(); i++) { + delete mBlocks[i]; + } + mBlocks.clear(); + CuComponent* comp; + while ( NULL != (comp=CuNetwork::ComponentReader(rIn,NULL)) ) + mBlocks.push_back(comp); + } + + + void + CuCompound:: + WriteToStream(std::ostream& rOut) + { + for(int i=0; i<mBlocks.size(); i++) + CuNetwork::ComponentDumper(rOut,*mBlocks[i]); + rOut << "<endblock>\n"; + } + +} //namespace + diff --git a/src/CuTNetLib/cuCompDisc.h b/src/CuTNetLib/cuCompDisc.h new file mode 100644 index 0000000..5b3232e --- /dev/null +++ b/src/CuTNetLib/cuCompDisc.h @@ -0,0 +1,288 @@ +#ifndef _CUCOMPDISC_H_ +#define _CUCOMPDISC_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" +#include "cuNetwork.h" + +#include "Matrix.h" +#include "Vector.h" +#include "Error.h" + + +namespace TNet { + + /** + * \brief A layer of updatable compenents + * + * \ingroup CuNNUpdatable + * Each components are individually propagated and backpropagated with discrete inputs and outputs + * + * Enabling multipath topological structure within the network by layers + */ + + class CuLumpUpdatable : public CuUpdatableComponent + { + public: + CuLumpUpdatable(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred) + { } + + void LearnRate(BaseFloat rate) + { + mLearningRate = rate; + for (int i=0;i<mBlocks.size();++i) + if ( mBlocks[i]->IsUpdatable() ) + { + CuUpdatableComponent& rComp = dynamic_cast<CuUpdatableComponent&>(*mBlocks[i]); + rComp.LearnRate(rate); + } + } + + + void Momentum(BaseFloat mmt) + { + mMomentum = mmt; + for (int i=0;i<mBlocks.size();++i) + if ( mBlocks[i]->IsUpdatable() ) + { + CuUpdatableComponent& rComp = dynamic_cast<CuUpdatableComponent&>(*mBlocks[i]); + rComp.Momentum(mmt); + } + } + + void Weightcost(BaseFloat cost) + { + mWeightcost = cost; + for (int i=0;i<mBlocks.size();++i) + if ( mBlocks[i]->IsUpdatable() ) + { + CuUpdatableComponent& rComp = dynamic_cast<CuUpdatableComponent&>(*mBlocks[i]); + rComp.Weightcost(cost); + } + } + + void GradDivFrm(bool div) + { + mGradDivFrm = div; + for (int i=0;i<mBlocks.size();++i) + if ( mBlocks[i]->IsUpdatable() ) + { + CuUpdatableComponent& rComp = dynamic_cast<CuUpdatableComponent&>(*mBlocks[i]); + rComp.GradDivFrm(div); + } + } + + protected: + std::vector< CuComponent* > mBlocks; ///< vector with component, one component is one block + }; + + /** + * \brief A layer of updatable compenents + * + * \ingroup CuNNUpdatable + * Each components are individually propagated and backpropagated with inputs and outputs within one matrix to save space + * + */ + + class CuDiscrete : public CuLumpUpdatable + { + public: + + typedef struct posID{ int block,pos; posID(int b, int p):block(b),pos(p){}} posID; + + + CuDiscrete(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuDiscrete(); + + ComponentType GetType() const; + const char* GetName() const; + + void Propagate(); + void Backpropagate(); + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + void Update(); + + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + int GetInSect() + { + return inID.size(); + } + + int GetOutSect() + { + return outID.size(); + } + + CuComponent* FindInput(int &pos) + { + if (pos<0 or pos>=inID.size()) + Error("Position out of bound"); + int i=pos; + pos=inID[i].pos; + return mBlocks[inID[i].block]; + } + + CuComponent* FindOutput(int &pos) + { + if (pos<0 or pos>=outID.size()) + Error("Position out of bound"); + int i=pos; + pos=outID[i].pos; + return mBlocks[outID[i].block]; + } + + /// IO Data getters + CuMatrix<BaseFloat>& GetInput(int pos=0) + { + if (preComp!=NULL) + return preComp->GetOutput(pos); + return *mpInput; + } + CuMatrix<BaseFloat>& GetOutput(int pos=0) + { + CuComponent* pComp=FindOutput(pos); + return pComp->GetOutput(pos); + } + CuMatrix<BaseFloat>& GetErrorInput(int pos=0) + { + if (nxtComp!=NULL) + return nxtComp->GetErrorOutput(pos); + return *mpErrorInput; + } + CuMatrix<BaseFloat>& GetErrorOutput(int pos=0) + { + CuComponent* pComp=FindInput(pos); + return pComp->GetErrorOutput(pos); + } + + /// Set input vector (bind with the preceding NetworkComponent) + void SetInput(CuMatrix<BaseFloat>& rInput,int pos=0) + { + if (pos==0) + mpInput=&rInput; + CuComponent* pComp=FindInput(pos); + pComp->SetInput(rInput,pos); + } + /// Set error input vector (bind with the following NetworkComponent) + void SetErrorInput(CuMatrix<BaseFloat>& rErrorInput,int pos=0) + { + if (pos==0) + mpErrorInput=&rErrorInput; + CuComponent* pComp=FindOutput(pos); + pComp->SetErrorInput(rErrorInput,pos); + } + private: + std::vector< CuComponent* > mBlocks; + std::vector< posID > inID,outID; + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuDiscrete:: + inline + CuDiscrete:: + CuDiscrete(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuLumpUpdatable(nInputs, nOutputs, pPred) + { } + + + inline + CuDiscrete:: + ~CuDiscrete() + { + for(int i=0; i<mBlocks.size(); i++) { + delete mBlocks[i]; + } + mBlocks.clear(); + } + + inline CuComponent::ComponentType + CuDiscrete:: + GetType() const + { + return CuComponent::DISCRETE; + } + + inline const char* + CuDiscrete:: + GetName() const + { + return "<discrete>"; + } + + class CuCompound : public CuLumpUpdatable + { + public: + + CuCompound(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuCompound(); + + ComponentType GetType() const; + const char* GetName() const; + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + void Update(); + + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + void PropagateF(CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateF(CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + protected: + + std::vector< CuComponent* > mBlocks; ///< vector with component, one component is one block + + }; + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuLinearity:: + inline + CuCompound:: + CuCompound(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuLumpUpdatable(nInputs, nOutputs, pPred) + { } + + + inline + CuCompound:: + ~CuCompound() + { + for(int i=0; i<mBlocks.size(); i++) { + delete mBlocks[i]; + } + mBlocks.clear(); + } + + inline CuComponent::ComponentType + CuCompound:: + GetType() const + { + return CuComponent::COMPOUND; + } + + inline const char* + CuCompound:: + GetName() const + { + return "<compound>"; + } + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/cuComponent.h b/src/CuTNetLib/cuComponent.h new file mode 100644 index 0000000..6cc8462 --- /dev/null +++ b/src/CuTNetLib/cuComponent.h @@ -0,0 +1,505 @@ +#ifndef _CUNETWORK_COMPONENT_I_H +#define _CUNETWORK_COMPONENT_I_H + + +#include "Vector.h" +#include "Matrix.h" +#include "Error.h" + +#include "cumatrix.h" + +#include <iostream> +#include <stdexcept> +#include <vector> + +/// \defgroup CuNNLayer CuNN Layer types +/// \ingroup CuNNComp + +/// \defgroup CuNNUpdatable CuNN Updatable Layer +/// \ingroup CuNNLayer + +/// \defgroup CuNNActivation CuNN Activation Func Layer +/// \ingroup CuNNLayer + +/// \defgroup CuNNMisc CuNN Misc Layer +/// \ingroup CuNNLayer + +namespace TNet { + + + /** + * \brief Neural network building blocks + * + * Basic element of the network, + * it is a box with defined inputs and outputs, + * and functions to refresh outputs + * + * it is able to compute tranformation function (forward pass) + * and jacobian function (backward pass), + * which is to be implemented in descendents + * + * Compoments together form a doubly-linked list. + * + * Every input rows are a frame of data, + * and the input size are the number of columns + */ + class CuComponent + { + public: + /// Types of the net components + typedef enum { + UPDATABLE_COMPONENT = 0x0100, + BIASED_LINEARITY, + DISCRETE_LINEARITY, + SHARED_LINEARITY, + SPARSE_LINEARITY, + RBM, + RBM_SPARSE, + RECURRENT, + LINEARITY, + UPDATABLEBIAS, + DISCRETE, + COMPOUND, + + ACT_FUN = 0x0200, + SOFTMAX, + SIGMOID, + + OTHER = 0x0400, + EXPAND, + COPY, + TRANSPOSE, + BLOCK_LINEARITY, + WINDOW, + BIAS, + LOG, + PIPE, + LEARNSTOP, + DISTRIB, + COMBINE, + DIVIDE, + MERGE, + REORDER, + + BLOCK_ARRAY, + } ComponentType; + + typedef std::vector< CuMatrix<BaseFloat>* > MatrixPtrVec; + + ////////////////////////////////////////////////////////////// + // Constructor & Destructor + public: + CuComponent(size_t nInputs, size_t nOutputs, CuComponent *pPred); + virtual ~CuComponent(); + + ////////////////////////////////////////////////////////////// + // Interface specification (public) + public: + /// Get Type Identification of the component + virtual ComponentType GetType() const = 0; + /// Get Type Label of the component + virtual const char* GetName() const = 0; + /// Return if the component is UpdatableComponent + virtual bool IsUpdatable() const + { return false; } + + /// Get size of input vectors + size_t GetNInputs() const; + /// Get size of output vectors + size_t GetNOutputs() const; + + /// Set size of input vectors + size_t SetNInputs(size_t nInputs); + /// Set size of output vectors + size_t SetNOutputs(size_t nOutputs); + /// Set the previous component + void SetPrevious(CuComponent* pPred); + /// Set the next component + void SetNext(CuComponent* pNxt); + + /// Return the number of different inputs for complex component + int GetInSect(); + /// Return the number of different outputs for complex component + int GetOutSect(); + + /// IO Data getters + CuMatrix<BaseFloat>& GetInput(int pos=0); + CuMatrix<BaseFloat>& GetOutput(int pos=0); + CuMatrix<BaseFloat>& GetErrorInput(int pos=0); + CuMatrix<BaseFloat>& GetErrorOutput(int pos=0); + + /// Set input vector (bind with the preceding NetworkComponent) + void SetInput(CuMatrix<BaseFloat>& rInput,int pos=0); + /// Set error input vector (bind with the following NetworkComponent) + void SetErrorInput(CuMatrix<BaseFloat>& rErrorInput,int pos=0); + + /// Perform forward pass propagateion Input->Output, + /// wrapper for the PropagateFnc method + void Propagate(); + /// Perform backward pass propagateion ErrorInput->ErrorOutput, + /// wrapper for the BackpropagateFnc method + void Backpropagate(); + + /// Reads the component parameters from stream + virtual void ReadFromStream(std::istream& rIn) { } + /// Writes the components parameters to stream + virtual void WriteToStream(std::ostream& rOut) { } + + /// Public wrapper for PropagateFnc + void PropagateF(CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + /// Public wrapper for BackpropagateFnc + void BackpropagateF(CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + + /////////////////////////////////////////////////////////////// + // Nonpublic member functions used to update data outputs + protected: + /// Forward pass transformation (to be implemented by descendents...) + /// \param[in] X InputMatrix (Network input or Output from last layer) + /// \param[out] Y OutputMatrix (Network output or input of the next layer) + virtual void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) = 0; + /// Backward pass transformation (to be implemented by descendents...) + /// \param[in] X InputMatrix (Network Error, objective func output, or Error output from the next layer) + /// \param[out] Y OutputMatrix (Error input of the last layer) + virtual void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) = 0; + + + /////////////////////////////////////////////////////////////// + // data members + protected: + + size_t mNInputs; ///< Size of input vectors + size_t mNOutputs; ///< Size of output vectors + + CuMatrix<BaseFloat>* mpInput; ///< inputs are NOT OWNED by component + CuMatrix<BaseFloat>* mpErrorInput;///< inputs are NOT OWNED by component + + CuMatrix<BaseFloat> mOutput; ///< outputs are OWNED by component + CuMatrix<BaseFloat> mErrorOutput; ///< outputs are OWNED by component + + CuComponent* preComp;///< The preceding component in the Network + CuComponent* nxtComp;///< The following component in the Network + }; + + + /** + * \brief Class UpdatableComponent is a box which has some + * parameters adjustable by learning + * + * you can set the learning rate, lock the params, + * and learn from each data observation + */ + class CuUpdatableComponent : public CuComponent + { + ////////////////////////////////////////////////////////////// + // Constructor & Destructor + public: + CuUpdatableComponent(size_t nInputs, size_t nOutputs, CuComponent *pPred); + virtual ~CuUpdatableComponent(); + + + ////////////////////////////////////////////////////////////// + // Interface specification (public) + public: + /// Return if CuUpdatableComponent is updatable? + virtual bool IsUpdatable() const + { return true; } + + /// get gradient and update the parameters in one step + virtual void Update() = 0; + + /// Sets the learning rate of gradient descent + void LearnRate(BaseFloat rate); + /// Gets the learning rate of gradient descent + BaseFloat LearnRate(); + + /// Sets the momentum + void Momentum(BaseFloat mmt); + BaseFloat Momentum(); + + /// Set the weight decay rate to penalize large weights + void Weightcost(BaseFloat cost); + BaseFloat Weightcost(); + + /// Set whether gradient is divided by frames + void GradDivFrm(bool div); + bool GradDivFrm(); + + protected: + BaseFloat mLearningRate; + BaseFloat mMomentum; + BaseFloat mWeightcost; + bool mGradDivFrm; + + }; + + + + + ////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuComponent:: + inline + CuComponent:: + CuComponent(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : mNInputs(nInputs), mNOutputs(nOutputs), + mpInput(NULL), mpErrorInput(NULL), + mOutput(), mErrorOutput(),preComp(pPred) + { + /* DOUBLE LINK the Components */ + if (pPred != NULL) { + SetPrevious(pPred); + pPred->SetNext(this); + } + } + + inline void + CuComponent:: + SetPrevious(CuComponent* pPred) + { + preComp=pPred; + /* DOUBLE LINK the Components */ + if (pPred != NULL) { + SetInput(pPred->GetOutput()); + } + } + + inline void + CuComponent:: + SetNext(CuComponent* pNxt) + { + nxtComp=pNxt; + if (pNxt != NULL) { + SetErrorInput(pNxt->GetErrorOutput()); + } + } + + inline + CuComponent:: + ~CuComponent() + { + ; + } + + inline void + CuComponent:: + Propagate() + { + //initialize output buffer + mOutput.Init(GetInput().Rows(),GetNOutputs()); + //do the dimensionality test + if(GetNInputs() != GetInput().Cols()) { + KALDI_ERR << "Non-matching INPUT dim!!! Network dim: " << GetNInputs() + << " Data dim: " << GetInput().Cols(); + } + //run transform + PropagateF(GetInput(),mOutput); + } + + + inline void + CuComponent:: + Backpropagate() + { + //re-initialize the output buffer + mErrorOutput.Init(GetErrorInput().Rows(),GetNInputs()); + + //do the dimensionality test + assert(GetErrorInput().Cols() == mNOutputs); + assert(mErrorOutput.Cols() == mNInputs); + assert(mErrorOutput.Rows() == GetErrorInput().Rows()); + + //transform + BackpropagateF(GetErrorInput(),mErrorOutput); + } + + + inline void + CuComponent:: + SetInput(CuMatrix<BaseFloat>& rInput,int pos) + { + mpInput = &rInput; + } + + + inline void + CuComponent:: + SetErrorInput(CuMatrix<BaseFloat>& rErrorInput,int pos) + { + mpErrorInput = &rErrorInput; + } + + inline CuMatrix<BaseFloat>& + CuComponent:: + GetInput(int pos) + { + if (NULL == mpInput) Error("mpInput is NULL"); + return *mpInput; + } + + inline CuMatrix<BaseFloat>& + CuComponent:: + GetOutput(int pos) + { + return mOutput; + } + + inline CuMatrix<BaseFloat>& + CuComponent:: + GetErrorInput(int pos) + { + if (NULL == mpErrorInput) Error("mpErrorInput is NULL"); + return *mpErrorInput; + } + + inline CuMatrix<BaseFloat>& + CuComponent:: + GetErrorOutput(int pos) + { + return mErrorOutput; + } + + inline size_t + CuComponent:: + GetNInputs() const + { + return mNInputs; + } + + inline size_t + CuComponent:: + GetNOutputs() const + { + return mNOutputs; + } + + inline int + CuComponent:: + GetInSect() + { + return 1; + } + + inline int + CuComponent:: + GetOutSect() + { + return 1; + } + + inline size_t + CuComponent:: + SetNInputs(size_t nInputs) + { + mNInputs=nInputs; + } + + inline size_t + CuComponent:: + SetNOutputs(size_t nOutputs) + { + mNOutputs=nOutputs; + } + + inline void + CuComponent:: + PropagateF(CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + PropagateFnc(X,Y); + } + inline void + CuComponent:: + BackpropagateF(CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + BackpropagateFnc(X,Y); + } + + + ////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // UpdatableComponent:: + + inline + CuUpdatableComponent:: + CuUpdatableComponent(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuComponent(nInputs, nOutputs, pPred), + mLearningRate(0.0), mMomentum(0), mWeightcost(0), mGradDivFrm(true) + { + ; + } + + + inline + CuUpdatableComponent:: + ~CuUpdatableComponent() + { + ; + } + + + inline void + CuUpdatableComponent:: + LearnRate(BaseFloat rate) + { + mLearningRate = rate; + } + + + inline BaseFloat + CuUpdatableComponent:: + LearnRate() + { + return mLearningRate; + } + + + inline void + CuUpdatableComponent:: + Momentum(BaseFloat mmt) + { + mMomentum = mmt; + } + + + inline BaseFloat + CuUpdatableComponent:: + Momentum() + { + return mMomentum; + } + + + inline void + CuUpdatableComponent:: + Weightcost(BaseFloat cost) + { + mWeightcost = cost; + } + + + inline BaseFloat + CuUpdatableComponent:: + Weightcost() + { + return mWeightcost; + } + + + inline void + CuUpdatableComponent:: + GradDivFrm(bool div) + { + mGradDivFrm = div; + } + + inline bool + CuUpdatableComponent:: + GradDivFrm() + { + return mGradDivFrm; + } + +} // namespace TNet + + +#endif diff --git a/src/CuTNetLib/cuConcatenate.cc b/src/CuTNetLib/cuConcatenate.cc new file mode 100644 index 0000000..f80fe9b --- /dev/null +++ b/src/CuTNetLib/cuConcatenate.cc @@ -0,0 +1,138 @@ + + +#include "cuConcatenate.h" +#include "cuNetwork.h" + + +namespace TNet +{ + + void + CuBlockArray:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + CuMatrix<BaseFloat> colsX; + CuMatrix<BaseFloat> colsY; + + int X_src_ori=0, Y_tgt_ori=0; + for(int i=0; i<mNBlocks; i++) { + //copy column stripe from the input X + int colsX_cnt=mBlocks[i]->GetNInputs(); + colsX.Init(X.Rows(),colsX_cnt); + colsX.CopyCols(colsX_cnt,X_src_ori,X,0); + + //propagate through the block(network) + mBlocks[i]->Propagate(colsX,colsY); + + //copy column stripe to the output Y + int colsY_cnt=mBlocks[i]->GetNOutputs(); + Y.CopyCols(colsY_cnt,0,colsY,Y_tgt_ori); + + //shift the origin coordinates + X_src_ori += colsX_cnt; + Y_tgt_ori += colsY_cnt; + } + + assert(X_src_ori == X.Cols()); + assert(Y_tgt_ori == Y.Cols()); + } + + /// @todo CuBlockArray::BackpropagateFnc not implemented + void + CuBlockArray:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + KALDI_ERR << "Unimplemented"; + } + + void + CuBlockArray:: + Update() + { + KALDI_ERR << "Unimplemented"; + } + + + void + CuBlockArray:: + ReadFromStream(std::istream& rIn) + { + if(mBlocks.size() > 0) { + KALDI_ERR << "Cannot read block vector, " + << "aleady filled bt " + << mBlocks.size() + << "elements"; + } + + rIn >> std::ws >> mNBlocks; + if(mNBlocks < 1) { + KALDI_ERR << "Bad number of blocks:" << mNBlocks; + } + + //read all the blocks + std::string tag; + int block_id; + for(int i=0; i<mNBlocks; i++) { + //read tag <block> + rIn >> std::ws >> tag; + //make it lowercase + std::transform(tag.begin(), tag.end(), tag.begin(), tolower); + //check + if(tag!="<block>") { + KALDI_ERR << "<block> keywotd expected"; + } + + //read block number + rIn >> std::ws >> block_id; + if(block_id != i+1) { + KALDI_ERR << "Expected block number:" << i+1 + << " read block number: " << block_id; + } + + //read the nnet + CuNetwork* p_nnet = new CuNetwork; + p_nnet->ReadNetwork(rIn); + if(p_nnet->Layers() == 0) { + KALDI_ERR << "Cannot read empty network to a block"; + } + + //add it to the vector + mBlocks.push_back(p_nnet); + } + + //check the declared dimensionality + int sum_inputs=0, sum_outputs=0; + for(int i=0; i<mNBlocks; i++) { + sum_inputs += mBlocks[i]->GetNInputs(); + sum_outputs += mBlocks[i]->GetNOutputs(); + } + if(sum_inputs != GetNInputs()) { + KALDI_ERR << "Non-matching number of INPUTS! Declared:" + << GetNInputs() + << " summed from blocks" + << sum_inputs; + } + if(sum_outputs != GetNOutputs()) { + KALDI_ERR << "Non-matching number of OUTPUTS! Declared:" + << GetNOutputs() + << " summed from blocks" + << sum_outputs; + } + } + + + void + CuBlockArray:: + WriteToStream(std::ostream& rOut) + { + rOut << " " << mBlocks.size() << " "; + for(int i=0; i<mBlocks.size(); i++) { + rOut << "<block> " << i+1 << "\n"; + mBlocks[i]->WriteNetwork(rOut); + rOut << "<endblock>\n"; + } + } + + +} //namespace + diff --git a/src/CuTNetLib/cuConcatenate.h b/src/CuTNetLib/cuConcatenate.h new file mode 100644 index 0000000..5454538 --- /dev/null +++ b/src/CuTNetLib/cuConcatenate.h @@ -0,0 +1,90 @@ +#ifndef _CUBLOCK_ARRAY_H_ +#define _CUBLOCK_ARRAY_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + + class CuNetwork; + /** + * \brief Updatable compenent consists of several networks + * + * \ingroup CuNNUpdatable + * Each network is individually propagated and backpropagated with non-overlaping input and output. + * + * Enabling multipath topological structure within the network. + */ + class CuBlockArray : public CuUpdatableComponent + { + public: + + CuBlockArray(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuBlockArray(); + + ComponentType GetType() const; + const char* GetName() const; + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + void Update(); + + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + std::vector<CuNetwork*> mBlocks; ///< vector with networks, one network is one block + size_t mNBlocks; + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuBlockArray:: + inline + CuBlockArray:: + CuBlockArray(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred), + mNBlocks(0) + { } + + + inline + CuBlockArray:: + ~CuBlockArray() + { + for(int i=0; i<mBlocks.size(); i++) { + delete mBlocks[i]; + } + mBlocks.clear(); + } + + inline CuComponent::ComponentType + CuBlockArray:: + GetType() const + { + return CuComponent::BLOCK_ARRAY; + } + + inline const char* + CuBlockArray:: + GetName() const + { + return "<blockarray>"; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/cuDiscreteLinearity.cc b/src/CuTNetLib/cuDiscreteLinearity.cc new file mode 100644 index 0000000..befde24 --- /dev/null +++ b/src/CuTNetLib/cuDiscreteLinearity.cc @@ -0,0 +1,160 @@ + + +#include "cuDiscreteLinearity.h" +#include "cumath.h" + +namespace TNet +{ + + void + CuDiscreteLinearity:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + //Y.SetConst(0.0); + + //precopy bias + Y.AddScaledRow(1.0,mBias,0.0); + + //mulitply with the matrices + int offset_in=0, offset_out=0; + for (int i=0; i<mNBlocks; i++) { + CuMath<BaseFloat>::OffsetGemm('N','N', 1.0, X, mLinearity[i], 1.0, Y, + offset_in, 0, offset_out); + offset_in += mLinearity[i].Rows(); + offset_out += mLinearity[i].Cols(); + } + } + + + void + CuDiscreteLinearity:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + //Y.SetConst(0.0); + + int offset_in=0, offset_out=0; + for(int i=0; i<mNBlocks; i++) { + CuMath<BaseFloat>::OffsetGemm('N', 'T', 1.0, X, mLinearity[i], 0.0, Y, + offset_in, 0, offset_out); + offset_in += mLinearity[i].Cols(); + offset_out += mLinearity[i].Rows(); + } + } + + + void + CuDiscreteLinearity:: + Update() + { + //new implementation + BaseFloat N = 1; + if(mGradDivFrm) { + N = static_cast<BaseFloat>(GetInput().Rows()); + } + BaseFloat mmt_gain = static_cast<BaseFloat>(1.0/(1.0-mMomentum)); + N *= mmt_gain; //compensate higher gradient estimates due to momentum + + //get gradients of discrete linearities + int offset_in=0, offset_out=0; + for(int i=0; i<mNBlocks; i++) { + CuMath<BaseFloat>::OffsetGemm('T','N',1.0, + GetInput(),GetErrorInput(), + mMomentum, mLinearityCorrection[i], + offset_in,offset_out,0); + offset_in += mLinearity[i].Rows(); + offset_out += mLinearity[i].Cols(); + } + for(int i=0; i<mNBlocks; i++) { + //perform update + mLinearity[i].AddScaled(-mLearningRate/N,mLinearityCorrection[i],1.0); + //regularization weight decay + mLinearity[i].AddScaled(-mLearningRate*mWeightcost,mLinearity[i],1.0); + } + + //get gradient of bias + mBiasCorrection.AddColSum(1.0,GetErrorInput(),mMomentum); + //update biases + mBias.AddScaled(-mLearningRate/N,mBiasCorrection,1.0); + } + + + void + CuDiscreteLinearity:: + ReadFromStream(std::istream& rIn) + { + rIn >> std::ws >> mNBlocks; + if(mNBlocks < 1) { + KALDI_ERR << "Bad number of blocks:" << mNBlocks; + } + + mLinearity.resize(mNBlocks); + mLinearityCorrection.resize(mNBlocks); + + int in_dim = 0, out_dim = 0; + for(int i=0; i<mNBlocks; i++) { + //matrix is stored transposed as SNet does + BfMatrix transpose; + rIn >> transpose; + mLinearity[i].CopyFrom(BfMatrix(transpose, TRANS)); + + if(transpose.Cols()*transpose.Rows() == 0) { + Error("Missing linearity matrix in network file"); + } + //allocate training buffers + mLinearityCorrection[i].Init(mLinearity[i].Rows(),mLinearity[i].Cols()); + mLinearityCorrection[i].SetConst(0.0); + + in_dim += transpose.Cols(); + out_dim += transpose.Rows(); + } + + //biases stored normally + BfVector bias; + rIn >> bias; + mBias.CopyFrom(bias); + if(bias.Dim() == 0) { + Error("Missing bias vector in network file"); + } + mBiasCorrection.Init(mBias.Dim()); + mBiasCorrection.SetConst(0.0); + + if(out_dim != GetNOutputs() || + in_dim != GetNInputs() || + mBias.Dim() != GetNOutputs() + ){ + std::ostringstream os; + os << "Wrong dimensionalities of matrix/vector in network file\n" + << "Inputs:" << GetNInputs() + << "Outputs:" << GetNOutputs() + << "\n" + << "linearityCols:" << in_dim + << "linearityRows:" << out_dim + << "biasDims:" << mBias.Dim() + << "\n"; + Error(os.str()); + } + } + + + void + CuDiscreteLinearity:: + WriteToStream(std::ostream& rOut) + { + rOut << mNBlocks << "\n"; + for(int i=0; i< mNBlocks; i++) { + //matrix is stored transposed as SNet does + BfMatrix tmp; + mLinearity[i].CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + } + //biases stored normally + BfVector vec; + mBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + } + + +} //namespace + diff --git a/src/CuTNetLib/cuDiscreteLinearity.h b/src/CuTNetLib/cuDiscreteLinearity.h new file mode 100644 index 0000000..8a70fa7 --- /dev/null +++ b/src/CuTNetLib/cuDiscreteLinearity.h @@ -0,0 +1,97 @@ +#ifndef _CUDISCRETE_LINEARITY_H_ +#define _CUDISCRETE_LINEARITY_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +#include "Matrix.h" +#include "Vector.h" + +#include <vector> + + +namespace TNet { + /** + * \brief CuDiscreteLinearity Summation function + * + * \ingroup CuNNUpdatable + * Similar to CuSharedLinearity except that mBias is not shared, + * only mLinearity is used as multiple instances. + * \sa CuBiasedLinearity + */ + class CuDiscreteLinearity : public CuUpdatableComponent + { + public: + + CuDiscreteLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuDiscreteLinearity(); + + ComponentType GetType() const; + const char* GetName() const; + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + void Update(); + + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + std::vector<CuMatrix<BaseFloat> > mLinearity; ///< Matrix with neuron weights + CuVector<BaseFloat> mBias; ///< Vector with biases + + std::vector<CuMatrix<BaseFloat> > mLinearityCorrection; ///< Matrix for linearity updates + CuVector<BaseFloat> mBiasCorrection; ///< Vector for bias updates + + size_t mNBlocks; + + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuDiscreteLinearity:: + inline + CuDiscreteLinearity:: + CuDiscreteLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred), + //mLinearity(nInputs,nOutputs), mBias(nOutputs), + //mLinearityCorrection(nInputs,nOutputs), mBiasCorrection(nOutputs) + mNBlocks(0) + { + //mLinearityCorrection.SetConst(0.0); + //mBiasCorrection.SetConst(0.0); + } + + + inline + CuDiscreteLinearity:: + ~CuDiscreteLinearity() + { } + + inline CuComponent::ComponentType + CuDiscreteLinearity:: + GetType() const + { + return CuComponent::DISCRETE_LINEARITY; + } + + inline const char* + CuDiscreteLinearity:: + GetName() const + { + return "<discretelinearity>"; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/cuLinearity.cc b/src/CuTNetLib/cuLinearity.cc new file mode 100644 index 0000000..5fb247d --- /dev/null +++ b/src/CuTNetLib/cuLinearity.cc @@ -0,0 +1,107 @@ + + +#include "cuLinearity.h" + + +namespace TNet +{ + + void + CuLinearity:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + //Y.SetConst(0.0); + Y.Gemm('N','N', 1.0, X, mLinearity, 0.0); + } + + + void + CuLinearity:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + //Y.SetConst(0.0); + Y.Gemm('N', 'T', 1.0, X, mLinearity, 0.0); + } + + + void + CuLinearity:: + Update() + { +#if 0 + //former implementation + BaseFloat N = static_cast<BaseFloat>(GetInput().Rows()); + + mLinearityCorrection.Gemm('T','N',-mLearningRate/N,GetInput(),GetErrorInput(),mMomentum); + mBiasCorrection.AddColSum(-mLearningRate/N,GetErrorInput(),mMomentum); + + //regularization weight decay + mLinearityCorrection.AddScaled(-mLearningRate*mWeightcost,mLinearity,1.0); + + mLinearity.AddScaled(1.0,mLinearityCorrection,1.0); + mBias.AddScaled(1.0,mBiasCorrection,1.0); +#endif + +#if 1 + //new implementation + BaseFloat N = 1; + if(mGradDivFrm) { + N = static_cast<BaseFloat>(GetInput().Rows()); + } + BaseFloat mmt_gain = static_cast<BaseFloat>(1.0/(1.0-mMomentum)); + N *= mmt_gain; + + mLinearityCorrection.Gemm('T','N',1.0,GetInput(),GetErrorInput(),mMomentum); + + mLinearity.AddScaled(-mLearningRate/N,mLinearityCorrection,1.0); + + //regularization weight decay (from actual weights only) + BaseFloat L2_decay = -mLearningRate*mWeightcost*(mGradDivFrm?1.0:GetInput().Rows()); + mLinearity.AddScaled(L2_decay, mLinearity,1.0); +#endif + } + + + void + CuLinearity:: + ReadFromStream(std::istream& rIn) + { + //matrix is stored transposed as SNet does + BfMatrix transpose; + rIn >> transpose; + mLinearity.CopyFrom(BfMatrix(transpose, TRANS)); + + /*if(transpose.Cols()*transpose.Rows() == 0) { + Error("Missing linearity matrix in network file"); + }*/ + if(mLinearity.Cols() != GetNOutputs() || + mLinearity.Rows() != GetNInputs() + ){ + std::ostringstream os; + os << "Wrong dimensionalities of matrix/vector in network file\n" + << "Inputs:" << GetNInputs() + << "Outputs:" << GetNOutputs() + << "\n" + << "linearityCols:" << mLinearity.Cols() + << "linearityRows:" << mLinearity.Rows() + << "\n"; + Error(os.str()); + } + } + + + void + CuLinearity:: + WriteToStream(std::ostream& rOut) + { + //matrix is stored transposed as SNet does + BfMatrix tmp; + mLinearity.CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + rOut << std::endl; + } + + +} //namespace + diff --git a/src/CuTNetLib/cuLinearity.h b/src/CuTNetLib/cuLinearity.h new file mode 100644 index 0000000..050591d --- /dev/null +++ b/src/CuTNetLib/cuLinearity.h @@ -0,0 +1,94 @@ +#ifndef _CULINEARITY_H_ +#define _CULINEARITY_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + /** + * \brief CuLinearity summation function + * + * \ingroup CuNNUpdatable + * Implements forward pass: \f[ Y_j=\Sigma_{i=0}^{i=N-1}w_ij X_i +{\beta}_j \f] + * Error propagation: \f[ E_i = \Sigma_{i=0}^{i=N-1} w_ij e_j \f] + * + * Weight adjustion: \f[ W_{ij} = (1-D)(w_{ij} - \alpha(1-\mu)x_i e_j - \mu \Delta) \f] + * where + * - D for weight decay => penalizing large weight + * - \f$ \alpha \f$ for learning rate + * - \f$ \mu \f$ for momentum => avoiding oscillation + */ + class CuLinearity : public CuUpdatableComponent + { + public: + + CuLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuLinearity(); + + ComponentType GetType() const; + const char* GetName() const; + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + void Update(); + + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + CuMatrix<BaseFloat> mLinearity; ///< Matrix with neuron weights + + CuMatrix<BaseFloat> mLinearityCorrection; ///< Matrix for linearity updates + + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuLinearity:: + inline + CuLinearity:: + CuLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred), + mLinearity(nInputs,nOutputs), + mLinearityCorrection(nInputs,nOutputs) + { + mLinearityCorrection.SetConst(0.0); + } + + + inline + CuLinearity:: + ~CuLinearity() + { } + + inline CuComponent::ComponentType + CuLinearity:: + GetType() const + { + return CuComponent::LINEARITY; + } + + inline const char* + CuLinearity:: + GetName() const + { + return "<linearity>"; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/cuMisc.h b/src/CuTNetLib/cuMisc.h new file mode 100644 index 0000000..7319adf --- /dev/null +++ b/src/CuTNetLib/cuMisc.h @@ -0,0 +1,555 @@ +#ifndef _CUMISC_H_ +#define _CUMISC_H_ + +#include <vector> + +#include "cuComponent.h" +#include "cumatrix.h" + + +#include "Matrix.h" +#include "Vector.h" +#include "Error.h" + + +namespace TNet { + /** + * \brief A pipe for input and errorinput propagation(doesn't incurr copy) + * + * \ingroup CuNNMisc + */ + class CuPipe : public CuComponent + { + public: + CuPipe(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred) + { } + + ~CuPipe() + { } + + ComponentType GetType() const + { return PIPE; } + + const char* GetName() const + { return "<pipe>"; } + + void ReadFromStream(std::istream& rIn) + { } + + void WriteToStream(std::ostream& rOut) + { } + + void Propagate() + { + if (NULL == mpInput) Error("mpInput is NULL"); + mOutput.Init(*mpInput); + } + void BackPropagate() + { + if (NULL == mpErrorInput) Error("mpErrorInput is NULL"); + mErrorOutput.Init(*mpErrorInput); + } + + protected: + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Y.CopyFrom(X);} + + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Y.CopyFrom(X); } + + }; + + /** + * \brief A pipe for input propagation(doesn't incurr copy) and set any error to zero + * + * \ingroup CuNNMisc + * + * @todo have to be set to zero on every pass(setup a common zeroed space?!) + */ + class CuLearnStop : public CuComponent + { + public: + CuLearnStop(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred) + { } + + ~CuLearnStop() + { } + + ComponentType GetType() const + { return LEARNSTOP; } + + const char* GetName() const + { return "<learnstop>"; } + + void ReadFromStream(std::istream& rIn) + { } + + void WriteToStream(std::ostream& rOut) + { } + + void Propagate() + { + if (NULL == mpInput) Error("mpInput is NULL"); + mOutput.Init(*mpInput); + } + + protected: + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Y.CopyFrom(X);} + + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Y.SetZero(); } + + }; + + /** + * \brief Distribute the input to several output + * + * \ingroup CuNNMisc + * + */ + class CuDistrib : public CuComponent + { + public: + CuDistrib(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred),size(0),ErrInputVec() + { + } + + ~CuDistrib() + { } + + ComponentType GetType() const + { return DISTRIB; } + + const char* GetName() const + { return "<distrib>"; } + + void ReadFromStream(std::istream& rIn) + { + rIn >> std::ws >> size; + ErrInputVec.clear(); + for (int i=0; i<size;++i) + ErrInputVec.push_back(NULL); + } + + void WriteToStream(std::ostream& rOut) + { + rOut<<size<<std::endl; + } + + void Propagate() + { + if (NULL == mpInput) Error("mpInput is NULL"); + mOutput.Init(*mpInput); + } + + int GetOutSect() + { + return size; + } + + CuMatrix<BaseFloat>& GetErrorInput(int pos=0) + { + if (pos>=0 && pos<size) + return *ErrInputVec[pos]; + return *ErrInputVec[0]; + } + + void SetErrorInput(CuMatrix<BaseFloat>& rErrorInput,int pos=0) + { + if (pos==0) + mpErrorInput=&rErrorInput; + if (pos>=0 && pos<size) + ErrInputVec[pos]=&rErrorInput; + } + + protected: + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Y.CopyFrom(X);} + + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + Y.SetZero(); + for (int i=0;i<size;++i) + Y.AddScaled(1.0,*ErrInputVec[i],1.0); + } + + int size; + MatrixPtrVec ErrInputVec; + Vector<BaseFloat> Scale; + }; + + /** + * \brief Combining(Adding) several inputs together + * + * \ingroup CuNNMisc + * + */ + class CuCombine : public CuComponent + { + public: + CuCombine(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred),size(0),InputVec() + { + } + + ~CuCombine() + { } + + ComponentType GetType() const + { return COMBINE; } + + const char* GetName() const + { return "<combine>"; } + + void ReadFromStream(std::istream& rIn) + { + rIn >> std::ws >> size; + InputVec.clear(); + for (int i=0; i<size;++i) + InputVec.push_back(NULL); + } + + void WriteToStream(std::ostream& rOut) + { + rOut<<size<<std::endl; + } + + void Backpropagate() + { + if (NULL == mpErrorInput) Error("mpErrorInput is NULL"); + mErrorOutput.Init(*mpErrorInput); + } + + int GetInSect() + { + return size; + } + + /// IO Data getters + CuMatrix<BaseFloat>& GetInput(int pos=0) + { + if (pos>=0 && pos<size) + return *InputVec[pos]; + return *InputVec[0]; + } + + /// Set input vector (bind with the preceding NetworkComponent) + void SetInput(CuMatrix<BaseFloat>& rInput,int pos=0) + { + if (pos==0) + mpInput=&rInput; + if (pos>=0 && pos<size) + InputVec[pos]=&rInput; + } + + protected: + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + Y.SetZero(); + for (int i=0;i<size;++i) + Y.AddScaled(1.0,*InputVec[i],1.0); + } + + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + Y.CopyFrom(X); + } + + int size; + MatrixPtrVec InputVec; + }; + + /** + * \brief Divide the input matrix to several outputs + * + * \ingroup CuNNMisc + * + */ + class CuDivide : public CuComponent + { + public: + CuDivide(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred),size(0) + { } + + ~CuDivide() + { } + + ComponentType GetType() const + { return DIVIDE; } + + const char* GetName() const + { return "<divide>"; } + + int GetOutSect() + { + return size; + } + + void ReadFromStream(std::istream& rIn) + { + int len; + for (int i=0; i<size;++i) + delete OutputVec[i]; + rIn >> std::ws >> size; + OutputVec.clear(); + for (int i=0; i<size;++i) + { + rIn>>len; + OutputVec.push_back(new CuMatrix<BaseFloat>()); + SectLen.push_back(len); + } + } + + void WriteToStream(std::ostream& rOut) + { + rOut<<size<<" "; + for (int i=0; i<size;++i) + rOut<<SectLen[i]<<" "; + rOut<<std::endl; + } + + void Propagate() + { + if (NULL == mpInput) Error("mpInput is NULL"); + int loc=0; + for (int i=0;i<size;++i) + { + OutputVec[i]->Init(*mpInput,loc,SectLen[i]); + loc+=SectLen[i]; + } + } + + protected: + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Error("__func__ Nonsense"); } + + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + int loc=0; + for (int i=0;i<size;++i) + { + Y.CopyCols(SectLen[i], 0, X, loc); + loc+=SectLen[i]; + } + } + + int size; + MatrixPtrVec OutputVec; + MatrixPtrVec ErrorInputVec; + std::vector<int> SectLen; + + }; + + /** + * \brief Merge several input matrices to one single output + * + * \ingroup CuNNMisc + * + */ + class CuMerge : public CuComponent + { + public: + CuMerge(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred) + { } + + ~CuMerge() + { } + + ComponentType GetType() const + { return MERGE; } + + const char* GetName() const + { return "<merge>"; } + + int GetInSect() + { + return size; + } + + void ReadFromStream(std::istream& rIn) + { + int len; + for (int i=0; i<size;++i) + delete ErrorOutputVec[i]; + rIn >> std::ws >> size; + ErrorOutputVec.clear(); + for (int i=0; i<size;++i) + { + rIn>>len; + ErrorOutputVec.push_back(new CuMatrix<BaseFloat>()); + SectLen.push_back(len); + } + } + + void WriteToStream(std::ostream& rOut) + { + rOut<<size<<" "; + for (int i=0; i<size;++i) + rOut<<SectLen[i]<<" "; + rOut<<std::endl; + } + + void Backpropagate() + { + if (NULL == mpErrorInput) Error("mpErrorInput is NULL"); + int loc=0; + for (int i=0;i<size;++i) + { + ErrorOutputVec[i]->Init(*mpErrorInput,loc,SectLen[i]); + loc+=SectLen[i]; + } + } + + protected: + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + int loc=0; + for (int i=0;i<size;++i) + { + Y.CopyCols(SectLen[i], 0, X, loc); + loc+=SectLen[i]; + } + } + + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Error("__func__ Nonsense"); } + + int size; + + MatrixPtrVec InputVec; + MatrixPtrVec ErrorOutputVec; + std::vector<int> SectLen; + + }; + + /** + * \brief Reordering several inputs + * + * \ingroup CuNNMisc + * + */ + class CuReorder : public CuComponent + { + public: + CuReorder(size_t nInputs, size_t nOutputs, CuComponent* pPred) + : CuComponent(nInputs,nOutputs,pPred) + { } + + ~CuReorder() + { } + + ComponentType GetType() const + { return REORDER; } + + const char* GetName() const + { return "<reorder>"; } + + int GetInSect() + { + return size; + } + + int GetOutSect() + { + return size; + } + + void ReadFromStream(std::istream& rIn) + { + int pos; + for (int i=0; i<size;++i) + delete PipeVec[i]; + rIn >> std::ws >> size; + Order.clear(); + PipeVec.clear(); + for (int i=0; i<size;++i) + { + rIn>>pos; + Order.push_back(pos); + PipeVec.push_back(new CuPipe(0,0,NULL)); + } + } + + void WriteToStream(std::ostream& rOut) + { + rOut << size<< " "; + for (int i=0; i<size;++i) + rOut<<Order[i]<<" "; + rOut<<std::endl; + } + + void Propagate() + { + if (NULL == mpInput) Error("mpInput is NULL"); + for (int i=0; i<size;++i) + PipeVec[i]->Propagate(); + } + + void Backpropagate() + { + if (NULL == mpErrorInput) Error("mpErrorInput is NULL"); + for (int i=0; i<size;++i) + PipeVec[i]->Backpropagate(); + } + + /// IO Data getters + CuMatrix<BaseFloat>& GetInput(int pos=0) + { + return PipeVec[pos]->GetInput(); + } + CuMatrix<BaseFloat>& GetOutput(int pos=0) + { + return PipeVec[Order[pos]]->GetOutput(); + } + CuMatrix<BaseFloat>& GetErrorInput(int pos=0) + { + return PipeVec[Order[pos]]->GetErrorInput(); + } + CuMatrix<BaseFloat>& GetErrorOutput(int pos=0) + { + return PipeVec[pos]->GetErrorOutput(); + } + + /// Set input vector (bind with the preceding NetworkComponent) + void SetInput(CuMatrix<BaseFloat>& rInput,int pos=0) + { + PipeVec[pos]->SetInput(rInput); + } + /// Set error input vector (bind with the following NetworkComponent) + void SetErrorInput(CuMatrix<BaseFloat>& rErrorInput,int pos=0) + { + PipeVec[Order[pos]]->SetErrorInput(rErrorInput); + } + + protected: + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Error("__func__ Nonsense"); } + + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { Error("__func__ Nonsense"); } + + int size; + + std::vector<int> Order; + + std::vector< CuPipe* > PipeVec; + }; + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/cuNetwork.cc b/src/CuTNetLib/cuNetwork.cc new file mode 100644 index 0000000..e0d8549 --- /dev/null +++ b/src/CuTNetLib/cuNetwork.cc @@ -0,0 +1,442 @@ + +#include <algorithm> +//#include <locale> +#include <cctype> +#include <list> +#include <sstream> + +#include "cuNetwork.h" + +#include "cuDiscreteLinearity.h" +#include "cuSharedLinearity.h" +#include "cuSparseLinearity.h" +#include "cuRbm.h" +#include "cuRbmSparse.h" +#include "cuRecurrent.h" +#include "cuBlockArray.h" +#include "cuLinearity.h" +#include "cuUpdatableBias.h" +#include "cuMisc.h" +#include "cuCompDisc.h" + +namespace TNet { + + + + + void + CuNetwork:: + ReadNetwork(const char* pSrc) + { + std::ifstream in(pSrc); + if(!in.good()) { + Error(std::string("Error, cannot read model: ")+pSrc); + } + ReadNetwork(in); + in.close(); + } + + + + void + CuNetwork:: + WriteNetwork(const char* pDst) + { + std::ofstream out(pDst); + if(!out.good()) { + Error(std::string("Error, cannot write model: ")+pDst); + } + WriteNetwork(out); + out.close(); + } + + + + void + CuNetwork:: + ReadNetwork(std::istream& rIn) + { + //get the network elements from a factory + CuComponent *pComp; + while(NULL != (pComp = ComponentFactory(rIn))) { + mNetComponents.push_back(pComp); + } + } + + + + void + CuNetwork:: + WriteNetwork(std::ostream& rOut) + { + //dump all the componetns + LayeredType::iterator it; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + ComponentDumper(rOut, **it); + } + } + + + void + CuNetwork:: + SetLearnRate(BaseFloat learnRate, const char* pLearnRateFactors) + { + //parse the learn rate factors: "0.1:0.5:0.6:1.0" to std::list + std::list<BaseFloat> lr_factors; + if(NULL != pLearnRateFactors) { + //replace ':' by ' ' + std::string str(pLearnRateFactors); + size_t pos = 0; + while((pos = str.find(':',pos)) != std::string::npos) str[pos] = ' '; + while((pos = str.find(',',pos)) != std::string::npos) str[pos] = ' '; + + //parse to std::list + std::istringstream is(str); + is >> std::skipws; + BaseFloat f; + while(!is.eof()) { + if(!(is >> f).fail()) { lr_factors.push_back(f); } + else break; + } + } + + //initialize rate factors iterator + BaseFloat scale = 1.0f; + + //store global learning rate + mGlobLearnRate = learnRate; + mpLearnRateFactors = pLearnRateFactors; + + //give scaled learning rate to components + LayeredType::iterator it; + bool stopper_given = false; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + if((*it)->IsUpdatable()) { + //get next scale factor + if(NULL != pLearnRateFactors) { + if(!(lr_factors.size() > 0)) { + Error("Too few learninig rate scale factors"); + } + scale = lr_factors.front(); + lr_factors.pop_front(); + } + //set scaled learning rate to the component + dynamic_cast<CuUpdatableComponent*>(*it)->LearnRate(learnRate*scale); + //set the stopper component for backpropagation + if(!stopper_given && (learnRate*scale > 0.0)) { + mpPropagErrorStopper = *it; stopper_given = true; + } + } + } + if(lr_factors.size() > 0) { + Error("Too much learninig rate scale factors"); + } + } + + + BaseFloat + CuNetwork:: + GetLearnRate() + { + return mGlobLearnRate; + } + + + void + CuNetwork:: + PrintLearnRate() + { + assert(mNetComponents.size() > 0); + std::cout << "Learning rate: global " << mGlobLearnRate; + std::cout << " components' "; + for(size_t i=0; i<mNetComponents.size(); i++) { + if(mNetComponents[i]->IsUpdatable()) { + std::cout << " " << dynamic_cast<CuUpdatableComponent*>(mNetComponents[i])->LearnRate(); + } + } + std::cout << "\n" << std::flush; + } + + + + void + CuNetwork:: + SetMomentum(BaseFloat momentum) + { + LayeredType::iterator it; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + if((*it)->IsUpdatable()) { + dynamic_cast<CuUpdatableComponent*>(*it)->Momentum(momentum); + } + } + } + + void + CuNetwork:: + SetWeightcost(BaseFloat weightcost) + { + LayeredType::iterator it; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + if((*it)->IsUpdatable()) { + dynamic_cast<CuUpdatableComponent*>(*it)->Weightcost(weightcost); + } + } + } + + void + CuNetwork:: + SetL1(BaseFloat l1) + { + LayeredType::iterator it; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + if((*it)->GetType() == CuComponent::SPARSE_LINEARITY) { + dynamic_cast<CuSparseLinearity*>(*it)->L1(l1); + } + } + } + + void + CuNetwork:: + SetGradDivFrm(bool div) + { + LayeredType::iterator it; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + if((*it)->IsUpdatable()) { + dynamic_cast<CuUpdatableComponent*>(*it)->GradDivFrm(div); + } + } + } + + + CuComponent* + CuNetwork:: + ComponentFactory(std::istream& rIn) + { + rIn >> std::ws; + if(rIn.eof()) return NULL; + + CuComponent* pRet=NULL; + CuComponent* pPred=NULL; + + //make coupling with predecessor + if(mNetComponents.size() != 0) { + pPred = mNetComponents.back(); + } + + pRet=ComponentReader(rIn, pPred); + + //return + return pRet; + } + + CuComponent* + CuNetwork:: + ComponentReader(std::istream& rIn, CuComponent* pPred) + { + CuComponent* pRet=NULL; + + std::string componentTag; + size_t nInputs, nOutputs; + + rIn >> std::ws; + rIn >> componentTag; + if(componentTag == "") return NULL; //nothing left in the file + + //make it lowercase + std::transform(componentTag.begin(), componentTag.end(), + componentTag.begin(), tolower); + + if(componentTag[0] != '<' || componentTag[componentTag.size()-1] != '>') { + Error(std::string("Invalid component tag:")+componentTag); + } + + //the 'endblock' tag terminates the network + if(componentTag == "<endblock>") return NULL; + + rIn >> std::ws; + rIn >> nOutputs; + rIn >> std::ws; + rIn >> nInputs; + assert(nInputs > 0 && nOutputs > 0); + + //array with list of component tags + static const std::string TAGS[] = { + "<biasedlinearity>", + "<discretelinearity>", + "<sharedlinearity>", + "<sparselinearity>", + "<rbm>", + "<rbmsparse>", + "<recurrent>", + + "<softmax>", + "<sigmoid>", + + "<expand>", + "<copy>", + "<transpose>", + "<blocklinearity>", + "<bias>", + "<window>", + "<log>", + + "<blockarray>", + + "<linearity>", + "<updatablebias>", + "<pipe>", + "<learnstop>", + "<distrib>", + "<combine>", + "<compound>", + "<discrete>", + "<divide>", + "<merge>", + "<reorder>", + + }; + + static const int n_tags = sizeof(TAGS) / sizeof(TAGS[0]); + int i; + for(i=0; i<n_tags; i++) { + if(componentTag == TAGS[i]) break; + } + + //switch according to position in array TAGS + switch(i) { + case 0: pRet = new CuBiasedLinearity(nInputs,nOutputs,pPred); break; + case 1: pRet = new CuDiscreteLinearity(nInputs,nOutputs,pPred); break; + case 2: pRet = new CuSharedLinearity(nInputs,nOutputs,pPred); break; + case 3: pRet = new CuSparseLinearity(nInputs,nOutputs,pPred); break; + case 4: pRet = new CuRbm(nInputs,nOutputs,pPred); break; + case 5: pRet = new CuRbmSparse(nInputs,nOutputs,pPred); break; + case 6: pRet = new CuRecurrent(nInputs,nOutputs,pPred); break; + + case 7: pRet = new CuSoftmax(nInputs,nOutputs,pPred); break; + case 8: pRet = new CuSigmoid(nInputs,nOutputs,pPred); break; + + case 9: pRet = new CuExpand(nInputs,nOutputs,pPred); break; + case 10: pRet = new CuCopy(nInputs,nOutputs,pPred); break; + case 11: pRet = new CuTranspose(nInputs,nOutputs,pPred); break; + case 12: pRet = new CuBlockLinearity(nInputs,nOutputs,pPred); break; + case 13: pRet = new CuBias(nInputs,nOutputs,pPred); break; + case 14: pRet = new CuWindow(nInputs,nOutputs,pPred); break; + case 15: pRet = new CuLog(nInputs,nOutputs,pPred); break; + + case 16: pRet = new CuBlockArray(nInputs,nOutputs,pPred); break; + + case 17: pRet = new CuLinearity(nInputs,nOutputs,pPred); break; + case 18: pRet = new CuUpdatableBias(nInputs,nOutputs,pPred); break; + case 19: pRet = new CuPipe(nInputs,nOutputs,pPred); break; + case 20: pRet = new CuLearnStop(nInputs,nOutputs,pPred); break; + case 21: pRet = new CuDistrib(nInputs,nOutputs,pPred); break; + case 22: pRet = new CuCombine(nInputs,nOutputs,pPred); break; + case 23: pRet = new CuCompound(nInputs,nOutputs,pPred); break; + case 24: pRet = new CuDiscrete(nInputs,nOutputs,pPred); break; + case 25: pRet = new CuDivide(nInputs,nOutputs,pPred); break; + case 26: pRet = new CuMerge(nInputs,nOutputs,pPred); break; + case 27: pRet = new CuReorder(nInputs,nOutputs,pPred); break; + + + default: Error(std::string("Unknown Component tag:")+componentTag); + } + + //read components content + pRet->ReadFromStream(rIn); + + //return + return pRet; + } + + + void + CuNetwork:: + ComponentDumper(std::ostream& rOut, CuComponent& rComp) + { + //use tags of all the components; or the identification codes + //array with list of component tags + static const CuComponent::ComponentType TYPES[] = { + CuComponent::BIASED_LINEARITY, + CuComponent::DISCRETE_LINEARITY, + CuComponent::SHARED_LINEARITY, + CuComponent::SPARSE_LINEARITY, + CuComponent::RBM, + CuComponent::RBM_SPARSE, + CuComponent::RECURRENT, + CuComponent::LINEARITY, + CuComponent::UPDATABLEBIAS, + CuComponent::COMPOUND, + CuComponent::DISCRETE, + + CuComponent::SIGMOID, + CuComponent::SOFTMAX, + + CuComponent::EXPAND, + CuComponent::COPY, + CuComponent::TRANSPOSE, + CuComponent::BLOCK_LINEARITY, + CuComponent::BIAS, + CuComponent::WINDOW, + CuComponent::LOG, + CuComponent::PIPE, + CuComponent::LEARNSTOP, + CuComponent::DISTRIB, + CuComponent::COMBINE, + CuComponent::DIVIDE, + CuComponent::MERGE, + CuComponent::REORDER, + + CuComponent::BLOCK_ARRAY, + }; + static const std::string TAGS[] = { + "<biasedlinearity>", + "<discretelinearity>", + "<sharedlinearity>", + "<sparselinearity>", + "<rbm>", + "<rbmsparse>", + "<recurrent>", + "<linearity>", + "<updatablebias>", + "<compound>", + "<discrete>", + + "<sigmoid>", + "<softmax>", + + "<expand>", + "<copy>", + "<transpose>", + "<blocklinearity>", + "<bias>", + "<window>", + "<log>", + "<pipe>", + "<learnstop>", + "<distrib>", + "<combine>", + "<divide>", + "<merge>", + "<reorder>", + + "<blockarray>", + }; + static const int MAX = sizeof TYPES / sizeof TYPES[0]; + + int i; + for(i=0; i<MAX; ++i) { + if(TYPES[i] == rComp.GetType()) break; + } + if(i == MAX) Error("Unknown ComponentType"); + + //dump the component tag + rOut << TAGS[i] << " " + << rComp.GetNOutputs() << " " + << rComp.GetNInputs() << std::endl; + + //write components content + rComp.WriteToStream(rOut); + } + +} //namespace + diff --git a/src/CuTNetLib/cuNetwork.h b/src/CuTNetLib/cuNetwork.h new file mode 100644 index 0000000..05e0ecb --- /dev/null +++ b/src/CuTNetLib/cuNetwork.h @@ -0,0 +1,227 @@ +#ifndef _CUNETWORK_H_ +#define _CUNETWORK_H_ + +#include "cuComponent.h" + +#include "cuBiasedLinearity.h" +//#include "cuBlockLinearity.h" +//#include "cuBias.h" +//#include "cuWindow.h" + +#include "cuActivation.h" + +#include "cuCRBEDctFeat.h" + +#include "Vector.h" + +#include <vector> + +/** + * \file cuNetwork.h + * \brief CuNN manipulation class + */ + +/// \defgroup CuNNComp CuNN Components + +namespace TNet { + /** + * \brief Nural Network Manipulator & public interfaces + * + * \ingroup CuNNComp + */ + class CuNetwork + { + ////////////////////////////////////// + // Typedefs + typedef std::vector<CuComponent*> LayeredType; + + ////////////////////////////////////// + // Disable copy construction, assignment and default constructor + private: + CuNetwork(CuNetwork&); + CuNetwork& operator=(CuNetwork&); + + public: + CuNetwork() { } + CuNetwork(std::istream& rIn); + ~CuNetwork(); + + void AddLayer(CuComponent* layer); + + int Layers() + { return mNetComponents.size(); } + + CuComponent& Layer(int i) + { return *mNetComponents[i]; } + + /// forward the data to the output + void Propagate(CuMatrix<BaseFloat>& in, CuMatrix<BaseFloat>& out); + + /// backpropagate the error while updating weights + void Backpropagate(CuMatrix<BaseFloat>& globerr); + + void ReadNetwork(const char* pSrc); ///< read the network from file + void WriteNetwork(const char* pDst); ///< write network to file + + void ReadNetwork(std::istream& rIn); ///< read the network from stream + void WriteNetwork(std::ostream& rOut); ///< write network to stream + + size_t GetNInputs() const; ///< Dimensionality of the input features + size_t GetNOutputs() const; ///< Dimensionality of the desired vectors + + /// set the learning rate + void SetLearnRate(BaseFloat learnRate, const char* pLearnRateFactors = NULL); + BaseFloat GetLearnRate(); ///< get the learning rate value + void PrintLearnRate(); ///< log the learning rate values + + void SetMomentum(BaseFloat momentum); + void SetWeightcost(BaseFloat weightcost); + void SetL1(BaseFloat l1); + + void SetGradDivFrm(bool div); + + /// Reads a component from a stream + static CuComponent* ComponentReader(std::istream& rIn, CuComponent* pPred); + /// Dumps component into a stream + static void ComponentDumper(std::ostream& rOut, CuComponent& rComp); + + + private: + /// Creates a component by reading from stream + CuComponent* ComponentFactory(std::istream& In); + + + private: + LayeredType mNetComponents; ///< container with the network layers + CuComponent* mpPropagErrorStopper; + BaseFloat mGlobLearnRate; ///< The global (unscaled) learn rate of the network + const char* mpLearnRateFactors; ///< The global (unscaled) learn rate of the network + + + //friend class NetworkGenerator; //<< For generating networks... + + }; + + ////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuNetwork:: + inline + CuNetwork:: + CuNetwork(std::istream& rSource) + : mpPropagErrorStopper(NULL), mGlobLearnRate(0.0), mpLearnRateFactors(NULL) + { + ReadNetwork(rSource); + } + + + inline + CuNetwork:: + ~CuNetwork() + { + //delete all the components + LayeredType::iterator it; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + delete *it; + *it = NULL; + } + mNetComponents.resize(0); + } + + + inline void + CuNetwork:: + AddLayer(CuComponent* layer) + { + if(mNetComponents.size() > 0) { + if(GetNOutputs() != layer->GetNInputs()) { + Error("Nonmatching dims"); + } + layer->SetPrevious(mNetComponents.back()); + mNetComponents.back()->SetNext(layer); + } + mNetComponents.push_back(layer); + } + + + inline void + CuNetwork:: + Propagate(CuMatrix<BaseFloat>& in, CuMatrix<BaseFloat>& out) + { + //empty network => copy input + if(mNetComponents.size() == 0) { + out.CopyFrom(in); + return; + } + + //check dims + if(in.Cols() != GetNInputs()) { + std::ostringstream os; + os << "Nonmatching dims" + << " data dim is: " << in.Cols() + << " network needs: " << GetNInputs(); + Error(os.str()); + } + mNetComponents.front()->SetInput(in); + + //propagate + LayeredType::iterator it; + for(it=mNetComponents.begin(); it!=mNetComponents.end(); ++it) { + (*it)->Propagate(); + } + + //copy the output + out.CopyFrom(mNetComponents.back()->GetOutput()); + } + + + + + inline void + CuNetwork:: + Backpropagate(CuMatrix<BaseFloat>& globerr) + { + mNetComponents.back()->SetErrorInput(globerr); + + // back-propagation + LayeredType::reverse_iterator it; + for(it=mNetComponents.rbegin(); it!=mNetComponents.rend(); ++it) { + //stopper component does not propagate error (no updatable predecessors) + if(*it != mpPropagErrorStopper) { + //compute errors for preceding network components + (*it)->Backpropagate(); + } + //update weights if updatable component + if((*it)->IsUpdatable()) { + CuUpdatableComponent& rComp = dynamic_cast<CuUpdatableComponent&>(**it); + if(rComp.LearnRate() > 0.0f) { + rComp.Update(); + } + } + //stop backprop if no updatable components precede current component + if(mpPropagErrorStopper == *it) break; + } + } + + + inline size_t + CuNetwork:: + GetNInputs() const + { + if(!mNetComponents.size() > 0) return 0; + return mNetComponents.front()->GetNInputs(); + } + + + inline size_t + CuNetwork:: + GetNOutputs() const + { + if(!mNetComponents.size() > 0) return 0; + return mNetComponents.back()->GetNOutputs(); + } + +} //namespace + +#endif + + diff --git a/src/CuTNetLib/cuObjectiveFunction.cc b/src/CuTNetLib/cuObjectiveFunction.cc new file mode 100644 index 0000000..e2b0a1d --- /dev/null +++ b/src/CuTNetLib/cuObjectiveFunction.cc @@ -0,0 +1,87 @@ + +#include "cuObjectiveFunction.h" + +#include "Error.h" +#include "cumath.h" + + +namespace TNet +{ + + + + CuObjectiveFunction* + CuObjectiveFunction:: + Factory(ObjFunType type) { + CuObjectiveFunction* ret = NULL; + switch(type) { + case MEAN_SQUARE_ERROR: ret = new CuMeanSquareError; break; + case CROSS_ENTROPY: ret = new CuCrossEntropy; break; + default: Error("Unknown ObjFun type"); + } + return ret; + } + + + void + CuMeanSquareError:: + Evaluate(const CuMatrix<BaseFloat>& rNetOutput, const CuMatrix<BaseFloat>& rDesired, CuMatrix<BaseFloat>& rNetError) + { + //get the global error + rNetError.CopyFrom(rNetOutput); + rNetError.AddScaled(-1.0,rDesired,1.0); + + //calculate the MSE + mAuxMat.CopyFrom(rNetError); + mAuxMat.MulElem(mAuxMat); + + mAuxVec.Init(mAuxMat.Cols()); + mAuxVec.AddColSum(1.0,mAuxMat,0.0); + mAuxVec.CopyTo(mAuxVecHost); + + mError += mAuxVecHost.Sum(); + + //count the frames + mFrames += rNetError.Rows(); + } + + void + CuCrossEntropy:: + Evaluate(const CuMatrix<BaseFloat>& rNetOutput, const CuMatrix<BaseFloat>& rDesired, CuMatrix<BaseFloat>& rNetError) + { + if(rDesired.Cols() != rNetOutput.Cols()) { + std::ostringstream os; + os << "Non-matching dimensions of network output with training targets!!!" + << " Netoutput:" << rNetOutput.Cols() + << " Targets:" << rDesired.Cols(); + Error(os.str()); + } + + //get the global error + //dXent/dSoftmax_in = y-d + rNetError.CopyFrom(rNetOutput); + rNetError.AddScaled(-1.0,rDesired,1.0); + + //check classification + mClassifyVec.Init(rNetOutput.Rows()); + CuMath<BaseFloat>::CheckClass(rNetOutput,rDesired,mClassifyVec); + mClassifyVec.CopyTo(mClassifyVecHost); + mCorrect += mClassifyVecHost.Sum(); + + //calculate Xent + mAuxMat.CopyFrom(rNetOutput); + mAuxMat.LogElem(); + mAuxMat.MulElem(rDesired); + + mAuxVec.Init(mAuxMat.Cols()); + mAuxVec.AddColSum(-1.0,mAuxMat,0.0); + mAuxVec.CopyTo(mAuxVecHost); + + mError += mAuxVecHost.Sum(); + + //count the frames + mFrames += rNetError.Rows(); + } + + +} // namespace TNet diff --git a/src/CuTNetLib/cuObjectiveFunction.h b/src/CuTNetLib/cuObjectiveFunction.h new file mode 100644 index 0000000..4dd0c32 --- /dev/null +++ b/src/CuTNetLib/cuObjectiveFunction.h @@ -0,0 +1,185 @@ +#ifndef _CUOBJ_FUN_I_ +#define _CUOBJ_FUN_I_ + +#include <cassert> +#include <limits> +#include <cmath> +#include <sstream> + +#include "Vector.h" +#include "cuvector.h" +#include "cumatrix.h" + +/** + * \file cuObjectiveFunction.h + * \brief Objective Functions used to compare the model and data + */ + +/** + * \defgroup CuModelObj CuNN Objective Functions + * \ingroup CuNNComp + */ + +namespace TNet +{ + + + /** + * \brief General interface for objective functions + */ + class CuObjectiveFunction + { + public: + /// Enum with objective function types + typedef enum { + OBJ_FUN_I = 0x0300, + MEAN_SQUARE_ERROR, + CROSS_ENTROPY, + } ObjFunType; + + /// Factory for creating objective function instances + static CuObjectiveFunction* Factory(ObjFunType type); + + ////////////////////////////////////////////////////////////// + // Interface specification + public: + CuObjectiveFunction() + { } + + virtual ~CuObjectiveFunction() + { } + + virtual ObjFunType GetTypeId() = 0; + virtual const char* GetTypeLabel() = 0; + + /// evaluates the data, calculate global error + /// \param[in] rNetOutput CuNN output as generated by model + /// \param[in] rDesired Desired output specified by data + /// \param[out] rNetError Derivative of the Energy Function + virtual void Evaluate(const CuMatrix<BaseFloat>& rNetOutput, const CuMatrix<BaseFloat>& rDesired, CuMatrix<BaseFloat>& rNetError) = 0; + + ///get the average per frame error + virtual double GetError() = 0; + ///the number of processed frames + virtual size_t GetFrames() = 0; + ///report the error to std::cout + virtual std::string Report() = 0; + }; + + + + + /** + * \brief Means square error, useful for autoencoders, RBMs et al. + * + * \ingroup CuModelObj + * Calculate: \f[ ||\vec{ModelOutput}-\vec{Label}||^2 \f] + */ + class CuMeanSquareError : public CuObjectiveFunction + { + public: + CuMeanSquareError() + : mError(0), mFrames(0) + { } + + virtual ~CuMeanSquareError() + { } + + ObjFunType GetTypeId() + { return CuObjectiveFunction::MEAN_SQUARE_ERROR; } + + const char* GetTypeLabel() + { return "<mean_square_error>"; } + + void Evaluate(const CuMatrix<BaseFloat>& rNetOutput, const CuMatrix<BaseFloat>& rDesired, CuMatrix<BaseFloat>& rNetError); + + double GetError() + { return mError; } + + size_t GetFrames() + { return mFrames; } + + std::string Report() + { + std::ostringstream ss; + ss << "Mse:" << mError << " frames:" << mFrames + << " err/frm:" << mError/mFrames << "\n"; + return ss.str(); + } + + private: + double mError; + size_t mFrames; + + CuMatrix<BaseFloat> mAuxMat; + CuVector<BaseFloat> mAuxVec; + Vector<BaseFloat> mAuxVecHost; + + }; + + + /** + * \brief Cross entropy, it assumes desired vectors as output values + * + * \ingroup CuModelObj + * Calculate: \f[ -\ln(\vec{ModelOutput}) \cdot \vec{Label} \f] + */ + class CuCrossEntropy : public CuObjectiveFunction + { + public: + CuCrossEntropy() + : mError(0), mFrames(0), mCorrect(0) + { } + + ~CuCrossEntropy() + { } + + ObjFunType GetTypeId() + { return CuObjectiveFunction::CROSS_ENTROPY; } + + const char* GetTypeLabel() + { return "<cross_entropy>"; } + + void Evaluate(const CuMatrix<BaseFloat>& rNetOutput, const CuMatrix<BaseFloat>& rDesired, CuMatrix<BaseFloat>& rNetError); + + double GetError() + { return mError; } + + size_t GetFrames() + { return mFrames; } + + std::string Report() + { + std::ostringstream ss; + //for compatibility with SNet + //ss << " correct: >> " << 100.0*mCorrect/mFrames << "% <<\n"; + + //current new format... + ss << "Xent:" << mError << " frames:" << mFrames + << " err/frm:" << mError/mFrames + << " correct[" << 100.0*mCorrect/mFrames << "%]" + << "\n"; + return ss.str(); + } + + private: + double mError; + size_t mFrames; + size_t mCorrect; + + CuMatrix<BaseFloat> mAuxMat; + CuVector<BaseFloat> mAuxVec; + Vector<BaseFloat> mAuxVecHost; + + CuVector<int> mClassifyVec; + Vector<int> mClassifyVecHost; + }; + + + + + +} //namespace TNet + + +#endif diff --git a/src/CuTNetLib/cuRbm.cc b/src/CuTNetLib/cuRbm.cc new file mode 100644 index 0000000..3d0699d --- /dev/null +++ b/src/CuTNetLib/cuRbm.cc @@ -0,0 +1,244 @@ + +#include <string> +#include <sstream> + +#include "cuRbm.h" + +#include "cumath.h" + + +namespace TNet +{ + + void + CuRbm:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + Y.SetConst(0.0); + Y.AddScaledRow(1.0,mHidBias,0.0); + Y.Gemm('N','N', 1.0, X, mVisHid, 1.0); + if(mHidType == BERNOULLI) { + CuMath<BaseFloat>::Sigmoid(Y,Y); + } + } + + + void + CuRbm:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + if(mHidType == BERNOULLI) { + mBackpropErrBuf.Init(X.Rows(),X.Cols()); + CuMath<BaseFloat>::DiffSigmoid(mBackpropErrBuf,X,GetOutput()); + } else { + mBackpropErrBuf.CopyFrom(X); + } + + Y.SetConst(0.0); + Y.Gemm('N', 'T', 1.0, mBackpropErrBuf, mVisHid, 0.0); + } + + + void + CuRbm:: + Update() + { + //THIS IS DONE TWICE BECAUSE OF THE BACKPROP STOPPER!!! + if(mHidType == BERNOULLI) { + mBackpropErrBuf.Init(GetErrorInput().Rows(),GetErrorInput().Cols()); + CuMath<BaseFloat>::DiffSigmoid(mBackpropErrBuf,GetErrorInput(),GetOutput()); + } else { + mBackpropErrBuf.CopyFrom(GetErrorInput()); + } + +/* + std::cout << " " << GetInput().Rows() + << " " << GetInput().Cols() + << " " << mBackpropErrBuf.Rows() + << " " << mBackpropErrBuf.Cols() + << " " << mVisHidCorrection.Rows() + << " " << mVisHidCorrection.Cols() + ; +*/ + +#if 0 + //former implementation + BaseFloat N = static_cast<BaseFloat>(GetInput().Rows()); + + mVisHidCorrection.Gemm('T','N',-mLearningRate/N,GetInput(),mBackpropErrBuf,mMomentum); + mHidBiasCorrection.AddColSum(-mLearningRate/N,mBackpropErrBuf,mMomentum); + + //regularization weight decay + mVisHidCorrection.AddScaled(-mLearningRate*mWeightcost,mVisHid,1.0); + + mVisHid.AddScaled(1.0,mVisHidCorrection,1.0); + mHidBias.AddScaled(1.0,mHidBiasCorrection,1.0); +#endif + +#if 1 + //new implementation + BaseFloat N = 1; + if(mGradDivFrm) { + N = static_cast<BaseFloat>(GetInput().Rows()); + } + BaseFloat mmt_gain = static_cast<BaseFloat>(1.0/(1.0-mMomentum)); + N *= mmt_gain; + + mVisHidCorrection.Gemm('T','N',1.0,GetInput(),mBackpropErrBuf,mMomentum); + mHidBiasCorrection.AddColSum(1.0,mBackpropErrBuf,mMomentum); + + mVisHid.AddScaled(-mLearningRate/N,mVisHidCorrection,1.0); + mHidBias.AddScaled(-mLearningRate/N,mHidBiasCorrection,1.0); + + //regularization weight decay (from actual weights only) + mVisHid.AddScaled(-mLearningRate*mWeightcost,mVisHid,1.0); +#endif + + } + + + + void + CuRbm:: + Propagate(const CuMatrix<BaseFloat>& visProbs, CuMatrix<BaseFloat>& hidProbs) + { + if(visProbs.Cols() != GetNInputs()) { + std::ostringstream os; + os << " Nonmatching input dim, needs:" << GetNInputs() + << " got:" << visProbs.Cols() << "\n"; + Error(os.str()); + } + + hidProbs.Init(visProbs.Rows(),GetNOutputs()); + + PropagateFnc(visProbs, hidProbs); + } + + void + CuRbm:: + Reconstruct(const CuMatrix<BaseFloat>& hidState, CuMatrix<BaseFloat>& visProbs) + { + visProbs.Init(hidState.Rows(),mNInputs); + visProbs.SetConst(0.0); + visProbs.AddScaledRow(1.0,mVisBias,0.0); + visProbs.Gemm('N','T', 1.0, hidState, mVisHid, 1.0); + if(mVisType == BERNOULLI) { + CuMath<BaseFloat>::Sigmoid(visProbs,visProbs); + } + } + + + void + CuRbm:: + RbmUpdate(const CuMatrix<BaseFloat>& pos_vis, const CuMatrix<BaseFloat>& pos_hid, const CuMatrix<BaseFloat>& neg_vis, const CuMatrix<BaseFloat>& neg_hid) + { + assert(pos_vis.Rows() == pos_hid.Rows() && + pos_vis.Rows() == neg_vis.Rows() && + pos_vis.Rows() == neg_hid.Rows() && + pos_vis.Cols() == neg_vis.Cols() && + pos_hid.Cols() == neg_hid.Cols() && + pos_vis.Cols() == mNInputs && + pos_hid.Cols() == mNOutputs); + + // UPDATE vishid matrix + // + // vishidinc = momentum*vishidinc + ... + // epsilonw*( (posprods-negprods)/numcases - weightcost*vishid); + // + // vishidinc[t] = -(epsilonw/numcases)*negprods + momentum*vishidinc[t-1] + // +(epsilonw/numcases)*posprods + // -(epsilonw*weightcost)*vishid[t-1] + // + BaseFloat N = static_cast<BaseFloat>(pos_vis.Rows()); + mVisHidCorrection.Gemm('T','N',-mLearningRate/N,neg_vis,neg_hid,mMomentum); + mVisHidCorrection.Gemm('T','N',+mLearningRate/N,pos_vis,pos_hid,1.0); + mVisHidCorrection.AddScaled(-mLearningRate*mWeightcost,mVisHid,1.0); + mVisHid.AddScaled(1.0,mVisHidCorrection,1.0); + + // UPDATE visbias vector + // + // visbiasinc = momentum*visbiasinc + (epsilonvb/numcases)*(posvisact-negvisact); + // + mVisBiasCorrection.AddColSum(-mLearningRate/N,neg_vis,mMomentum); + mVisBiasCorrection.AddColSum(+mLearningRate/N,pos_vis,1.0); + mVisBias.AddScaled(1.0,mVisBiasCorrection,1.0); + + // UPDATE hidbias vector + // + // hidbiasinc = momentum*hidbiasinc + (epsilonhb/numcases)*(poshidact-neghidact); + // + mHidBiasCorrection.AddColSum(-mLearningRate/N,neg_hid,mMomentum); + mHidBiasCorrection.AddColSum(+mLearningRate/N,pos_hid,1.0); + mHidBias.AddScaled(1.0/*0.0*/,mHidBiasCorrection,1.0); + + } + + + void + CuRbm:: + ReadFromStream(std::istream& rIn) + { + //type of the units + std::string str; + + rIn >> std::ws >> str; + if(0 == str.compare("bern")) { + mVisType = BERNOULLI; + } else if(0 == str.compare("gauss")) { + mVisType = GAUSSIAN; + } else Error(std::string("Invalid unit type: ")+str); + + rIn >> std::ws >> str; + if(0 == str.compare("bern")) { + mHidType = BERNOULLI; + } else if(0 == str.compare("gauss")) { + mHidType = GAUSSIAN; + } else Error(std::string("Invalid unit type: ")+str); + + + //matrix is stored transposed as SNet does + BfMatrix transpose; + rIn >> transpose; + mVisHid.CopyFrom(BfMatrix(transpose, TRANS)); + //biases stored normally + BfVector bias; + rIn >> bias; + mVisBias.CopyFrom(bias); + rIn >> bias; + mHidBias.CopyFrom(bias); + } + + + void + CuRbm:: + WriteToStream(std::ostream& rOut) + { + //store unit type info + if(mVisType == BERNOULLI) { + rOut << " bern "; + } else { + rOut << " gauss "; + } + if(mHidType == BERNOULLI) { + rOut << " bern\n"; + } else { + rOut << " gauss\n"; + } + + //matrix is stored transposed as SNet does + BfMatrix tmp; + mVisHid.CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + //biases stored normally + BfVector vec; + mVisBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + mHidBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + } + + +} //namespace diff --git a/src/CuTNetLib/cuRbm.h b/src/CuTNetLib/cuRbm.h new file mode 100644 index 0000000..c1e984b --- /dev/null +++ b/src/CuTNetLib/cuRbm.h @@ -0,0 +1,146 @@ +#ifndef _CU_RBM_H_ +#define _CU_RBM_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + + class CuRbmBase : public CuUpdatableComponent + { + public: + typedef enum { + BERNOULLI, + GAUSSIAN + } RbmUnitType; + + CuRbmBase(size_t nInputs, size_t nOutputs, CuComponent *pPred) : + CuUpdatableComponent(nInputs, nOutputs, pPred) + { } + + + virtual void Propagate( + const CuMatrix<BaseFloat>& visProbs, + CuMatrix<BaseFloat>& hidProbs + ) = 0; + virtual void Reconstruct( + const CuMatrix<BaseFloat>& hidState, + CuMatrix<BaseFloat>& visProbs + ) = 0; + virtual void RbmUpdate( + const CuMatrix<BaseFloat>& pos_vis, + const CuMatrix<BaseFloat>& pos_hid, + const CuMatrix<BaseFloat>& neg_vis, + const CuMatrix<BaseFloat>& neg_hid + ) = 0; + + virtual RbmUnitType VisType() = 0; + virtual RbmUnitType HidType() = 0; + }; + + + class CuRbm : public CuRbmBase + { + public: + + CuRbm(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuRbm(); + + ComponentType GetType() const; + const char* GetName() const; + + //CuUpdatableComponent API + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + void Update(); + + //RBM training API + void Propagate(const CuMatrix<BaseFloat>& visProbs, CuMatrix<BaseFloat>& hidProbs); + void Reconstruct(const CuMatrix<BaseFloat>& hidState, CuMatrix<BaseFloat>& visProbs); + void RbmUpdate(const CuMatrix<BaseFloat>& pos_vis, const CuMatrix<BaseFloat>& pos_hid, const CuMatrix<BaseFloat>& neg_vis, const CuMatrix<BaseFloat>& neg_hid); + + RbmUnitType VisType() + { return mVisType; } + + RbmUnitType HidType() + { return mHidType; } + + //static void BinarizeProbs(const CuMatrix<BaseFloat>& probs, CuMatrix<BaseFloat>& states); + + //I/O + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + CuMatrix<BaseFloat> mVisHid; ///< Matrix with neuron weights + CuVector<BaseFloat> mVisBias; ///< Vector with biases + CuVector<BaseFloat> mHidBias; ///< Vector with biases + + CuMatrix<BaseFloat> mVisHidCorrection; ///< Matrix for linearity updates + CuVector<BaseFloat> mVisBiasCorrection; ///< Vector for bias updates + CuVector<BaseFloat> mHidBiasCorrection; ///< Vector for bias updates + + CuMatrix<BaseFloat> mBackpropErrBuf; + + RbmUnitType mVisType; + RbmUnitType mHidType; + + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuRbm:: + inline + CuRbm:: + CuRbm(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuRbmBase(nInputs, nOutputs, pPred), + mVisHid(nInputs,nOutputs), + mVisBias(nInputs), mHidBias(nOutputs), + mVisHidCorrection(nInputs,nOutputs), + mVisBiasCorrection(nInputs), mHidBiasCorrection(nOutputs), + mBackpropErrBuf(), + mVisType(BERNOULLI), + mHidType(BERNOULLI) + { + mVisHidCorrection.SetConst(0.0); + mVisBiasCorrection.SetConst(0.0); + mHidBiasCorrection.SetConst(0.0); + } + + + inline + CuRbm:: + ~CuRbm() + { } + + inline CuComponent::ComponentType + CuRbm:: + GetType() const + { + return CuComponent::RBM; + } + + inline const char* + CuRbm:: + GetName() const + { + return "<rbm>"; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/cuRbmSparse.cc b/src/CuTNetLib/cuRbmSparse.cc new file mode 100644 index 0000000..e0b7352 --- /dev/null +++ b/src/CuTNetLib/cuRbmSparse.cc @@ -0,0 +1,269 @@ + +#include <string> +#include <sstream> + +#include "cuRbmSparse.h" + +#include "cumath.h" + + +namespace TNet +{ + + void + CuRbmSparse:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + Y.SetConst(0.0); + Y.AddScaledRow(1.0,mHidBias,0.0); + Y.Gemm('N','N', 1.0, X, mVisHid, 1.0); + if(mHidType == BERNOULLI) { + CuMath<BaseFloat>::Sigmoid(Y,Y); + } + } + + + void + CuRbmSparse:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + if(mHidType == BERNOULLI) { + mBackpropErrBuf.Init(X.Rows(),X.Cols()); + CuMath<BaseFloat>::DiffSigmoid(mBackpropErrBuf,X,GetOutput()); + } else { + mBackpropErrBuf.CopyFrom(X); + } + + Y.SetConst(0.0); + Y.Gemm('N', 'T', 1.0, mBackpropErrBuf, mVisHid, 0.0); + } + + + void + CuRbmSparse:: + Update() + { + //THIS IS DONE TWICE BECAUSE OF THE BACKPROP STOPPER!!! + if(mHidType == BERNOULLI) { + mBackpropErrBuf.Init(GetErrorInput().Rows(),GetErrorInput().Cols()); + CuMath<BaseFloat>::DiffSigmoid(mBackpropErrBuf,GetErrorInput(),GetOutput()); + } else { + mBackpropErrBuf.CopyFrom(GetErrorInput()); + } + +/* + std::cout << " " << GetInput().Rows() + << " " << GetInput().Cols() + << " " << mBackpropErrBuf.Rows() + << " " << mBackpropErrBuf.Cols() + << " " << mVisHidCorrection.Rows() + << " " << mVisHidCorrection.Cols() + ; +*/ + +#if 0 + //former implementation + BaseFloat N = static_cast<BaseFloat>(GetInput().Rows()); + + mVisHidCorrection.Gemm('T','N',-mLearningRate/N,GetInput(),mBackpropErrBuf,mMomentum); + mHidBiasCorrection.AddColSum(-mLearningRate/N,mBackpropErrBuf,mMomentum); + + //regularization weight decay + mVisHidCorrection.AddScaled(-mLearningRate*mWeightcost,mVisHid,1.0); + + mVisHid.AddScaled(1.0,mVisHidCorrection,1.0); + mHidBias.AddScaled(1.0,mHidBiasCorrection,1.0); +#endif + +#if 1 + //new implementation + BaseFloat N = 1; + if(mGradDivFrm) { + N = static_cast<BaseFloat>(GetInput().Rows()); + } + BaseFloat mmt_gain = static_cast<BaseFloat>(1.0/(1.0-mMomentum)); + N *= mmt_gain; + + mVisHidCorrection.Gemm('T','N',1.0,GetInput(),mBackpropErrBuf,mMomentum); + mHidBiasCorrection.AddColSum(1.0,mBackpropErrBuf,mMomentum); + + mVisHid.AddScaled(-mLearningRate/N,mVisHidCorrection,1.0); + mHidBias.AddScaled(-mLearningRate/N,mHidBiasCorrection,1.0); + + //regularization weight decay (from actual weights only) + mVisHid.AddScaled(-mLearningRate*mWeightcost,mVisHid,1.0); +#endif + + } + + + + void + CuRbmSparse:: + Propagate(const CuMatrix<BaseFloat>& visProbs, CuMatrix<BaseFloat>& hidProbs) + { + if(visProbs.Cols() != GetNInputs()) { + std::ostringstream os; + os << " Nonmatching input dim, needs:" << GetNInputs() + << " got:" << visProbs.Cols() << "\n"; + Error(os.str()); + } + + hidProbs.Init(visProbs.Rows(),GetNOutputs()); + + PropagateFnc(visProbs, hidProbs); + } + + void + CuRbmSparse:: + Reconstruct(const CuMatrix<BaseFloat>& hidState, CuMatrix<BaseFloat>& visProbs) + { + visProbs.Init(hidState.Rows(),mNInputs); + visProbs.SetConst(0.0); + visProbs.AddScaledRow(1.0,mVisBias,0.0); + visProbs.Gemm('N','T', 1.0, hidState, mVisHid, 1.0); + if(mVisType == BERNOULLI) { + CuMath<BaseFloat>::Sigmoid(visProbs,visProbs); + } + } + + + void + CuRbmSparse:: + RbmUpdate(const CuMatrix<BaseFloat>& pos_vis, const CuMatrix<BaseFloat>& pos_hid, const CuMatrix<BaseFloat>& neg_vis, const CuMatrix<BaseFloat>& neg_hid) + { + assert(pos_vis.Rows() == pos_hid.Rows() && + pos_vis.Rows() == neg_vis.Rows() && + pos_vis.Rows() == neg_hid.Rows() && + pos_vis.Cols() == neg_vis.Cols() && + pos_hid.Cols() == neg_hid.Cols() && + pos_vis.Cols() == mNInputs && + pos_hid.Cols() == mNOutputs); + + //:SPARSITY: + if(mHidType==BERNOULLI) { + //get expected node activity from current batch + mSparsityQCurrent.AddColSum(1.0/pos_hid.Rows(),pos_hid,0.0); + //get smoothed expected node activity + mSparsityQ.AddScaled(1.0-mLambda,mSparsityQCurrent,mLambda); + //subtract the prior: (q-p) + mSparsityQCurrent.SetConst(-mSparsityPrior); + mSparsityQCurrent.AddScaled(1.0,mSparsityQ,1.0); + //get mean pos_vis + mVisMean.AddColSum(1.0/pos_vis.Rows(),pos_vis,0.0); + } + + // UPDATE vishid matrix + // + // vishidinc = momentum*vishidinc + ... + // epsilonw*( (posprods-negprods)/numcases - weightcost*vishid) + // -sparsitycost*mean_posvis'*(q-p); + // + // vishidinc[t] = -(epsilonw/numcases)*negprods + momentum*vishidinc[t-1] + // +(epsilonw/numcases)*posprods + // -(epsilonw*weightcost)*vishid[t-1] + // + BaseFloat N = static_cast<BaseFloat>(pos_vis.Rows()); + mVisHidCorrection.Gemm('T','N',-mLearningRate/N,neg_vis,neg_hid,mMomentum); + mVisHidCorrection.Gemm('T','N',+mLearningRate/N,pos_vis,pos_hid,1.0); + mVisHidCorrection.AddScaled(-mLearningRate*mWeightcost,mVisHid,1.0);//L2 + if(mHidType==BERNOULLI) { + mVisHidCorrection.BlasGer(-mSparsityCost,mVisMean,mSparsityQCurrent);//sparsity + } + mVisHid.AddScaled(1.0,mVisHidCorrection,1.0); + + // UPDATE visbias vector + // + // visbiasinc = momentum*visbiasinc + (epsilonvb/numcases)*(posvisact-negvisact); + // + mVisBiasCorrection.AddColSum(-mLearningRate/N,neg_vis,mMomentum); + mVisBiasCorrection.AddColSum(+mLearningRate/N,pos_vis,1.0); + mVisBias.AddScaled(1.0,mVisBiasCorrection,1.0); + + // UPDATE hidbias vector + // + // hidbiasinc = momentum*hidbiasinc + (epsilonhb/numcases)*(poshidact-neghidact); + // + mHidBiasCorrection.AddColSum(-mLearningRate/N,neg_hid,mMomentum); + mHidBiasCorrection.AddColSum(+mLearningRate/N,pos_hid,1.0); + if(mHidType==BERNOULLI) { + mHidBiasCorrection.AddScaled(-mSparsityCost,mSparsityQCurrent,1.0);//sparsity + } + mHidBias.AddScaled(1.0/*0.0*/,mHidBiasCorrection,1.0); + + } + + + void + CuRbmSparse:: + ReadFromStream(std::istream& rIn) + { + //type of the units + std::string str; + + rIn >> std::ws >> str; + if(0 == str.compare("bern")) { + mVisType = BERNOULLI; + } else if(0 == str.compare("gauss")) { + mVisType = GAUSSIAN; + } else Error(std::string("Invalid unit type: ")+str); + + rIn >> std::ws >> str; + if(0 == str.compare("bern")) { + mHidType = BERNOULLI; + } else if(0 == str.compare("gauss")) { + mHidType = GAUSSIAN; + } else Error(std::string("Invalid unit type: ")+str); + + + //matrix is stored transposed as SNet does + BfMatrix transpose; + rIn >> transpose; + mVisHid.CopyFrom(BfMatrix(transpose, TRANS)); + //biases stored normally + BfVector bias; + rIn >> bias; + mVisBias.CopyFrom(bias); + rIn >> bias; + mHidBias.CopyFrom(bias); + + rIn >> std::ws >> mSparsityCost; + std::cout << "RBM::mSparsityCost=" << mSparsityCost; + } + + + void + CuRbmSparse:: + WriteToStream(std::ostream& rOut) + { + //store unit type info + if(mVisType == BERNOULLI) { + rOut << " bern "; + } else { + rOut << " gauss "; + } + if(mHidType == BERNOULLI) { + rOut << " bern\n"; + } else { + rOut << " gauss\n"; + } + + //matrix is stored transposed as SNet does + BfMatrix tmp; + mVisHid.CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + //biases stored normally + BfVector vec; + mVisBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + mHidBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + //store the sparsity cost + rOut << mSparsityCost << std::endl; + } + + +} //namespace diff --git a/src/CuTNetLib/cuRbmSparse.h b/src/CuTNetLib/cuRbmSparse.h new file mode 100644 index 0000000..9d7e304 --- /dev/null +++ b/src/CuTNetLib/cuRbmSparse.h @@ -0,0 +1,134 @@ +#ifndef _CU_RBM_SPARSE_H_ +#define _CU_RBM_SPARSE_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" +#include "cuRbm.h" + + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + + class CuRbmSparse : public CuRbmBase + { + public: + + CuRbmSparse(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuRbmSparse(); + + ComponentType GetType() const; + const char* GetName() const; + + //CuUpdatableComponent API + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + void Update(); + + //RBM training API + void Propagate(const CuMatrix<BaseFloat>& visProbs, CuMatrix<BaseFloat>& hidProbs); + void Reconstruct(const CuMatrix<BaseFloat>& hidState, CuMatrix<BaseFloat>& visProbs); + void RbmUpdate(const CuMatrix<BaseFloat>& pos_vis, const CuMatrix<BaseFloat>& pos_hid, const CuMatrix<BaseFloat>& neg_vis, const CuMatrix<BaseFloat>& neg_hid); + + RbmUnitType VisType() + { return mVisType; } + + RbmUnitType HidType() + { return mHidType; } + + //static void BinarizeProbs(const CuMatrix<BaseFloat>& probs, CuMatrix<BaseFloat>& states); + + //I/O + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + CuMatrix<BaseFloat> mVisHid; ///< Matrix with neuron weights + CuVector<BaseFloat> mVisBias; ///< Vector with biases + CuVector<BaseFloat> mHidBias; ///< Vector with biases + + CuMatrix<BaseFloat> mVisHidCorrection; ///< Matrix for linearity updates + CuVector<BaseFloat> mVisBiasCorrection; ///< Vector for bias updates + CuVector<BaseFloat> mHidBiasCorrection; ///< Vector for bias updates + + CuMatrix<BaseFloat> mBackpropErrBuf; + + RbmUnitType mVisType; + RbmUnitType mHidType; + + ////// sparsity + BaseFloat mSparsityPrior; ///< sparsity target (unit activity prior) + BaseFloat mLambda; ///< exponential decay factor for q (observed probability of unit to be active) + BaseFloat mSparsityCost; ///< sparsity cost coef. + + CuVector<BaseFloat> mSparsityQ; + CuVector<BaseFloat> mSparsityQCurrent; + CuVector<BaseFloat> mVisMean; ///< buffer for mean visible + + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuRbmSparse:: + inline + CuRbmSparse:: + CuRbmSparse(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuRbmBase(nInputs, nOutputs, pPred), + mVisHid(nInputs,nOutputs), + mVisBias(nInputs), mHidBias(nOutputs), + mVisHidCorrection(nInputs,nOutputs), + mVisBiasCorrection(nInputs), mHidBiasCorrection(nOutputs), + mBackpropErrBuf(), + mVisType(BERNOULLI), + mHidType(BERNOULLI), + + mSparsityPrior(0.0001), + mLambda(0.95), + mSparsityCost(1e-7), + mSparsityQ(nOutputs), + mSparsityQCurrent(nOutputs), + mVisMean(nInputs) + { + mVisHidCorrection.SetConst(0.0); + mVisBiasCorrection.SetConst(0.0); + mHidBiasCorrection.SetConst(0.0); + + mSparsityQ.SetConst(mSparsityPrior); + mSparsityQCurrent.SetConst(0.0); + mVisMean.SetConst(0.0); + } + + + inline + CuRbmSparse:: + ~CuRbmSparse() + { } + + inline CuComponent::ComponentType + CuRbmSparse:: + GetType() const + { + return CuComponent::RBM_SPARSE; + } + + inline const char* + CuRbmSparse:: + GetName() const + { + return "<rbmsparse>"; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/cuRecurrent.cc b/src/CuTNetLib/cuRecurrent.cc new file mode 100644 index 0000000..428df2c --- /dev/null +++ b/src/CuTNetLib/cuRecurrent.cc @@ -0,0 +1,191 @@ + +#include <string> +#include <sstream> + +#include "cuRecurrent.h" + +#include "cumath.h" +#include "cuda_runtime.h" + + +namespace TNet +{ + + void + CuRecurrent:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + assert(X.Rows() == 1); + assert(Y.Rows() == 1); + if(mInputHistory.Rows() == 0) { + Error("Bptt order was not set"); + } + + //pushback the history + CuMatrix<BaseFloat> tmp(mInputHistory.Rows()-1,mInputHistory.Cols()); + tmp.CopyRows(tmp.Rows(),0,mInputHistory,0); + mInputHistory.CopyRows(tmp.Rows(),0,tmp,1); + + //compose the input vector to 0th row, use input X and previous Y + cudaMemcpy(mInputHistory.pCUData(), X.pCUData(), + sizeof(BaseFloat)*X.Cols(), cudaMemcpyDeviceToDevice); + cudaMemcpy(mInputHistory.pCUData()+X.Cols(), Y.pCUData(), + sizeof(BaseFloat)*Y.Cols(), cudaMemcpyDeviceToDevice); + + //extract first row + //CuMatrix<BaseFloat> first_row(1,mInputHistory.Cols()); + //first_row.CopyRows(1,0,mInputHistory,0); + + //calculate the output + Y.AddScaledRow(1.0,mBias,0.0); + //take 0th vector of history, propagate + CuMath<BaseFloat>::OffsetGemv('T',1.0,mLinearity,mInputHistory.pCUData(),mInputHistory.Cols(),1.0,Y.pCUData(),Y.Cols(),0); + //Y.Gemm('N','N', 1.0, first_row, mLinearity, 1.0); + CuMath<BaseFloat>::Sigmoid(Y,Y); + + /* + std::cout << "-------------------------------------" << std::endl; + X.Print(); + Y.Print(); + mInputHistory.Print(); + */ + + } + + + void + CuRecurrent:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + assert(Y.Rows() == 1); + assert(X.Rows() == 1); + + //apply diff sigmoid + CuMatrix<BaseFloat> diff_sigm(1,X.Cols()); + CuMath<BaseFloat>::DiffSigmoid(diff_sigm,X,GetOutput()); + + //:TODO: inefficent to calculate all the input errors!!! + // we need only part of them! + // + //backward-multiply by weights + /* + CuMatrix<BaseFloat> err_prev(1,mLinearity.Rows()); + err_prev.Gemm('N', 'T', 1.0, diff_sigm, mLinearity, 0.0); + + //copy out the interval + cudaMemcpy(Y.pCUData(),err_prev.pCUData(), + sizeof(BaseFloat)*Y.Cols(),cudaMemcpyDeviceToDevice); + */ + + //backward-multiply by weights + CuMath<BaseFloat>::OffsetGemv('N',1.0,mLinearity,diff_sigm.pCUData(),diff_sigm.Cols(),1.0,Y.pCUData(),Y.Cols(),0); + + } + + + void + CuRecurrent:: + Update() + { + // + //correction from PRESENT input x error pair + // + //apply diff sigmoid + CuMatrix<BaseFloat> diff_sigm(1,GetOutput().Cols()); + CuMath<BaseFloat>::DiffSigmoid(diff_sigm,GetErrorInput(),GetOutput()); + + //get 0th row of history (present time) + CuMatrix<BaseFloat> history_row(1,mInputHistory.Cols()); + history_row.CopyRows(1,0,mInputHistory,0); + + //calculate update + //mLinearityCorrection.Gemm('T','N',-mLearningRate,history_row,diff_sigm,mMomentum); + mLinearityCorrection.SetConst(0.0); //:TODO: should be scale/momentum + CuMath<BaseFloat>::BlasGer(-mLearningRate,history_row.pCUData(),history_row.Cols(),diff_sigm.pCUData(),diff_sigm.Cols(),mLinearityCorrection); + + mBiasCorrection.AddColSum(-mLearningRate,diff_sigm,mMomentum); + + // + //BPTT (backprop through time) + // + CuMatrix<BaseFloat> err_prev(1,mLinearity.Rows()); + CuMatrix<BaseFloat> err_prev_part(1,diff_sigm.Cols()); + CuMatrix<BaseFloat> history_output(1,GetOutput().Cols()); + for(int i=1; i<=mBpttOrder; i++) { + //:TODO: inefficent to calculate all the input errors!!! + // we need only part of them! + // + /* + //get previous error + err_prev.Gemm('N','T',1.0,diff_sigm,mLinearity,0.0); + //select interval + cudaMemcpy(err_prev_part.pCUData(),err_prev.pCUData()+GetNInputs(), + sizeof(BaseFloat)*err_prev_part.Cols(),cudaMemcpyDeviceToDevice); + */ + + //backward-multiply by weights + CuMath<BaseFloat>::OffsetGemv('N',1.0,mLinearity,diff_sigm.pCUData(),diff_sigm.Cols(),0.0,err_prev_part.pCUData(),err_prev_part.Cols(),GetInput().Cols()); + + //apply diff sigmoid with activations of HISTORY frame!!! + cudaMemcpy(history_output.pCUData(), mInputHistory.pCURowData(i-1)+GetInput().Cols(), + sizeof(BaseFloat)*history_output.Cols(), cudaMemcpyDeviceToDevice); + CuMath<BaseFloat>::DiffSigmoid(diff_sigm,err_prev_part,history_output); + + //get history row + history_row.CopyRows(1,i,mInputHistory,0); + + //accu the update + //mLinearityCorrection.Gemm('T','N',-mLearningRate,history_row,diff_sigm,1.0); + CuMath<BaseFloat>::BlasGer(-mLearningRate,history_row.pCUData(),history_row.Cols(),diff_sigm.pCUData(),diff_sigm.Cols(),mLinearityCorrection); + mBiasCorrection.AddColSum(-mLearningRate,diff_sigm,1.0); + } + + // + //update the weights + // + //regularization weight decay + mLinearityCorrection.AddScaled(-mLearningRate*mWeightcost,mLinearity,1.0); + + //perform update + mLinearity.AddScaled(1.0,mLinearityCorrection,1.0); + mBias.AddScaled(1.0,mBiasCorrection,1.0); + + } + + + + + void + CuRecurrent:: + ReadFromStream(std::istream& rIn) + { + //matrix is stored transposed as SNet does + BfMatrix transpose; + rIn >> transpose; + mLinearity.CopyFrom(BfMatrix(transpose, TRANS)); + //biases stored normally + BfVector bias; + rIn >> bias; + mBias.CopyFrom(bias); + } + + + void + CuRecurrent:: + WriteToStream(std::ostream& rOut) + { + //matrix is stored transposed as SNet does + BfMatrix tmp; + mLinearity.CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + //biases stored normally + BfVector vec; + mBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + } + + +} //namespace + diff --git a/src/CuTNetLib/cuRecurrent.h b/src/CuTNetLib/cuRecurrent.h new file mode 100644 index 0000000..e487b27 --- /dev/null +++ b/src/CuTNetLib/cuRecurrent.h @@ -0,0 +1,101 @@ +#ifndef _CU_RECURRENT_H_ +#define _CU_RECURRENT_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + + class CuRecurrent : public CuUpdatableComponent + { + public: + + CuRecurrent(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuRecurrent(); + + ComponentType GetType() const; + const char* GetName() const; + + //CuUpdatableComponent API + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + void Update(); + + //Recurrent training API + void BpttOrder(int ord) { + mBpttOrder = ord; + mInputHistory.Init(ord+1,GetNInputs()+GetNOutputs()); + } + void ClearHistory() { + mInputHistory.SetConst(0.0); + if(mOutput.MSize() > 0) { + mOutput.SetConst(0.0); + } + } + + //I/O + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + CuMatrix<BaseFloat> mLinearity; + CuVector<BaseFloat> mBias; + + CuMatrix<BaseFloat> mLinearityCorrection; + CuVector<BaseFloat> mBiasCorrection; + + CuMatrix<BaseFloat> mInputHistory; + + int mBpttOrder; + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuRecurrent:: + inline + CuRecurrent:: + CuRecurrent(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred), + mLinearity(nInputs+nOutputs,nOutputs), + mBias(nOutputs), + mLinearityCorrection(nInputs+nOutputs,nOutputs), + mBiasCorrection(nOutputs) + { } + + + inline + CuRecurrent:: + ~CuRecurrent() + { } + + inline CuComponent::ComponentType + CuRecurrent:: + GetType() const + { + return CuComponent::RECURRENT; + } + + inline const char* + CuRecurrent:: + GetName() const + { + return "<recurrent>"; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/cuSharedLinearity.cc b/src/CuTNetLib/cuSharedLinearity.cc new file mode 100644 index 0000000..8d5ec09 --- /dev/null +++ b/src/CuTNetLib/cuSharedLinearity.cc @@ -0,0 +1,179 @@ + + +#include "cuSharedLinearity.h" +#include "cumath.h" + + +namespace TNet +{ + + void + CuSharedLinearity:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + CuMath<BaseFloat>::VecExpand(mBias,mBiasExpand); /// [ 1 2 3 ] -> [ 1 2 3 1 2 3 ... ] + Y.AddScaledRow(1.0,mBiasExpand,0.0); + + //mBiasExpand.Print(); + + for(int i=0; i<mNInstances; i++) { + CuMath<BaseFloat>::OffsetGemm('N','N', 1.0, X, mLinearity, 1.0, Y, + i*mLinearity.Rows(), 0, i*mLinearity.Cols()); + } + //std::cout << CuDevice::Instantiate().GetFreeMemory(); + //GetInput().Print(); + //GetOutput().Print(); + } + + + void + CuSharedLinearity:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + for(int i=0; i<mNInstances; i++) { + CuMath<BaseFloat>::OffsetGemm('N', 'T', 1.0, X, mLinearity, 0.0, Y, + i*mLinearity.Cols(), 0, i*mLinearity.Rows()); + } + } + + + void + CuSharedLinearity:: + Update() + { +#if 0 + //former implementation + BaseFloat N = static_cast<BaseFloat>(GetInput().Rows()); + + for(int i=0; i<mNInstances; i++) { + CuMath<BaseFloat>::OffsetGemm('T','N',-mLearningRate/(N*mNInstances), + GetInput(),GetErrorInput(), + ((i==0)?mMomentum:1.0f), mLinearityCorrection, + i*mLinearity.Rows(),i*mLinearity.Cols(),0); + } + mBiasCorrectionExpand.AddColSum(1.0,GetErrorInput(),0.0); + CuMath<BaseFloat>::VecAddColSum(-mLearningRate/(N*mNInstances),mBiasCorrectionExpand,mMomentum,mBiasCorrection); + + + //regularization weight decay + mLinearityCorrection.AddScaled(-mLearningRate*mWeightcost,mLinearity,1.0); + + mLinearity.AddScaled(1.0,mLinearityCorrection,1.0); + mBias.AddScaled(1.0,mBiasCorrection,1.0); +#endif + +#if 1 + //new implementation + BaseFloat N = 1; + if(mGradDivFrm) { + N = static_cast<BaseFloat>(GetInput().Rows()); + } + BaseFloat mmt_gain = static_cast<BaseFloat>(1.0/(1.0-mMomentum)); + N *= mmt_gain; //compensate higher gradient estimates due to momentum + + //compensate augmented dyn. range of gradient caused by multiple instances + N *= static_cast<BaseFloat>(mNInstances); + + //get gradient of shared linearity + for(int i=0; i<mNInstances; i++) { + CuMath<BaseFloat>::OffsetGemm('T','N',1.0, + GetInput(),GetErrorInput(), + ((i==0)?mMomentum:1.0f), mLinearityCorrection, + i*mLinearity.Rows(),i*mLinearity.Cols(),0); + } + //get gradient of shared bias + mBiasCorrectionExpand.AddColSum(1.0,GetErrorInput(),0.0); + CuMath<BaseFloat>::VecAddColSum(1.0,mBiasCorrectionExpand,mMomentum,mBiasCorrection); + + //perform update + mLinearity.AddScaled(-mLearningRate/N,mLinearityCorrection,1.0); + mBias.AddScaled(-mLearningRate/N,mBiasCorrection,1.0); + + //regularization weight decay + mLinearity.AddScaled(-mLearningRate*mWeightcost,mLinearity,1.0); +#endif + + } + + + void + CuSharedLinearity:: + ReadFromStream(std::istream& rIn) + { + //number of instances of shared weights in layer + rIn >> std::ws >> mNInstances; + if(mNInstances < 1) { + std::ostringstream os; + os << "Bad number of instances:" << mNInstances; + Error(os.str()); + } + if(GetNInputs() % mNInstances != 0 || GetNOutputs() % mNInstances != 0) { + std::ostringstream os; + os << "Number of Inputs/Outputs must be divisible by number of instances" + << " Inputs:" << GetNInputs() + << " Outputs" << GetNOutputs() + << " Intances:" << mNInstances; + Error(os.str()); + } + + //matrix is stored transposed as SNet does + BfMatrix transpose; + rIn >> transpose; + mLinearity.CopyFrom(BfMatrix(transpose, TRANS)); + //biases stored normally + BfVector bias; + rIn >> bias; + mBias.CopyFrom(bias); + + if(transpose.Cols()*transpose.Rows() == 0) { + Error("Missing linearity matrix in network file"); + } + if(bias.Dim() == 0) { + Error("Missing bias vector in network file"); + } + + + if(mLinearity.Cols() != GetNOutputs() / mNInstances || + mLinearity.Rows() != GetNInputs() / mNInstances || + mBias.Dim() != GetNOutputs() / mNInstances + ){ + std::ostringstream os; + os << "Wrong dimensionalities of matrix/vector in network file\n" + << "Inputs:" << GetNInputs() + << "Outputs:" << GetNOutputs() + << "\n" + << "linearityCols:" << mLinearity.Cols() + << "linearityRows:" << mLinearity.Rows() + << "biasDims:" << mBias.Dim() + << "\n"; + Error(os.str()); + } + + mLinearityCorrection.Init(mLinearity.Rows(),mLinearity.Cols()); + mBiasCorrection.Init(mBias.Dim()); + + mBiasExpand.Init(mBias.Dim()*mNInstances); + mBiasCorrectionExpand.Init(mBias.Dim()*mNInstances); + } + + + void + CuSharedLinearity:: + WriteToStream(std::ostream& rOut) + { + rOut << mNInstances << std::endl; + + //matrix is stored transposed as SNet does + BfMatrix tmp; + mLinearity.CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + //biases stored normally + BfVector vec; + mBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + } + + +} //namespace diff --git a/src/CuTNetLib/cuSharedLinearity.h b/src/CuTNetLib/cuSharedLinearity.h new file mode 100644 index 0000000..76133eb --- /dev/null +++ b/src/CuTNetLib/cuSharedLinearity.h @@ -0,0 +1,94 @@ +#ifndef _CUSHARED_LINEARITY_H_ +#define _CUSHARED_LINEARITY_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + + /** + * \brief CuSharedLinearity summation function + * + * \ingroup CuNNUpdatable + * Weights and bias are shared between output, + * which means the interconnections are segmented. + * every time the outputs (blocks) are generated individually, and bias is expanded to speed up? + * \sa CuBiasedLinearity + */ + class CuSharedLinearity : public CuUpdatableComponent + { + public: + + CuSharedLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuSharedLinearity(); + + ComponentType GetType() const; + const char* GetName() const; + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + void Update(); + + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + protected: + CuMatrix<BaseFloat> mLinearity; ///< Matrix with neuron weights + CuVector<BaseFloat> mBias; ///< Vector with biases + + CuMatrix<BaseFloat> mLinearityCorrection; ///< Matrix for linearity updates + CuVector<BaseFloat> mBiasCorrection; ///< Vector for bias updates + + int mNInstances; ///< Number of times the bias and weights are shared + CuVector<BaseFloat> mBiasExpand; ///< Bias expanded by mNInstances times + CuVector<BaseFloat> mBiasCorrectionExpand;///< Bias correction for the expanded bias vector + + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuSharedLinearity:: + inline + CuSharedLinearity:: + CuSharedLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred), + mNInstances(0) + { } + + + inline + CuSharedLinearity:: + ~CuSharedLinearity() + { } + + inline CuComponent::ComponentType + CuSharedLinearity:: + GetType() const + { + return CuComponent::SHARED_LINEARITY; + } + + inline const char* + CuSharedLinearity:: + GetName() const + { + return "<sharedlinearity>"; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/cuSparseLinearity.cc b/src/CuTNetLib/cuSparseLinearity.cc new file mode 100644 index 0000000..7209630 --- /dev/null +++ b/src/CuTNetLib/cuSparseLinearity.cc @@ -0,0 +1,190 @@ + + +#include "cuSparseLinearity.h" +#include <cmath> +#include <cstdlib> + + +namespace TNet +{ + + void + CuSparseLinearity:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + Y.AddScaledRow(1.0,mBias,0.0); + Y.Gemm('N','N', 1.0, X, mLinearity, 1.0); + } + + + void + CuSparseLinearity:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + Y.Gemm('N', 'T', 1.0, X, mLinearity, 0.0); + } + + + void + CuSparseLinearity:: + Update() + { + BaseFloat N = 1; + if(mGradDivFrm) { + N = static_cast<BaseFloat>(GetInput().Rows()); + } + BaseFloat mmt_gain = static_cast<BaseFloat>(1.0/(1.0-mMomentum)); + N *= mmt_gain; + + mLinearityCorrection.Gemm('T','N',1.0,GetInput(),GetErrorInput(),mMomentum); + mBiasCorrection.AddColSum(1.0,GetErrorInput(),mMomentum); + + mLinearity.AddScaled(-mLearningRate/N,mLinearityCorrection,1.0); + mBias.AddScaled(-mLearningRate/N,mBiasCorrection,1.0); + + mLinearityCorrectionAccu.AddScaled(1.0,mLinearityCorrection,1.0); + mLinearity.ApplyMask(mSparsityMask); + + //L1 regularization lasso... + //each update? everty 1000th update? + if(mL1Const > 0) { + BaseFloat L1_const = mLearningRate*mL1Const*(mGradDivFrm?1.0:GetInput().Rows()); + mLinearity.ApplyL1(L1_const); + } + + //L2 regularization weight decay (from actual weights only) + if(mWeightcost > 0) { + BaseFloat L2_decay = -mLearningRate*mWeightcost*(mGradDivFrm?1.0:GetInput().Rows()); + mLinearity.AddScaled(L2_decay, mLinearity,1.0); + } + + mNFrames += GetInput().Rows(); + + } + + + void + CuSparseLinearity:: + UpdateMask() + { + //move data to host + Matrix<BaseFloat> linearity, linearity_correction_accu; + Matrix<BaseFloat> sparsity_mask; + + mLinearity.CopyTo(linearity); + mLinearityCorrectionAccu.CopyTo(linearity_correction_accu); + mSparsityMask.CopyTo(sparsity_mask); + + //decide on new sparsity mask + for(size_t r=0; r<sparsity_mask.Rows(); r++) { + for(size_t c=0; c<sparsity_mask.Cols(); c++) { + if(sparsity_mask(r,c) == 1.0f) { //weight active + if(fabs(linearity(r,c)) < mSparsifyWeightThreshold) { + sparsity_mask(r,c) = 0;//deactivate + linearity(r,c) = 0; + } + } else { //weight inactive + if(abs(linearity_correction_accu(r,c))/(BaseFloat)mNFrames > mUnsparsifyAccu) { + sparsity_mask(r,c) = 1;//activate + } + } + } + } + + //move data to the device + mLinearity.CopyFrom(linearity); + mSparsityMask.CopyFrom(sparsity_mask); + } + + + void + CuSparseLinearity:: + ReadFromStream(std::istream& rIn) + { + //matrix is stored transposed as SNet does + BfMatrix transpose; + rIn >> transpose; + mLinearity.CopyFrom(BfMatrix(transpose, TRANS)); + //biases stored normally + BfVector bias; + rIn >> bias; + mBias.CopyFrom(bias); + + //sparsity mask + rIn >> std::ws; + Matrix<BaseFloat> mask_transp; + if(rIn.peek() == 'm') {//load from file + rIn >> mask_transp; + } else {//or set all elements active + mask_transp.Init(transpose.Rows(),transpose.Cols()); + int items=transpose.Rows()*transpose.Stride(); + BaseFloat* p = mask_transp.pData(); + for(int i=0; i<items; i++) {//set all elements to one + *p++ = 1; + } + } + mSparsityMask.CopyFrom(BfMatrix(mask_transp,TRANS)); + + //dummy matrix with acumulated gradients + rIn >> std::ws; + if(rIn.peek() == 'm') {//load from file + BfMatrix dummy; + rIn >> dummy; + } + + if(transpose.Cols()*transpose.Rows() == 0) { + Error("Missing linearity matrix in network file"); + } + if(bias.Dim() == 0) { + Error("Missing bias vector in network file"); + } + if(mLinearity.Cols() != GetNOutputs() || + mLinearity.Rows() != GetNInputs() || + mBias.Dim() != GetNOutputs() + ){ + std::ostringstream os; + os << "Wrong dimensionalities of matrix/vector in network file\n" + << "Inputs:" << GetNInputs() + << "Outputs:" << GetNOutputs() + << "\n" + << "linearityCols:" << mLinearity.Cols() + << "linearityRows:" << mLinearity.Rows() + << "biasDims:" << mBias.Dim() + << "\n"; + Error(os.str()); + } + + assert(mLinearity.Rows() == mSparsityMask.Rows()); + assert(mLinearity.Cols() == mSparsityMask.Cols()); + + } + + + void + CuSparseLinearity:: + WriteToStream(std::ostream& rOut) + { + UpdateMask(); + + //matrix is stored transposed as SNet does + BfMatrix tmp; + mLinearity.CopyTo(tmp); + BfMatrix transpose(tmp, TRANS); + rOut << transpose; + //biases stored normally + BfVector vec; + mBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + //store mask + mSparsityMask.CopyTo(tmp); + rOut << BfMatrix(tmp,TRANS); + //store accu + mLinearityCorrectionAccu.CopyTo(tmp); + rOut << BfMatrix(tmp,TRANS); + + } + + +} //namespace + diff --git a/src/CuTNetLib/cuSparseLinearity.h b/src/CuTNetLib/cuSparseLinearity.h new file mode 100644 index 0000000..3cdf078 --- /dev/null +++ b/src/CuTNetLib/cuSparseLinearity.h @@ -0,0 +1,115 @@ +#ifndef _CUSPARSE_LINEARITY_H_ +#define _CUSPARSE_LINEARITY_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + + /** + * \brief CuSparseLinearity summation function + * + * \ingroup CuNNUpdatable + * Using weight masks to avoid fluctuation in the output + * -Weights are masked when it is lower than certain threshold - mSparsifyWeightThreshold + * -Weights are activated when the accumulated change is larger than certan value - mUnsparsifyAccu + * -L1 lasso function zeroing weights + * . + * \sa CuBiasedLinearity + */ + class CuSparseLinearity : public CuUpdatableComponent + { + public: + + CuSparseLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuSparseLinearity(); + + ComponentType GetType() const; + const char* GetName() const; + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + void Update(); + void UpdateMask(); + + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + void L1(BaseFloat l1) { + mL1Const = l1; + } + + protected: + CuMatrix<BaseFloat> mLinearity; ///< Matrix with neuron weights + CuVector<BaseFloat> mBias; ///< Vector with biases + CuMatrix<BaseFloat> mSparsityMask; ///< Mask which selects active weights + + CuMatrix<BaseFloat> mLinearityCorrection; ///< Matrix for linearity updates + CuVector<BaseFloat> mBiasCorrection; ///< Vector for bias updates + + CuMatrix<BaseFloat> mLinearityCorrectionAccu; ///< Accumulator for linearity updates + + BaseFloat mL1Const; ///< L1 regularization constant + + size_t mNFrames; ///< Number of accumulated frames + BaseFloat mSparsifyWeightThreshold; ///< Cutoff + BaseFloat mUnsparsifyAccu; ///< Threshold to unsparsify the Cutoff + + + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuSparseLinearity:: + inline + CuSparseLinearity:: + CuSparseLinearity(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred), + mLinearity(nInputs,nOutputs), mBias(nOutputs), mSparsityMask(nInputs,nOutputs), + mLinearityCorrection(nInputs,nOutputs), mBiasCorrection(nOutputs), + mLinearityCorrectionAccu(nInputs,nOutputs), + mNFrames(0), mSparsifyWeightThreshold(1.0e-3), + mUnsparsifyAccu(1e20f) + { + mLinearityCorrection.SetConst(0.0f); + mBiasCorrection.SetConst(0.0f); + mLinearityCorrectionAccu.SetConst(0.0f); + } + + + inline + CuSparseLinearity:: + ~CuSparseLinearity() + { } + + inline CuComponent::ComponentType + CuSparseLinearity:: + GetType() const + { + return CuComponent::SPARSE_LINEARITY; + } + + inline const char* + CuSparseLinearity:: + GetName() const + { + return "<sparselinearity>"; + } + + + +} //namespace + + + +#endif diff --git a/src/CuTNetLib/cuUpdatableBias.cc b/src/CuTNetLib/cuUpdatableBias.cc new file mode 100644 index 0000000..2a9cbed --- /dev/null +++ b/src/CuTNetLib/cuUpdatableBias.cc @@ -0,0 +1,96 @@ + + +#include "cuUpdatableBias.h" + + +namespace TNet +{ + + void + CuUpdatableBias:: + PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + //Y.SetConst(0.0); + Y.AddScaledRow(1.0,mBias,0.0); + Y.AddScaled(1.0,X,1.0); + } + + + void + CuUpdatableBias:: + BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y) + { + //Y.SetConst(0.0); + Y.CopyFrom(X); + } + + + void + CuUpdatableBias:: + Update() + { +#if 0 + //former implementation + BaseFloat N = static_cast<BaseFloat>(GetInput().Rows()); + + mBiasCorrection.AddColSum(-mLearningRate/N,GetErrorInput(),mMomentum); + + mBias.AddScaled(1.0,mBiasCorrection,1.0); +#endif + +#if 1 + //new implementation + BaseFloat N = 1; + if(mGradDivFrm) { + N = static_cast<BaseFloat>(GetInput().Rows()); + } + BaseFloat mmt_gain = static_cast<BaseFloat>(1.0/(1.0-mMomentum)); + N *= mmt_gain; + + mBiasCorrection.AddColSum(1.0,GetErrorInput(),mMomentum); + + mBias.AddScaled(-mLearningRate/N,mBiasCorrection,1.0); + +#endif + } + + + void + CuUpdatableBias:: + ReadFromStream(std::istream& rIn) + { + //biases stored normally + BfVector bias; + rIn >> bias; + mBias.CopyFrom(bias); + + /*if(bias.Dim() == 0) { + Error("Missing bias vector in network file"); + }*/ + if( mBias.Dim() != GetNOutputs() + ){ + std::ostringstream os; + os << "Wrong dimensionalities of matrix/vector in network file\n" + << "Inputs:" << GetNInputs() + << "Outputs:" << GetNOutputs() + << "\n" + << "biasDims:" << mBias.Dim() + << "\n"; + Error(os.str()); + } + } + + + void + CuUpdatableBias:: + WriteToStream(std::ostream& rOut) + { + BfVector vec; + mBias.CopyTo(vec); + rOut << vec; + rOut << std::endl; + } + + +} //namespace + diff --git a/src/CuTNetLib/cuUpdatableBias.h b/src/CuTNetLib/cuUpdatableBias.h new file mode 100644 index 0000000..df8066a --- /dev/null +++ b/src/CuTNetLib/cuUpdatableBias.h @@ -0,0 +1,109 @@ +#ifndef _CUUPDATABLE_BIAS_H_ +#define _CUUPDATABLE_BIAS_H_ + + +#include "cuComponent.h" +#include "cumatrix.h" + + +#include "Matrix.h" +#include "Vector.h" + + +namespace TNet { + /** + * \brief CuUpdatableBias summation function + * + * \ingroup CuNNUpdatable + * Implements forward pass: \f[ Y_i=X_i +{\beta}_i \f] + * Error propagation: \f[ E_i = e_i \f] + * + * Weight adjust: + * for bias: \f[ {\Beta}_i = {\beta}_i - \alpha(1-\mu)e_i - \mu \Delta \f] + * where + * - D for weight decay => penalizing large weight + * - \f$ \alpha \f$ for learning rate + * - \f$ \mu \f$ for momentum => avoiding oscillation + */ + class CuUpdatableBias : public CuUpdatableComponent + { + public: + + CuUpdatableBias(size_t nInputs, size_t nOutputs, CuComponent *pPred); + ~CuUpdatableBias(); + + ComponentType GetType() const; + const char* GetName() const; + + const CuMatrix<float>& GetErrorOutput(); + + void PropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + void BackpropagateFnc(const CuMatrix<BaseFloat>& X, CuMatrix<BaseFloat>& Y); + + void Update(); + + void ReadFromStream(std::istream& rIn); + void WriteToStream(std::ostream& rOut); + + void Backpropagate(); + + protected: + CuVector<BaseFloat> mBias; ///< Vector with biases + + CuVector<BaseFloat> mBiasCorrection; ///< Vector for bias updates + + }; + + + + + //////////////////////////////////////////////////////////////////////////// + // INLINE FUNCTIONS + // CuUpdatableBias:: + inline + CuUpdatableBias:: + CuUpdatableBias(size_t nInputs, size_t nOutputs, CuComponent *pPred) + : CuUpdatableComponent(nInputs, nOutputs, pPred), + mBias(nOutputs), mBiasCorrection(nOutputs) + { + mBiasCorrection.SetConst(0.0); + } + + + inline + CuUpdatableBias:: + ~CuUpdatableBias() + { } + + inline CuComponent::ComponentType + CuUpdatableBias:: + GetType() const + { + return CuComponent::UPDATABLEBIAS; + } + + inline const char* + CuUpdatableBias:: + GetName() const + { + return "<updatablebias>"; + } + + inline void + CuUpdatableBias:: + Backpropagate() + { + } + + inline const CuMatrix<BaseFloat>& + CuUpdatableBias:: + GetErrorOutput() + { + return GetErrorInput(); + } + +} //namespace + + + +#endif |