path: root/src/TNetCu.cc

/***************************************************************************
 *   copyright            : (C) 2011 by Karel Vesely,UPGM,FIT,VUT,Brno     *
 *   email                : iveselyk@fit.vutbr.cz                          *
 ***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the APACHE License as published by the          *
 *   Apache Software Foundation; either version 2.0 of the License,        *
 *   or (at your option) any later version.                                *
 *                                                                         *
 ***************************************************************************/

#define SVN_DATE       "$Date: 2012-03-23 14:22:49 +0100 (Fri, 23 Mar 2012) $"
#define SVN_AUTHOR     "$Author: iveselyk $"
#define SVN_REVISION   "$Revision: 110 $"
#define SVN_ID         "$Id: TNetCu.cc 110 2012-03-23 13:22:49Z iveselyk $"

#define MODULE_VERSION "1.0.0 "__TIME__" "__DATE__" "SVN_ID  

/**
 * \file TNetCu.cc
 * \brief DNN training Entry Program CUDA-version
 */


/*** TNetLib includes */
#include "Error.h"
#include "Timer.h"
#include "Features.h"
#include "Labels.h"
#include "Common.h"
#include "MlfStream.h"
#include "UserInterface.h"
#include "Timer.h"

/*** TNet includes */
#include "cuObjectiveFunction.h"
#include "cuNetwork.h"
#include "cuCache.h"

/*** STL includes */
#include <iostream>
#include <sstream>
#include <numeric>




//////////////////////////////////////////////////////////////////////
// DEFINES
//

#define SNAME "TNET"

using namespace TNet;

void usage(const char* progname) 
{
  const char *tchrptr;
  if ((tchrptr = strrchr(progname, '\\')) != NULL) progname = tchrptr+1;
  if ((tchrptr = strrchr(progname, '/')) != NULL) progname = tchrptr+1;
  fprintf(stderr,
"\n%s version " MODULE_VERSION "\n"
"\nUSAGE: %s [options] DataFiles...\n\n"
" Option                                                     Default\n\n"
" -c         Enable crossvalidation                          off\n"
" -m file    Set label map of NN outputs                     \n"
" -n f       Set learning rate to f                          0.06\n"
" -o ext     Set target model ext                            None\n"
" -A         Print command line arguments                    Off\n" 
" -C cf      Set config file to cf                           Default\n"
" -D         Display configuration variables                 Off\n"
" -H mmf     Load NN macro file                              \n"
" -I mlf     Load master label file mlf                      \n"
" -L dir     Set input label (or net) dir                    Current\n"
" -M dir     Dir to write NN macro files                     Current\n"
" -O fn      Objective function [mse,xent]                   xent\n"
" -S file    Set script file                                 None\n"
" -T N       Set trace flags to N                            0\n" 
" -V         Print version information                       Off\n"
" -X ext     Set input label file ext                        lab\n"
"\n"
"BUNCHSIZE CACHESIZE CROSSVALIDATE FEATURETRANSFORM GPUSELECT GRADDIVFRM L1 LEARNINGRATE LEARNRATEFACTORS MLFTRANSC MOMENTUM NATURALREADORDER OBJECTIVEFUNCTION OUTPUTLABELMAP PRINTCONFIG PRINTVERSION RANDOMIZE SCRIPT SEED SOURCEMLF SOURCEMMF SOURCETRANSCDIR SOURCETRANSCEXT TARGETMMF TARGETMODELDIR TARGETMODELEXT TRACE WEIGHTCOST\n"
"\n"
"STARTFRMEXT ENDFRMEXT CMEANDIR CMEANMASK VARSCALEDIR VARSCALEMASK VARSCALEFN TARGETKIND DERIVWINDOWS DELTAWINDOW ACCWINDOW THIRDWINDOW\n"
"\n"
" %s is Copyright (C) 2010-2011 Karel Vesely\n"
" licensed under the APACHE License, version 2.0\n"
" Bug reports, feedback, etc, to: iveselyk@fit.vutbr.cz\n"
"\n", progname, progname, progname);
  exit(-1);
}




///////////////////////////////////////////////////////////////////////
// MAIN FUNCTION
//

/**
 * \brief Main Procedure
 *
 * Handles params extraction and all other inputs
 * Entry point of CUDA based neural network training methods.
 */
int main(int argc, char *argv[]) try
{
  const char* p_option_string =
    " -c n   CROSSVALIDATE=TRUE"
    " -m r   OUTPUTLABELMAP" 
    " -n r   LEARNINGRATE" 
    " -o r   TARGETMODELEXT" 
    " -D n   PRINTCONFIG=TRUE"
    " -H l   SOURCEMMF"
    " -I r   SOURCEMLF"
    " -L r   SOURCETRANSCDIR"
    " -M r   TARGETMODELDIR"
    " -O r   OBJECTIVEFUNCTION" 
    " -S l   SCRIPT"
    " -T r   TRACE"
    " -V n   PRINTVERSION=TRUE"
    " -X r   SOURCETRANSCEXT";


  UserInterface        ui;
  /** \brief Feature specified in params& scp*/
  FeatureRepository    feature_repo;       
  /** \brief Label file*/
  LabelRepository      label_repo;
  /** \brief DNN network*/
  CuNetwork            network;
  /** \brief Transform network*/
  CuNetwork            transform_network;
  /** \brief Objective Function*/
  CuObjectiveFunction* p_obj_function = NULL;
  Timer                timer;
  Timer                timer_frontend;
  double               time_frontend = 0.0;

 
  const char*                       p_script;
  const char*                       p_output_label_map;
  BaseFloat                         learning_rate;
  const char*                       learning_rate_factors;
  BaseFloat                         momentum;
  BaseFloat                         weightcost;
  BaseFloat                         l1;
  bool                              grad_div_frm;
  CuObjectiveFunction::ObjFunType   obj_fun_id;

  const char*                       p_source_mmf_file;
  const char*                       p_input_transform;
  //const char*                       p_input_transform2;

  const char*                       p_targetmmf; ///< SNet legacy --TARGETMMF
        char                        p_trg_mmf_file[4096];
  const char*                       p_trg_mmf_dir;
  const char*                       p_trg_mmf_ext;

  const char*                       p_source_mlf_file;
  const char*                       p_src_lbl_dir;
  const char*                       p_src_lbl_ext;
        char                        p_lbl_file[4096];
  bool                              mlf_transc;

  int                               bunch_size;
  int                               cache_size;
  bool                              randomize;
  long int                          seed;

  bool                              cross_validate;

  int                               trace;

  int                               gpu_select;

  // variables for feature repository
  bool                              swap_features;
  int                               target_kind;
  int                               deriv_order;
  int*                              p_deriv_win_lenghts;
  int                               start_frm_ext;
  int                               end_frm_ext;
        char*                       cmn_path;
        char*                       cmn_file;
  const char*                       cmn_mask;
        char*                       cvn_path;
        char*                       cvn_file;
  const char*                       cvn_mask;
  const char*                       cvg_file;

 
  /// OPTION PARSING ........ use the STK option parsing
  if (argc == 1) { usage(argv[0]); return 1; }
  int args_parsed = ui.ParseOptions(argc, argv, p_option_string, SNAME);


  /// OPTION RETRIEVAL ........ extract the feature parameters
  swap_features = !ui.GetBool(SNAME":NATURALREADORDER", TNet::IsBigEndian());
  
  target_kind = ui.GetFeatureParams(&deriv_order, &p_deriv_win_lenghts,
       &start_frm_ext, &end_frm_ext, &cmn_path, &cmn_file, &cmn_mask,
       &cvn_path, &cvn_file, &cvn_mask, &cvg_file, SNAME":", 0);


  /// extract other parameters
  p_source_mmf_file   = ui.GetStr(SNAME":SOURCEMMF",     NULL);
  p_input_transform   = ui.GetStr(SNAME":FEATURETRANSFORM",  NULL);
  
  p_targetmmf         = ui.GetStr(SNAME":TARGETMMF",     NULL);///< has higher priority than "dir/file.ext" composition (SNet legacy)
  p_trg_mmf_dir       = ui.GetStr(SNAME":TARGETMODELDIR",  "");///< dir for composition
  p_trg_mmf_ext       = ui.GetStr(SNAME":TARGETMODELEXT",  "");///< ext for composition

  p_script            = ui.GetStr(SNAME":SCRIPT",         NULL);
  p_output_label_map  = ui.GetStr(SNAME":OUTPUTLABELMAP", NULL);
  learning_rate       = ui.GetFlt(SNAME":LEARNINGRATE"  , 0.06f);
  learning_rate_factors = ui.GetStr(SNAME":LEARNRATEFACTORS", NULL);
  momentum            = ui.GetFlt(SNAME":MOMENTUM"      , 0.0);
  weightcost          = ui.GetFlt(SNAME":WEIGHTCOST"    , 0.0);
  l1                  = ui.GetFlt(SNAME":L1"            , 0.0);
  grad_div_frm        = ui.GetBool(SNAME":GRADDIVFRM",     true);

  obj_fun_id          = static_cast<CuObjectiveFunction::ObjFunType>(
                        ui.GetEnum(SNAME":OBJECTIVEFUNCTION", 
                                   CuObjectiveFunction::CROSS_ENTROPY, //< default
                                   "xent", CuObjectiveFunction::CROSS_ENTROPY,
                                   "mse", CuObjectiveFunction::MEAN_SQUARE_ERROR
                        ));

  p_source_mlf_file   = ui.GetStr(SNAME":SOURCEMLF",       NULL);
  p_src_lbl_dir       = ui.GetStr(SNAME":SOURCETRANSCDIR", NULL);
  p_src_lbl_ext       = ui.GetStr(SNAME":SOURCETRANSCEXT", "lab");
  mlf_transc          = ui.GetBool(SNAME":MLFTRANSC",      true);



  bunch_size          = ui.GetInt(SNAME":BUNCHSIZE", 256);
  cache_size          = ui.GetInt(SNAME":CACHESIZE", 12800);
  randomize           = ui.GetBool(SNAME":RANDOMIZE", true);

  //cannot get long int
  seed                = ui.GetInt(SNAME":SEED", 0);

  cross_validate      = ui.GetBool(SNAME":CROSSVALIDATE",  false);

  trace       = ui.GetInt(SNAME":TRACE",               0);
  if(trace&4) { CuDevice::Instantiate().Verbose(true); }

  gpu_select  = ui.GetInt(SNAME":GPUSELECT", -1);
  if(gpu_select >= 0) { CuDevice::Instantiate().SelectGPU(gpu_select); }



  /// process the parameters
  if(ui.GetBool(SNAME":PRINTCONFIG", false)) {
    std::cout << std::endl;
    ui.PrintConfig(std::cout);
    std::cout << std::endl;
  }
  if(ui.GetBool(SNAME":PRINTVERSION", false)) {
    std::cout << std::endl;
    std::cout << "======= TNET v"MODULE_VERSION" =======" << std::endl;
    std::cout << std::endl;
  }
  ui.CheckCommandLineParamUse();
  

  /// the rest of the parameters are the feature files
  for (; args_parsed < argc; args_parsed++) {
    feature_repo.AddFile(argv[args_parsed]);
  }

  //**************************************************************************
  //**************************************************************************
  /// OPTION PARSING DONE .....................................................


  /// read the input transform network from file p_input_transform
  if(NULL != p_input_transform) { 
    if(trace&1) TraceLog(std::string("Reading input transform network: ")+p_input_transform);
    transform_network.ReadNetwork(p_input_transform);
  }


  /// read the neural network from file p_source_mmf_file
  if(NULL != p_source_mmf_file) { 
    if(trace&1) TraceLog(std::string("Reading network: ")+p_source_mmf_file);
    network.ReadNetwork(p_source_mmf_file);
  } else {
    Error("Source MMF must be specified [-H]");
  }


  /// initialize the feature repository 
  feature_repo.Init(
    swap_features, start_frm_ext, end_frm_ext, target_kind,
    deriv_order, p_deriv_win_lenghts, 
    cmn_path, cmn_mask, cvn_path, cvn_mask, cvg_file
  );
  feature_repo.Trace(trace);
  if(NULL != p_script) {
    feature_repo.AddFileList(p_script);
  } else {
    Warning("WARNING: The script file is missing [-S]");
  }


  /// initialize the label repository
  if(mlf_transc) {
    if(NULL == p_source_mlf_file)
      Error("Source mlf file file is missing [-I]");
    if(NULL == p_output_label_map)
      Error("Output label map is missing [-m]");

    if(trace&1) TraceLog(std::string("Indexing labels: ")+p_source_mlf_file);
    label_repo.Init(p_source_mlf_file, p_output_label_map, p_src_lbl_dir, p_src_lbl_ext);
    label_repo.Trace(trace);
  }


  /// get objective function instance
  p_obj_function = CuObjectiveFunction::Factory(obj_fun_id);

  /// set the learnrate, momentum, weightcost
  network.SetLearnRate(learning_rate, learning_rate_factors);
  network.SetMomentum(momentum);
  network.SetWeightcost(weightcost);
  network.SetL1(l1);

  /// set division of gradient by number of frames -> grad_div_frm
  /// why grad div by frame num.
  network.SetGradDivFrm(grad_div_frm);

  /// seed the random number generator
  if(seed == 0) {
    struct timeval tv;
    if (gettimeofday(&tv, 0) == -1) {
      assert(0 && "gettimeofday does not work.");
      exit(-1);
    }
    seed = (int)(tv.tv_sec) + (int)tv.tv_usec;
  }
  srand48(seed);


  
  
  //**********************************************************************
  //**********************************************************************
  /// INITIALIZATION DONE .................................................
  //
  /// Start training
  timer.Start();
  std::cout << "===== TNET " 
            << (cross_validate?"CROSSVALIDATION":"TRAINING") 
            << " STARTED =====" << std::endl;
  std::cout << "Objective function: " 
            << p_obj_function->GetTypeLabel() << std::endl;
  if(!cross_validate) {
    network.PrintLearnRate();
    std::cout << "momentum: " << momentum
              << " weightcost: " << weightcost << std::endl;
    std::cout << "using seed: " << seed << std::endl;
  }

  /// make the cachesize divisible by bunchsize
  cache_size = (cache_size/bunch_size)*bunch_size;
  std::cout << "Bunchsize:" << bunch_size
            << " Cachesize:" << cache_size << "\n";

  CuCache cache;
  cache.Init(cache_size,bunch_size);
  cache.Trace(trace);
  feature_repo.Rewind();
  
  //**********************************************************************
  //**********************************************************************
  /// MAIN LOOP start
  /**
   * Main loop
   *  - Filling Cache from feature_repo
   *    - Read Features, perform transform, trim feature
   *    - Read labels (From label repo/HTK-matrix file)
   *    .
   *  - Randomize the Cache (Only time random ever used?!)
   *  - Training when Cache is not empty
   *    - Get training data from cache
   *    - Eval error using obj_fnc
   *    - BP
   *    .
   *  .
   */
  CuMatrix<BaseFloat> feats, output, labs, globerr;
  while(!feature_repo.EndOfList()) {
    timer_frontend.Start();
    //fill cache
    while(!cache.Full() && !feature_repo.EndOfList()) {
      Matrix<BaseFloat> feats_host;
      CuMatrix<BaseFloat> feats_original;
      CuMatrix<BaseFloat> feats_expanded;

      //read feats, perfrom feature transform
      feature_repo.ReadFullMatrix(feats_host);
      feats_host.CheckData(feature_repo.Current().Logical());
      feats_original.CopyFrom(feats_host);
      transform_network.Propagate(feats_original,feats_expanded);

      //trim the start/end context
      int rows = feats_expanded.Rows()-start_frm_ext-end_frm_ext;
      CuMatrix<BaseFloat> feats_trim(rows,feats_expanded.Cols());
      feats_trim.CopyRows(rows,start_frm_ext,feats_expanded,0);

      //read labels
      Matrix<BaseFloat> labs_host; CuMatrix<BaseFloat> labs_cu;
      if(mlf_transc) {
        //read from label repository
        label_repo.GenDesiredMatrix(labs_host,feats_trim.Rows(), 
                                    feature_repo.CurrentHeader().mSamplePeriod,
                                    feature_repo.Current().Logical().c_str());
      } else {
        //read targets from HTK-matrix file
        MakeHtkFileName(p_lbl_file,feature_repo.Current().Logical().c_str(), 
                        p_src_lbl_dir, p_src_lbl_ext);
        labs_host.LoadHTK(p_lbl_file);
      }
      labs_cu.CopyFrom(labs_host);
      //test number of rows
      if(labs_cu.Rows() != feats_trim.Rows()) {
        Error(std::string("Nonmatching number number of input/target examples")
                          + feature_repo.Current().Logical().c_str());
      }
      
      //add to cache
      cache.AddData(feats_trim,labs_cu);

      feature_repo.MoveNext();
    }
    timer_frontend.End(); time_frontend += timer_frontend.Val();
   
    if(randomize) { 
      //randomize the cache
      cache.Randomize();
    }

    while(!cache.Empty()) {
      //get training data
      cache.GetBunch(feats,labs);

      //forward pass
      network.Propagate(feats,output);
      //accumulate error, get global err
      p_obj_function->Evaluate(output,labs,globerr);

      //backward pass
      if(!cross_validate) {
        network.Backpropagate(globerr);
      }
      if(trace&2) std::cout << "." << std::flush; 
    }
  }



  //**********************************************************************
  //**********************************************************************
  /// TRAINING FINISHED .................................................
  //
  /// Let's store the network, report the log


  if(trace&1) TraceLog("Training finished");

  //write the network
  if(!cross_validate) {
    if (NULL != p_targetmmf) {
      if(trace&1) TraceLog(std::string("Writing network: ")+p_targetmmf);
      network.WriteNetwork(p_targetmmf);
    } else {
      MakeHtkFileName(p_trg_mmf_file, p_source_mmf_file, p_trg_mmf_dir, p_trg_mmf_ext);
      if(trace&1) TraceLog(std::string("Writing network: ")+p_trg_mmf_file);
      network.WriteNetwork(p_trg_mmf_file);
    }
  }

  timer.End();
  std::cout << "===== TNET "
            << (cross_validate?"CROSSVALIDATION":"TRAINING") 
            << " FINISHED ( " << timer.Val() << "s ) "
            << "[FPS:" << p_obj_function->GetFrames() / timer.Val() 
            << ",RT:" << 1.0f / (p_obj_function->GetFrames() / timer.Val() / 100.0f)
            << "] =====" << std::endl;

  //report objective function (accuracy, frame counts...)
  std::cout << "-- " << (cross_validate?"CV ":"TR ") << p_obj_function->Report();

  if(trace &4) {
    std::cout << "\n== PROFILE ==\nT-fe: " << time_frontend << std::endl;
  }
  
  return  0; ///finish OK

} catch (std::exception& rExc) {
  std::cerr << "Exception thrown" << std::endl;
  std::cerr << rExc.what() << std::endl;
  return  1;
}