From 20587ac550cfcb2d7b3d6ec16e46ba1a8d0af869 Mon Sep 17 00:00:00 2001
From: Joe Zhao <ztuowen@gmail.com>
Date: Wed, 13 May 2015 13:35:03 +0800
Subject: added split

---
 CMakeLists.txt             |   9 ++-
 main.cpp                   | 135 -----------------------------------
 split.cpp                  |  76 ++++++++++++++++++++
 tools/fileDataProvider.cpp | 173 +++++++++++++++++++++++++++++++++++++++++++++
 tools/fileDataProvider.h   |  91 ++----------------------
 train.cpp                  | 147 ++++++++++++++++++++++++++++++++++++++
 6 files changed, 408 insertions(+), 223 deletions(-)
 delete mode 100644 main.cpp
 create mode 100644 split.cpp
 create mode 100644 tools/fileDataProvider.cpp
 create mode 100644 train.cpp

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6920572..180456c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -11,6 +11,9 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
 FIND_PACKAGE( Boost COMPONENTS program_options REQUIRED )
 INCLUDE_DIRECTORIES( ${Boost_INCLUDE_DIR})
 
-set(SOURCE_FILES main.cpp ./model/ranksvm.cpp ./model/ranksvmtn.cpp ./model/rankaccu.cpp)
-add_executable(ranksvm ${SOURCE_FILES} model/rankaccu.h model/ranksvm.h model/ranksvmtn.h tools/dataProvider.h tools/easylogging++.h tools/matrixIO.h tools/fileDataProvider.h)
-TARGET_LINK_LIBRARIES( ranksvm ${Boost_LIBRARIES} )
\ No newline at end of file
+set(SOURCE_FILES model/ranksvm.cpp model/ranksvmtn.cpp model/rankaccu.cpp tools/fileDataProvider.cpp)
+add_executable(ranksvm train.cpp ${SOURCE_FILES} model/rankaccu.h model/ranksvm.h model/ranksvmtn.h tools/dataProvider.h tools/matrixIO.h tools/fileDataProvider.h)
+add_executable(split split.cpp ${SOURCE_FILES})
+add_dependencies(ranksvm split)
+TARGET_LINK_LIBRARIES( ranksvm ${Boost_LIBRARIES} )
+TARGET_LINK_LIBRARIES( split ${Boost_LIBRARIES})
\ No newline at end of file
diff --git a/main.cpp b/main.cpp
deleted file mode 100644
index e8666f8..0000000
--- a/main.cpp
+++ /dev/null
@@ -1,135 +0,0 @@
-#include <iostream>
-#include <Eigen/Dense>
-#include <boost/program_options.hpp>
-#include <list>
-#include "tools/easylogging++.h"
-#include "model/ranksvmtn.h"
-#include "tools/fileDataProvider.h"
-#include "model/rankaccu.h"
-
-INITIALIZE_EASYLOGGINGPP
-
-using namespace Eigen;
-using namespace std;
-namespace po = boost::program_options;
-
-po::variables_map vm;
-
-typedef int (*mainFunc)(DataProvider &dp);
-
-int train(DataProvider &dp) {
-    RSVM *rsvm;
-    rsvm = RSVM::loadModel(vm["model"].as<string>());
-
-    dp.open();
-    DataList D;
-
-    LOG(INFO)<<"Training started";
-    dp.getAllDataSet(D);
-    LOG(INFO)<<"Read "<<D.getSize()<<" entries with "<< D.getfSize()<<" features";
-    rsvm->train(D);
-    vector<double> L;
-    rsvm->predict(D,L);
-
-    LOG(INFO)<<"Training finished,saving model";
-
-    dp.close();
-    rsvm->saveModel(vm["output"].as<string>().c_str());
-    delete rsvm;
-    return 0;
-}
-
-int predict(DataProvider &dp) {
-    RSVM *rsvm;
-    rsvm = RSVM::loadModel(vm["model"].as<string>().c_str());
-
-    dp.open();
-    DataList D;
-    vector<double> L;
-    CMC cmc;
-    LOG(INFO)<<"Prediction started";
-
-    ofstream fout;
-
-    ostream* ot;
-
-    if (vm.count("output")) {
-        fout.open(vm["output"].as<string>().c_str());
-        ot=&fout;
-    }
-    else
-        ot=&cout;
-
-    while (!dp.EOFile())
-    {
-        dp.getDataSet(D);
-        LOG(INFO)<<"Read "<<D.getSize()<<" entries with "<< D.getfSize()<<" features";
-        rsvm->predict(D,L);
-
-        if (vm.count("validate"))
-        {
-            rank_accu(D,L);
-            if (vm.count("cmc"))
-                rank_CMC(D,L,cmc);
-        }
-
-        if (vm.count("output") || !vm.count("validate"))
-            for (int i=0; i<L.size();++i)
-                *ot<<L[i]<<endl;
-    }
-
-    LOG(INFO)<<"Finished";
-    if (vm.count("cmc"))
-    {
-        LOG(INFO)<< "CMC accounted over " <<cmc.getCount() << " queries";
-        *ot << "CMC"<<endl;
-        vector<double> cur = cmc.getAcc();
-        for (int i = 0;i<CMC_MAX;++i)
-        *ot << cur[i]<<endl;
-    }
-    if (vm.count("output"))
-        fout.close();
-    dp.close();
-    delete rsvm;
-    return 0;
-}
-
-int main(int argc, char **argv) {
-    // Defining program options
-    po::options_description desc("Allowed options");
-    desc.add_options()
-            ("help,h", "produce help message")
-            ("train,T", "training model")
-            ("validate,V", "validate model")
-            ("predict,P", "use model for prediction")
-            ("cmc,C", "enable cmc auditing")
-            ("model,m", po::value<string>(), "set input model file")
-            ("output,o", po::value<string>(), "set output model/prediction file")
-            ("feature,i", po::value<string>(), "set input feature file");
-
-    // Parsing program options
-    po::store(po::parse_command_line(argc, argv, desc), vm);
-    po::notify(vm);
-
-    // Print help if necessary
-    if (vm.count("help") || !(vm.count("train") || vm.count("validate") || vm.count("predict"))) {
-        cout << desc;
-        return 0;
-    }
-    mainFunc mainf;
-    if (vm.count("train")) {
-        mainf = &train;
-    }
-    else if (vm.count("validate")||vm.count("predict")) {
-        mainf = &predict;
-    }
-    else return 0;
-    DataProvider* dp;
-    if (vm["feature"].as<string>().find(".rid") == string::npos)
-        dp = new FileDP(vm["feature"].as<string>());
-    else
-        dp = new RidFileDP(vm["feature"].as<string>());
-    mainf(*dp);
-    delete dp;
-    return 0;
-}
diff --git a/split.cpp b/split.cpp
new file mode 100644
index 0000000..be80545
--- /dev/null
+++ b/split.cpp
@@ -0,0 +1,76 @@
+//
+// Created by joe on 5/13/15.
+//
+
+#include <iostream>
+#include <boost/program_options.hpp>
+#include "tools/dataProvider.h"
+#include "tools/fileDataProvider.h"
+#include <vector>
+#include <fstream>
+
+INITIALIZE_EASYLOGGINGPP
+
+using namespace std;
+namespace po = boost::program_options;
+
+po::variables_map vm;
+
+int outputRid(vector<DataEntry*> a,int fsize,string fname)
+{
+    ofstream fout(fname.c_str());
+    fout<<fsize<<endl;
+    for (int i=0;i<a.size();++i)
+    {
+        fout<< a[i]->qid;
+        for (int j=0;j<fsize;++j)
+            fout<<" "<< a[i]->feature(j);
+        fout<<endl;
+    }
+    fout<<0;
+    fout.close();
+}
+
+int main(int argc, char **argv)
+{
+    el::Configurations defaultConf;
+    defaultConf.setToDefault();
+    // Values are always std::string
+    defaultConf.set(el::Level::Global,el::ConfigurationType::Enabled, "false");
+    // default logger uses default configurations
+    el::Loggers::reconfigureLogger("default", defaultConf);
+    po::options_description desc("Allowed options");
+    desc.add_options()
+            ("help,h", "produce help message")
+            ("query,Q", "Query person count")
+            ("count,c", po::value<int>(), "take number")
+            ("take,a", po::value<string>(), "set output rid file 1(taken)")
+            ("left,b", po::value<string>(), "set output rid file 2(left)")
+            ("input,i", po::value<string>(), "set input Rid file");
+
+    po::store(po::parse_command_line(argc, argv, desc), vm);
+    po::notify(vm);
+    // Print help if necessary
+    if (vm.count("help")) {
+        cout << desc;
+        return 0;
+    }
+
+    if (vm.count("query")){
+        RidFileDP dp(vm["input"].as<string>().c_str());
+        dp.open();
+        cout<<dp.getpSize()<<endl;
+        dp.close();
+        return 0;
+    }
+
+    RidFileDP dp(vm["input"].as<string>().c_str());
+    vector<DataEntry*> a;
+    vector<DataEntry*> b;
+    dp.open();
+    dp.take(vm["count"].as<int>(),a,b);
+    outputRid(a,dp.getfSize(),vm["take"].as<string>());
+    outputRid(b,dp.getfSize(),vm["left"].as<string>());
+    dp.close();
+    return 0;
+}
\ No newline at end of file
diff --git a/tools/fileDataProvider.cpp b/tools/fileDataProvider.cpp
new file mode 100644
index 0000000..e9b7f3d
--- /dev/null
+++ b/tools/fileDataProvider.cpp
@@ -0,0 +1,173 @@
+//
+// Created by joe on 5/13/15.
+//
+
+#include "fileDataProvider.h"
+#include <random>
+#include <ctime>
+
+using namespace std;
+
+mt19937 gen;
+
+int FileDP::getDataSet(DataList &out){
+    DataEntry* e;
+    out.clear();
+    int fsize;
+    fin>>fsize;
+    LOG(INFO)<<"Feature size:"<<fsize;
+    out.setfSize(fsize);
+    while (!fin.eof()) {
+        e = new DataEntry;
+        fin>>e->rank;
+        if (e->rank == 0)
+        {
+            delete e;
+            break;
+        }
+        fin>>e->qid;
+        e->feature.resize(fsize);
+        for (int i=0;i<fsize;++i) {
+            fin>>e->feature(i);
+        }
+        out.addEntry(e);
+    }
+    eof=true;
+    return 0;
+}
+
+void RidFileDP::readEntries() {
+    DataEntry *e;
+    int fsize;
+    d.clear();
+    fin >> fsize;
+    LOG(INFO) << "Feature size:" << fsize;
+    d.setfSize(fsize);
+    while (!fin.eof()) {
+        e = new DataEntry;
+        fin >> e->qid;
+        if (e->qid == "0") {
+            delete e;
+            break;
+        }
+        e->feature.resize(fsize);
+        e->rank=-1;
+        for (int i = 0; i < fsize; ++i) {
+            fin >> e->feature(i);
+        }
+        d.addEntry(e);
+    }
+    pos = 0;
+    qid = 1;
+    read = true;
+}
+
+int RidFileDP::getDataSet(DataList &out){
+    DataEntry *e;
+    int fsize;
+    if (!read)
+        readEntries();
+    out.clear();
+    fsize = d.getfSize();
+    out.setfSize(fsize);
+    std::vector<DataEntry*> & dat = d.getData();
+    for (int i=0;i<d.getSize();++i)
+        if (i!=pos)
+        {
+            if (dat[i]->qid == dat[pos]->qid)
+            {
+                e = new DataEntry;
+                e->rank=1;
+                dat[i]->rank=qid;
+            }
+            else
+            {
+                e = new DataEntry;
+                e->rank=-1;
+            }
+            e->feature.resize(d.getfSize());
+            e->qid=dat[pos]->qid;
+            for (int j = 0; j < fsize; ++j) {
+                e->feature(j) = fabs(dat[i]->feature(j) -dat[pos]->feature(j));
+            }
+            out.addEntry(e);
+        }
+    dat[pos]->qid=std::to_string(qid);
+    ++qid;
+    dat[pos]->rank=qid;
+    while (pos<dat.size() && dat[pos]->rank!=-1)
+        ++pos;
+    if (pos==d.getSize())
+        eof = true;
+    return 0;
+}
+
+int RidFileDP::getpSize() {
+    std::vector<string> p;
+    if (!read)
+        readEntries();
+    std::vector<DataEntry*> &dat = d.getData();
+    for (int i=0;i<dat.size();++i)
+    {
+        bool ext=false;
+        for (int j=0;j<p.size();++j)
+            if (p[j] == dat[i]->qid )
+            {
+                ext=true;
+                break;
+            }
+        if (!ext)
+            p.push_back(dat[i]->qid);
+    }
+    return p.size();
+};
+
+void scrambler(vector<DataEntry*> &dat)
+{
+    DataEntry* e;
+    int sz=(int)dat.size();
+    for (int i=0;i<sz;++i)
+    {
+        int pos = (int)(gen()%(sz-i));
+        e=dat[pos];
+        dat[pos] = dat[sz-i-1];
+        dat[sz-i-1] = e;
+    }
+}
+
+void RidFileDP::take(int n,vector<DataEntry*> &a,vector<DataEntry*> &b)
+{
+    gen.seed(time(NULL));
+    DataEntry *e;
+    if (!read)
+        readEntries();
+    vector<DataEntry*> tmp;
+    tmp.reserve(d.getSize());
+    a.clear();
+    b.clear();
+    std::vector<DataEntry*> &dat = d.getData();
+    scrambler(tmp);
+    for (int i=0;i<dat.size();++i)
+        tmp.push_back(dat[i]);
+    int pos = 0;
+    string qid;
+    for (int i=0;i<n;++i)
+    {
+        while (tmp[pos]==NULL)
+            ++pos;
+        qid = tmp[pos]->qid;
+        a.push_back(tmp[pos]);
+        tmp[pos]=NULL;
+        for (int j = pos+1; j< tmp.size();++j)
+            if (tmp[j]!=NULL &&tmp[j]->qid==qid)
+            {
+                a.push_back(tmp[j]);
+                tmp[j]=NULL;
+            }
+    }
+    for (int i=0;i<tmp.size();++i)
+        if (tmp[i]!=NULL)
+            b.push_back(tmp[i]);
+    scrambler(a);
+    scrambler(b);
+}
\ No newline at end of file
diff --git a/tools/fileDataProvider.h b/tools/fileDataProvider.h
index f54a38e..7bea92d 100644
--- a/tools/fileDataProvider.h
+++ b/tools/fileDataProvider.h
@@ -16,31 +16,7 @@ private:
     std::ifstream fin;
 public:
     FileDP(std::string fn=""):fname(fn){};
-    virtual int getDataSet(DataList &out){
-        DataEntry* e;
-        out.clear();
-        int fsize;
-        fin>>fsize;
-        LOG(INFO)<<"Feature size:"<<fsize;
-        out.setfSize(fsize);
-        while (!fin.eof()) {
-            e = new DataEntry;
-            fin>>e->rank;
-            if (e->rank == 0)
-            {
-                delete e;
-                break;
-            }
-            fin>>e->qid;
-            e->feature.resize(fsize);
-            for (int i=0;i<fsize;++i) {
-                fin>>e->feature(i);
-            }
-            out.addEntry(e);
-        }
-        eof=true;
-        return 0;
-    }
+    virtual int getDataSet(DataList &out);
     virtual int open(){fin.open(fname); eof=false;return 0;};
     virtual int close(){fin.close();return 0;};
 };
@@ -58,68 +34,13 @@ private:
     int qid;
 public:
     RidFileDP(std::string fn=""):fname(fn){read=false;};
-    virtual int getDataSet(DataList &out){
-        DataEntry *e;
-        int fsize;
-        if (!read) {
-            d.clear();
-            fin >> fsize;
-            LOG(INFO) << "Feature size:" << fsize;
-            d.setfSize(fsize);
-            while (!fin.eof()) {
-                e = new DataEntry;
-                fin >> e->qid;
-                if (e->qid == "0") {
-                    delete e;
-                    break;
-                }
-                e->feature.resize(fsize);
-                e->rank=-1;
-                for (int i = 0; i < fsize; ++i) {
-                    fin >> e->feature(i);
-                }
-                d.addEntry(e);
-            }
-            pos = 0;
-            qid = 1;
-            read = true;
-        }
-        out.clear();
-        fsize = d.getfSize();
-        out.setfSize(fsize);
-        std::vector<DataEntry*> & dat = d.getData();
-        for (int i=0;i<d.getSize();++i)
-            if (i!=pos)
-            {
-                if (dat[i]->qid == dat[pos]->qid)
-                {
-                    e = new DataEntry;
-                    e->rank=1;
-                    dat[i]->rank=qid;
-                }
-                else
-                {
-                    e = new DataEntry;
-                    e->rank=-1;
-                }
-                e->feature.resize(d.getfSize());
-                e->qid=dat[pos]->qid;
-                for (int j = 0; j < fsize; ++j) {
-                    e->feature(j) = fabs(dat[i]->feature(j) -dat[pos]->feature(j));
-                }
-                out.addEntry(e);
-            }
-        dat[pos]->qid=std::to_string(qid);
-        ++qid;
-        dat[pos]->rank=qid;
-        while (pos<dat.size() && dat[pos]->rank!=-1)
-            ++pos;
-        if (pos==d.getSize())
-            eof = true;
-        return 0;
-    }
+    void readEntries();
+    int getfSize() { if(!read) readEntries(); return d.getfSize();};
+    int getpSize();
+    virtual int getDataSet(DataList &out);
     virtual int open(){fin.open(fname); eof=false;return 0;};
     virtual int close(){fin.close(); d.clear();return 0;};
+    void take(int n,std::vector<DataEntry*> &a,std::vector<DataEntry*> &b);
 };
 
 #endif
\ No newline at end of file
diff --git a/train.cpp b/train.cpp
new file mode 100644
index 0000000..a0c62a9
--- /dev/null
+++ b/train.cpp
@@ -0,0 +1,147 @@
+#include <iostream>
+#include <Eigen/Dense>
+#include <boost/program_options.hpp>
+#include <list>
+#include "tools/easylogging++.h"
+#include "model/ranksvmtn.h"
+#include "tools/fileDataProvider.h"
+#include "model/rankaccu.h"
+
+INITIALIZE_EASYLOGGINGPP
+
+using namespace Eigen;
+using namespace std;
+namespace po = boost::program_options;
+
+po::variables_map vm;
+
+typedef int (*mainFunc)(DataProvider &dp);
+
+int train(DataProvider &dp) {
+    RSVM *rsvm;
+    rsvm = RSVM::loadModel(vm["model"].as<string>());
+
+    dp.open();
+    DataList D;
+
+    LOG(INFO)<<"Training started";
+    dp.getAllDataSet(D);
+    LOG(INFO)<<"Read "<<D.getSize()<<" entries with "<< D.getfSize()<<" features";
+    rsvm->train(D);
+    vector<double> L;
+    rsvm->predict(D,L);
+
+    LOG(INFO)<<"Training finished,saving model";
+
+    dp.close();
+    rsvm->saveModel(vm["output"].as<string>().c_str());
+    delete rsvm;
+    return 0;
+}
+
+int predict(DataProvider &dp) {
+    RSVM *rsvm;
+    rsvm = RSVM::loadModel(vm["model"].as<string>().c_str());
+
+    dp.open();
+    DataList D;
+    vector<double> L;
+    CMC cmc;
+    LOG(INFO)<<"Prediction started";
+
+    ofstream fout;
+
+    ostream* ot;
+
+    if (vm.count("output")) {
+        fout.open(vm["output"].as<string>().c_str());
+        ot=&fout;
+    }
+    else
+        ot=&cout;
+
+    while (!dp.EOFile())
+    {
+        dp.getDataSet(D);
+        LOG(INFO)<<"Read "<<D.getSize()<<" entries with "<< D.getfSize()<<" features";
+        rsvm->predict(D,L);
+
+        if (vm.count("validate"))
+        {
+            rank_accu(D,L);
+            if (vm.count("cmc"))
+                rank_CMC(D,L,cmc);
+        }
+
+        if (vm.count("output") || !vm.count("validate"))
+            for (int i=0; i<L.size();++i)
+                *ot<<L[i]<<endl;
+    }
+
+    LOG(INFO)<<"Finished";
+    if (vm.count("cmc"))
+    {
+        LOG(INFO)<< "CMC accounted over " <<cmc.getCount() << " queries";
+        *ot << "CMC"<<endl;
+        vector<double> cur = cmc.getAcc();
+        for (int i = 0;i<CMC_MAX;++i)
+        *ot << cur[i]<<endl;
+    }
+    if (vm.count("output"))
+        fout.close();
+    dp.close();
+    delete rsvm;
+    return 0;
+}
+
+int main(int argc, char **argv) {
+    el::Configurations defaultConf;
+    defaultConf.setToDefault();
+    // Values are always std::string
+    defaultConf.setGlobally(el::ConfigurationType::Format, "%datetime %level %msg");
+
+    // Defining program options
+    po::options_description desc("Allowed options");
+    desc.add_options()
+            ("help,h", "produce help message")
+            ("train,T", "training model")
+            ("validate,V", "validate model")
+            ("predict,P", "use model for prediction")
+            ("cmc,C", "enable cmc auditing")
+            ("debug,d", "show debug messages")
+            ("model,m", po::value<string>(), "set input model file")
+            ("output,o", po::value<string>(), "set output model/prediction file")
+            ("feature,i", po::value<string>(), "set input feature file");
+
+    // Parsing program options
+    po::store(po::parse_command_line(argc, argv, desc), vm);
+    po::notify(vm);
+
+    // Print help if necessary
+    if (vm.count("help") || !(vm.count("train") || vm.count("validate") || vm.count("predict"))) {
+        cout << desc;
+        return 0;
+    }
+
+    if (!vm.count("debug"))
+        defaultConf.setGlobally(el::ConfigurationType::Enabled, "false");
+    // default logger uses default configurations
+    el::Loggers::reconfigureLogger("default", defaultConf);
+
+    mainFunc mainf;
+    if (vm.count("train")) {
+        mainf = &train;
+    }
+    else if (vm.count("validate")||vm.count("predict")) {
+        mainf = &predict;
+    }
+    else return 0;
+    DataProvider* dp;
+    if (vm["feature"].as<string>().find(".rid") == string::npos)
+        dp = new FileDP(vm["feature"].as<string>());
+    else
+        dp = new RidFileDP(vm["feature"].as<string>());
+    mainf(*dp);
+    delete dp;
+    return 0;
+}
-- 
cgit v1.2.3-70-g09d2