// bin.C



#include <stdio.h>
#include <math.h>
#include <iostream.h>
#include <strstream.h>


#include "../Basics/pipeostream.h"
#include "../Basics/basic.h"
#include "../Basics/chrono.h"
#include "../Basics/exception.h"
#include "../Matrices/Matrix.h"
#include "../Matrices/binMatrix.h"
#include "main_def.h"
#include "multiDS.h"
#include "cutPtsSet.h"





int main () { try
{
   //-------------------------------------------
   //Collect general information for the batch
   //-------------------------------------------
   
   char inFile[80], outFile[80], choice[80]; 

   // Declaration of all parameters, with default values
   int Kfold=0;
   float percFrom=10.0; 
   float percTo=10.0; 
   float percInter=1.0; 
   int nbIter=1;
   int nbSizes=1; 
   int initSeed=0; 
   float confi=0.0; 
   int binMeasure=1; 
   int CPgeneration=0; 
   int CPOrdering=0; 
   boolean filter=false; 
   float filterLimit=1.0;
   boolean minimizeCP=true; 
   float separability=1.0; 
   boolean compress=false;

#if _batch_mode
   trace=0;
#else
   cout<<"Trace level (1=normal, 2=debug): ";
   cin>>trace; 
#endif
   
   cout.setf(ios::fixed | ios::showpoint);

   cout<<"Prefix XXX of the file (.all) with original data : "; 
   cin>>inFile; 
   cout<<"Prefix XXX for the files (.tra, .tes) with binarized data : "; 
   cin>>outFile;
   sprintf(choice,"%s-bin.out",outFile);
   ofstream fout(choice);
   fout<<"Prefix XXX of the file (.all) with original data : "
       <<inFile<<endl;
   fout<<"Prefix XXX for the files (.tra, .tes) with binarized data : "
       <<outFile<<endl; 
   sprintf(choice,"%s-bin.log",outFile); 
   flog.open(choice);
   sprintf(inFile,"%s.all",inFile);

   cout<<(fout<<=
     "Size K of the K-folding (enter 1 for independent random experiments) : ");
   cin>>Kfold; fout<<setw(2)<<Kfold<<endl;

   if ( Kfold>=-1 && Kfold<=1 )
   {
      Kfold=1;
      cout<<(fout<<="Training set's size (in %) from: "); 
      cin>>percFrom; fout<<setprecision(2)<<setw(6)<<percFrom<<endl;
      
      percTo = percFrom;
      if ( percFrom < 100.0 )
      {
	 cout<<(fout<<="Training set's size (in %) to  : "); 
	 cin>>percTo; fout<<setprecision(2)<<setw(6)<<percTo<<endl;
      }
      if ( percTo != percFrom )
      {
	 cout<<(fout<<="Interval in training set's size (in %): "); 
	 cin>>percInter; fout<<setprecision(2)<<setw(6)<<percInter<<endl;
      }
      if ( percFrom < 100.0 )
      {
	 cout<<(fout<<="#iterations of each experimentation: "); 
	 cin>>nbIter; fout<<setw(4)<<nbIter<<endl;
      }
   }
   else
   {
      percFrom=(percTo=(Kfold>1 ? 100.0-100.0/Kfold : 100.0/(-Kfold)));
      cout<<"#iterations of each " << absolute(Kfold) << "-folding : ";
      fout<<"#iterations of each " << absolute(Kfold) << "-folding : ";
      cin>>nbIter; fout<<setw(4)<<nbIter<<endl;
   }
   if ( percFrom < 100.0 )
   {
      cout<<(fout<<="Seed: ");
      cin>>initSeed; fout<<setw(8)<<initSeed<<endl;
   }

   nbSizes = 1 + int((percTo - percFrom)/percInter);
   const boolean singleExp = ((nbSizes * nbIter * absolute(Kfold)) == 1);

   cout<<(fout<<="Binarization method based on (1)binary, (2)continuous ");
   cout<<(fout<<="separability measure, (3) maps: ");
   cin>>binMeasure; 
   fout<<binMeasure<<endl;
   boolean showDistr=false;
   if ( binMeasure<0 ) { binMeasure=-binMeasure; showDistr=singleExp; }  

   if ( binMeasure<3 )
   {
      cout<<(fout<<="Apply point-in-a-box to reduce the # of pairs of ");
      cout<<(fout<<="points considered (y/n): ");
      cin>>choice; fout<<choice<<endl; compress=(choice[0]=='y');

      cout<<(fout<<="Confidence interval around each cut point (0..0.1): ");
      cin>>confi; fout<<setprecision(4)<<setw(6)<<confi<<endl;

      if ( binMeasure==2 )
      {
	 cout<<(fout<<=
	    "Cut points generation method (0=each change, 1=each pair): ");
	 cin>>CPgeneration; fout<<CPgeneration<<endl;
      }

      cout<<(fout<<="Filter cut points according to a specific order (y/n): ");
      cin>>choice; fout<<choice<<endl; filter=(choice[0]=='y'); 

      if ( filter )
      {
	 cout<<(fout<<="Ordering method for cut points (1=correl with ");
	 cout<<(fout<<="output, 2=gap, 3=global sep): ");
	 cin>>CPOrdering; fout<<CPOrdering<<endl;

	 if ( !showDistr )
	 {
	    if ( binMeasure==0 || binMeasure==1 ) 
	       cout<<(fout<<="Minimal # of CP separating each pair of pts");
	    else if ( binMeasure==2 ) 
	       cout<<(fout<<="Minimal separability of each pair of points");
	    cout<<(fout<<=" (filter limit): ");
	    cin>>filterLimit; 
	    fout<<setprecision(3)<<setw(6)<<filterLimit<<endl;
	 }
      }

      cout<<(fout<<="Minimize # of cut points (y/n): ");
      cin>>choice; fout<<choice<<endl; minimizeCP=(choice[0]=='y'); 

      if ( !showDistr && minimizeCP )
      {
	 if ( binMeasure==1 )
	    cout<<(fout<<="Minimal # of CP separating each pair of pts");
	 else if ( binMeasure==2 )
	    cout<<(fout<<="Minimal separability of each pair of points");
	 cout<<(fout<<=" (optimization): ");
	 cin>>separability; 
	 fout<<setprecision(3)<<setw(6)<<separability<<endl;
      }
   }

   //---------------
   // Set seed Table
   //---------------
   Matrix<int> seed(nbSizes,nbIter);
   fixSeed(initSeed);
   if(singleExp) seed(0,0)= initSeed;
   else
      for (int size=0; size<nbSizes; size++)
	 for (int iter=0; iter<nbIter; iter++)
	    seed(size,iter) = int(1E6 * randomR());

   multiDS<ST> whole, training, testing;
   Matrix<int> Kparts;


   //----------------------------------------------------------
   //Read the data, reorganize them and check for consistancy
   //----------------------------------------------------------
   dataSet<ST> wholeDS; 
   inFile>>wholeDS;
   whole.separate(wholeDS);
   whole.sort().checkMult().reorganize();  // Added
   if (!whole.isConsistant()) return 0;
   cout<<(fout<<="\nFile: ");
   cout<<(fout<<=inFile); cout<<(fout<<=", "); 
   cout<<(fout<<=whole.ds[1].card()); cout<<(fout<<=" positive points (");
   cout<<(fout<<=whole.ds[1].distinct()); cout<<(fout<<=" different),\n\t");
   cout<<(fout<<=whole.ds[0].card()); cout<<(fout<<=" negative points (");
   cout<<(fout<<=whole.ds[0].distinct()); cout<<(fout<<=" different), ");
   cout<<(fout<<=whole.ds[0].dim()); cout<<(fout<<=" attributes.\n\n");

   int cSize; float perc;
   for (perc=percFrom, cSize=0; perc<=percTo; perc+=percInter, cSize++)
   {
      Matrix<double> stat(1,8,0.0), statS(1,8,0.0), statSSQ(1,8,0.0),
	             statMin(1,8,+1E10), statMax(1,8,-1E10);
      int iterXfold=0;
    
      flog<<endl<<perc<<"% of data used for training"<<"\n"<<flush;
    
      // loop on iterations
      for (int Iter=0; Iter<nbIter && (Iter<1 || perc<100.0); Iter++)
      {
	 // print some heading titles
	 if ( nbIter>10 ? Iter%20==0 : (!Iter && (!cSize || 
						  (((cSize-1)*nbSizes)/20 !=
						  (cSize*nbSizes)/20))) )
	 {
	    cout<<(fout<<=
		   "Perc-  |#-cut-pts| |train-sz--and-test-sz| |min| |bin|\n");
	    cout<<(fout<<=
		   " -Iter |gen-final| |difP-difN---difP-difN| |err| |tim|\n");
	 }
	 else if ( Kfold!=1 )
	    cout<<(fout<<="\n");

	 fixSeed(seed(cSize, Iter));
	 flog<<seed(cSize, Iter)<<" "<<flush;

	 // divide the data set into training and testing
	 if ( Kfold==1 )
	 {
	    whole.split(training,testing,perc/100.0);
	    training.sort().checkMult().reorganize();
	 }
	 else
	    whole.partition(Kparts,absolute(Kfold));

	 for (int folding=1; folding<=absolute(Kfold); folding++)
	 {
	    iterXfold++;
	    int ii=0;
	    float separL=separability; 
	    float confiL=confi;

	    // Determine the training set and the data set
	    if ( Kfold!=1 )
	    {
	       whole.setFold(training,Kparts,-1*sgn(Kfold)*folding);
	       whole.setFold( testing,Kparts,   sgn(Kfold)*folding);
	    }
	    if ( trace>1 ) cout<<"\n*** training data:\n"<<training;

	    // start the binarization
	    chrono binarization;

	    if ( binMeasure<3 )
	    {
	       // generate the list of pairs of points in different classes
	       training.listOfPairs(compress,true);
	    
	       // generate the candidate cut points
	       cutPtsSet<ST> candidate(training,confiL,CPgeneration,CPOrdering);
	       if ( trace>1 )
		  cout<<"\n*** candidate cut points:\n"<<candidate;
	       stat(ii++) = candidate.nbCP();
	       if ( binMeasure==1 && confi==0.0 ) 
	       {
		  candidate.normalize(training);
		  if ( trace>1 )
		     cout<<"\n*** normalized training data:\n"<<training
			 <<"\n*** normalized cut points:\n"<<candidate;
	       }
	    
	       // Show the distribution of the weights and ask for the 
	       // parameters filterLimit and separability
	       if ( showDistr )
	       {
		  float garbage=0.0;
		  candidate.reduce(training, -binMeasure-1,garbage);
		  if ( filter )
		  {
		     if ( binMeasure==1 ) 
			cout<<"Minimal # of cut points separating each "
			    <<"pair of points";
		     else if ( binMeasure==2 )
			cout<<"Minimal separability of each pair of points";
		     cout<<" (filter limit): ";
		     cin>>filterLimit;
		  }
		  if ( binMeasure==1 )
		     cout<<"Minimal # of cut points separating each pair "
			 <<"of points";
		  else
		     cout<<"Minimal separability of each pair of points";
		  cout<<" (optimization): ";
		  cin>>separability; 
		  separL=separability;
	       }
	    
	       if ( filter ) // keep only the first candidates cut points
	       {
		  candidate.reduce(training,binMeasure-1,filterLimit,confiL);
		  if ( trace>1 )
		     cout<<"\n*** remaining cut points after filter:\n"
			 <<candidate;
	       }
	    
	       // extract a small number of cut points from the candidate
	       if ( minimizeCP )
	       {  // !!
		  candidate.reduce(training,binMeasure+1,separL,confiL);
		  if ( trace>1 )
		     cout<<"\n*** remaining cut points after optimization:\n"
			 <<candidate;
	       }
	       stat(ii++) = candidate.nbCP();
	    
	       // binarize the original data
	       if ( confiL==0.0 )
	       {
		  training.binarize(candidate);
		  testing.binarize(candidate);
	       }
	       else
	       {
		  training.binarize(candidate,confiL);
		  testing.binarize(candidate,confiL);
	       }
	    
	       // Save the cut points
	       if ( singleExp )
	       {
		  char fn[80];
		  sprintf(fn,"%s.cutPts",outFile);
		  ofstream cutPts_file(fn);
		  cutPts_file<<candidate;
	       } 
	    }
	    else // binMeasure>=3
	    {
	       Matrix<float> hcn;
	       cout<<"Size of the hypercube (2..20): ";
	       int hcSize; cin>>hcSize;
	       training.mapCube(hcn,hcSize);
	       training.binarize(hcn);
	       testing.binarize(hcn);
	       stat(ii++) = training.dim();
	    }

	    if ( trace>1 )
	       cout<<"\n*** binarized training data:\n"<<training;
	    stat(ii++) = training.ds[1].distinct();
	    stat(ii++) = training.ds[0].distinct();
	    stat(ii++) = testing.ds[1].distinct();
	    stat(ii++) = testing.ds[0].distinct();
	    stat(ii++) = testing.boundOnErrors(training);
	    stat(ii++) = binarization.time();
	 
	    // Save the binary files
	    char pref[80]; 
	    ostrstream buff(pref,80);
	    buff<<outFile<<"-"<<setw(2)<<setfill('0')<<int(perc)
	       <<"%"<<setw(3)<<iterXfold<<ends;
	    char fn[80];
	    sprintf(fn,"%s.tra",pref); fn<<training.merge();
	    sprintf(fn,"%s.tes",pref); fn<<testing.merge();

	    // Print the results
	    cout<<setfill('0')<<setw(2)<<int(perc)<<"%"
		<<setw(3)<<iterXfold<<setfill(' ')
		<<coefWidth(6)<<coefPrecision(-1)<<stat<<flush; 

	    fout<<setfill('0')<<setw(2)<<int(perc)<<"%"
 	        <<setw(3)<<iterXfold<<setfill(' ')
	        <<coefWidth(6)<<coefPrecision(-1)<<stat<<flush;

	    // Update the stat
	    statS   += stat;
	    statSSQ += stat.map(&Sqr);
	    statMin  = (stat ^ statMin).minC();
	    statMax  = (stat ^ statMax).maxC();
	 }
      }

      // Print the statistics
      if (nbIter*absolute(Kfold)>1)
      {
	 cout<<"  mean";
	 coefWidth(cout,6);	 
	 coefPrecision(cout,1);
	 cout<<statS/(nbIter*absolute(Kfold))
	     <<"   std"<<((statSSQ-(statS.map(&Sqr))/
			   (double(nbIter*absolute(Kfold))))/
			  (double(nbIter*absolute(Kfold)-1))).map(&SQRT)
	     <<"   min";
	 coefPrecision(cout,-1)<<statMin<<"   Max"<<statMax<<endl;
	 fout<<"  mean";
	 coefWidth(fout,6);
	 coefPrecision(fout,1)<<statS/(nbIter*absolute(Kfold))
	     <<"   std"<<((statSSQ-(statS.map(&Sqr))/
			   (double(nbIter*absolute(Kfold))))/
			  (double(nbIter*absolute(Kfold)-1))).map(&SQRT)
	     <<"   min";
	 coefPrecision(fout,-1)<<statMin<<"   Max"<<statMax<<endl;
      }
   }

   return 0;
}
catch (whereWhich e){ cout<<e; flog<<e; }
catch (whereOutOfRange e){ cout<<e; flog<<e; }
catch (whereDimMismatch e){ cout<<e; flog<<e; }
catch (nullArray e){ cout<<e; flog<<e; }
catch (whereException& e){ cout<<e; flog<<e; }
}