/*
data structure of class dataSet:
  private:
	Matrix<int> index;
	Matrix<int>	mult;
	Matrix<int>	ancest;
	int			totCard;
	int			nbDiff;
        int             type;  // output we want: file or screen
	boolean		sorted;
	boolean		i_is_i;

-   *this is derived from an MxN matrix. Each row correspond to one data.
-   INDEX is a row vector of size at least NBDIFF and gives the order of data
    in the data set, the I-th data is in row INDEX(I) of the matrix.
-   MULT and ANCEST are row vectors of size M, MULT(INDEX(I)) gives the
    number of identical data represented by the INDEX(I)-th row,
    ANCEST(INDEX(I)) is the label of one particular data represented by the
    INDEX(I)-th row.
-   TOTCARD gives the total number of data in the data set.
-   NBDIFF gives the number of data that as currently been distinguished.

Invariants:
    *this.m() == mult.n() == ancest.n() >= nbDiff
    index.n() >= nbDiff
    for all 0 <= i < nbDiff : 0 <= index(i) < *this.m()
    totCard = sum_{i=0,i<nbDiff} mult(index(i))
    sorted <=> for all 0 < i < nbDiff : *this(index(i-1)) <=lexi *this(index(i))
    i_is_i <=> for all 0 <=i < nbDiff : index(i)==i
*/



#include "dataSet.h"


#include <string.h>
#include <stdio.h>
#include <fstream.h>
//#include <iomanip.h>



#define _CHECK_INDEX


const char nonMonotonicAttr = 0;
const char positiveAttr = 1;
const char negativeAttr = 2;




//////////////////
// Constructors //
//////////////////


tcT dataSet<T>::dataSet (int nbData, int dim) :
   Matrix<T>(nbData,dim),
   index(nbData), mult(1,nbData,1), ancest(nbData), classcol(0),
   totCard(nbData), nbDiff(nbData), sorted(0), i_is_i(0), monotone()
{ }


tcT dataSet<T>& dataSet<T>::resize(int nbData/**=0**/, int dim/**=0**/)
{
   Matrix<T>::resize(nbData,dim); index.resize(1,nbData);
   mult.resize(1,nbData,1); ancest.resize(1,nbData);
   return *this;
}


tcT /*inline*/ dataSet<T>::dataSet () :
   Matrix<T>(), index(), mult(), ancest(), classcol(0),
   totCard(0), nbDiff(0), sorted(0), i_is_i(0), monotone()
{ }


tcT /*inline*/ dataSet<T>::dataSet (const dataSet<T>& d) :
   Matrix<T>(d.distinct(),d.dim()),
   index(d.distinct()), mult(d.distinct()), ancest(d.distinct()),
   classcol(d.classcol),totCard(d.totCard), nbDiff(d.distinct()), 
   sorted(d.sorted), i_is_i(true), monotone(d.monotone)
{ 
   if ( distinct() && dim() )
      if ( d.i_is_i )
      {
	 s(0,d.distinct(),0,d.dim()) = d.s(0,d.distinct(),0,d.dim());
	 index = d.index.s(0,1,0,d.distinct());
	 mult  = d.mult.s(0,1,0,d.distinct());
	 ancest= d.ancest.s(0,1,0,d.distinct());
      }
      else
	 for (int i=0; i<distinct(); i++)
	 {
	    s(i) = d.s(d.index(i));
	    index(i) = i;
	    mult(i) = d.mult(index(i));
	    ancest(i) = d.ancest(index(i));
	 }
}


tcT /*inline*/ dataSet<T>::~dataSet () { }


tcT dataSet<T>::dataSet
(const Array<T>& d, const Array<int>& multiple, const Array<int>& label,
 const Array<char>& Monotonicity) :
   Matrix<T>(d), index(d.m()), mult(multiple), ancest(label),classcol(0),
   totCard(0), nbDiff(0), sorted(0), i_is_i(0), monotone(Monotonicity)
{

#ifdef _CHECK_INDEX
   if ( d.m()!=multiple.n() || d.m()!=label.n() )
      throw whereDimMismatch(
	 "dataSet<T>::dataSet(const Array<T>&,const Array<int>&,const Array<int>&)",
	 d.m(),d.m(),label.n(),multiple.n());
#endif
   nbDiff = d.m(); i_is_i = 1;
   for (int i=0; i<d.m(); i++) { index(i) = i; totCard += mult(i); }
}


tcT dataSet<T>::dataSet
(const Array<T>& d, const Array<int>& multiple, const Array<int>& label) :
   Matrix<T>(d), index(d.m()), mult(multiple), ancest(label),classcol(0),
   totCard(0), nbDiff(0), sorted(0), i_is_i(0), monotone()
{

#ifdef _CHECK_INDEX
   if ( d.m()!=multiple.n() || d.m()!=label.n() )
      throw whereDimMismatch(
	 "dataSet<T>::dataSet(const Array<T>&,const Array<int>&,const Array<int>&)",
	 d.m(),d.m(),label.n(),multiple.n());
#endif
   nbDiff = d.m(); i_is_i = 1;
   for (int i=0; i<d.m(); i++) { index(i) = i; totCard += mult(i); }
}


tcT dataSet<T>::dataSet (const Array<T>& d, const Array<int>& label) :
   Matrix<T>(d), index(d.m()), mult(1,d.m(),1), ancest(label),classcol(0),
   totCard(0), nbDiff(0), sorted(0), i_is_i(0), monotone()
{
#ifdef _CHECK_INDEX
   if ( d.m()!=label.n() )
      throw whereDimMismatch(
	 "dataSet<T>::dataSet(const Array<T>&,const Array<int>&)",
	 1,d.m(),1,label.n());
#endif
   nbDiff = d.m(); i_is_i = 1;
   for (int i=0; i<d.m(); i++) index(i) = i;
}


//tcT dataSet<T>::dataSet (const Array<T>& d) : Matrix<T>(d)
//{
//index.resize(d.m()); mult.resize(1,d.m(),1); ancest.resize(d.m());
//	totCard = d.m(); nbDiff = d.m(); sorted = 0; i_is_i = 1;
//	for (int i=0; i<d.m(); i++) { index(i) = i; ancest(i) = i; }
//}


tcT dataSet<T>::dataSet (const Array<T>& d, const dataSet<T>& original) :
   Matrix<T>(d), index(d.m()), mult(d.m()), ancest(d.m()),classcol(0),
   totCard(0), nbDiff(0), sorted(0), i_is_i(0), monotone()
{
#ifdef _CHECK_INDEX
   if ( d.m()!=original.distinct() )
      throw whereDimMismatch(
	 "dataSet<T>::dataSet(const Array<T>&,const dataSet<T>&)",
	 1,d.m(),1,original.distinct());
#endif
   for (int i=0; i<d.m(); i++)
   {
      index(i) = i;
      mult(i) = original.mult(original.index(i));
      ancest(i) = original.ancest(original.index(i));
   }
   totCard = original.totCard; nbDiff = d.m(); sorted = 0; i_is_i = 1;
   if ( d.n()==original.dim() )
      monotone.resize(original.monotone);
}


tcT dataSet<T>::dataSet (const dataSet<T>& from, const Array<int>& varIndex) :
   Array<T> (from.distinct(), varIndex.n()),
   Matrix<T>(from.distinct(), varIndex.n()),
   index(from.index), mult(from.mult), ancest(from.ancest),
   classcol(from.classcol), totCard(from.card()), nbDiff(from.distinct()),
   sorted(0), i_is_i(from.i_is_i), monotone()
{
   int newdim = 0;
   for (int varI=0; varI<varIndex.n(); varI++)
      if ( varIndex(varI)>=0 && varIndex(varI)<from.dim() )
	 s(0,from.distinct(),newdim++,1) =
	 from.s(0,from.distinct(),varIndex(varI),1);
   if ( newdim < varIndex.n() ) resize_(newdim);
   if ( from.monotone.n() )
   {
     // check whether classcol in is varIndex
     boolean classinvar=false;
     for (int i=0; i<varIndex.n(); i++)
       if ( varIndex(i)==classcol ) { classinvar=true; break; }
     monotone.resize(newdim-classinvar);
     int ptr=0;
     for (int j=0; j<varIndex.n(); j++)
       if ( varIndex(j)>=0 && varIndex(j)<from.dim() && 
	   varIndex(j)!=classcol )
	 monotone(ptr++) = 
	   from.monotone(varIndex(j)-(classcol<varIndex(j))); 
   }
}


tcT dataSet<T>::dataSet (const dataSet<T>& from, double rate)
{
   if ( rate < 0.0 || rate > 1.0 )
      throw whereOutOfRange(
	 "dataSet<T>::dataSet(dataSet<T>& from, double rate)",
	 "rate",int(rate),0,1);
   totCard = int(rate * double(from.card()));
   int rest = from.card();
   int taken = 0;
   nbDiff = 0;
   Matrix<int> wm(1, from.distinct(), 0);
   for (int row = 0; row < from.distinct(); row++)
      for (int m = 0; m < from.mult(from.index(row)); m++)
      {
	 if ( randomR() < double(totCard-taken)/double(rest) )
	 {
	    taken++;
	    if ( wm(row)++ == 0 ) nbDiff++;
	 }
	 rest--;
      }
   resize(nbDiff, from.dim());
   int ptr = 0;
   for (row = 0; row < wm.n(); row++)
      if ( wm(row) )
      {
	 s(ptr) = from.s(from.index(row));
	 index(ptr) = ptr; mult(ptr) = wm(row);
	 ancest(ptr) = from.ancest(from.index(row));
	 ptr++;
      }
   sorted = from.sorted;
   i_is_i = 1;
   classcol = from.classcol;
   monotone.resize(from.monotone);
}



///////////////
// Selectors //
///////////////

tcT /*inline*/ int dataSet<T>::dim () const { return n(); }

tcT /*inline*/ int dataSet<T>::card () const { return totCard; }

tcT /*inline*/ int dataSet<T>::distinct () const { return nbDiff; }

tcT /*inline*/ int dataSet<T>::classCol () const { return classcol; }



tcT /*inline*/ Matrix<T> dataSet<T>::matrix () const
{
   if ( i_is_i ) return _(0,distinct());
   Matrix<T> res(distinct(),dim());
   for (int r=0; r<distinct(); r++) res.s(r) = s(index(r));
   return res;
}


tcT Array<T> dataSet<T>::operator () (int dIndex) const
{
#ifdef _CHECK_INDEX
   if ( dIndex<0 || dIndex>=distinct() )
      throw whereOutOfRange(
	 "Array<T> dataSet::operator()(int dIndex) const",
	 "dIndex",dIndex,0,distinct()-1);
#endif
   return s(index(dIndex),1,0,dim());
}


tcT Matrix<T> dataSet<T>::operator [] (int attrIndex) const
{
#ifdef _CHECK_INDEX
   if ( attrIndex<0 || attrIndex>=dim() )
      throw whereOutOfRange(
	 "Matrix<T> dataSet::operator[](int attrIndex) const",
	 "attrIndex",attrIndex,0,dim()-1);
#endif
   if ( i_is_i ) return _(0,distinct(),attrIndex,1);
   Matrix<T> res(distinct(),1);
   for (int r=0; r<distinct(); r++)
      res.s(r,1,attrIndex,1) = s(index(r),1,attrIndex,1);
   return res;
}


tcT T& dataSet<T>::operator () (int dIndex, int j) const
{
#ifdef _CHECK_INDEX
   if ( dIndex<0 || dIndex>=distinct() )
      throw whereOutOfRange(
	 "T& dataSet::operator()(int dIndex, int j) const",
	 "dIndex",dIndex,0,distinct()-1);
   if ( j<0 || j>=dim() )
      throw whereOutOfRange(
	 "T& dataSet::operator()(int dIndex, int j) const",
	 "j",j,0,dim()-1);
#endif
   return Matrix<T>::operator()(index(dIndex),j);
}


tcT int dataSet<T>::multiplicity (int dIndex) const
{
#ifdef _CHECK_INDEX
   if ( dIndex<0 || dIndex>=distinct() )
      throw whereOutOfRange(
	 "int dataSet<T>::multiplicity(int dIndex) const",
	 "dIndex",dIndex,0,distinct()-1);
#endif
   return mult(index(dIndex));
}


tcT int dataSet<T>::label (int dIndex) const
{
#ifdef _CHECK_INDEX
   if ( dIndex<0 || dIndex>=distinct() )
      throw whereOutOfRange(
	 "int dataSet<T>::label(int dIndex) const",
	 "dIndex",dIndex,0,distinct()-1);
#endif
   return ancest(index(dIndex));
}



///////////////
// Operators //
///////////////





tcT dataSet<T>& dataSet<T>::operator = (dataSet<T>& from)
{
   if ( m()!=from.m() || n()!=from.n() || index.n()!=from.index.n() )
      resize(from.m(), from.n());
   for (int i=0; i<from.distinct(); i++)
   {
      index(i)=i;
      s(i,1) = from.s(from.index(i),1);
      mult(i) = from.mult(from.index(i));
      ancest(i) = from.ancest(from.index(i));
   }
   totCard=from.totCard; nbDiff=from.nbDiff; sorted=from.sorted; i_is_i=1;
   monotone.resize(from.monotone);
   return *this;
}


tcT dataSet<T> dataSet<T>::operator + (dataSet<T>& d)
{

   if ( dim()!=d.dim() )
      throw whereDimMismatch(
	 "dataSet<T> dataSet<T>::operator+(dataSet<T>&)",
	 1,dim(),1,d.dim());
   dataSet<T> uni(distinct()+d.distinct(), dim());
   for (int i=0; i<distinct(); i++)
   {
      index(i)=i;
      uni.s(i,1) = s(index(i),1);
      uni.mult(i) = mult(index(i));
      uni.ancest(i) = ancest(index(i));
   }
   for (i=0; i<d.distinct(); i++)
   {
      index(i+distinct())=i;
      uni.s(i+distinct(),1) = d.s(d.index(i),1);
      uni.mult(i+distinct()) = d.mult(d.index(i));
      uni.ancest(i+distinct()) = d.ancest(d.index(i));
   }
   uni.totCard=totCard+d.totCard;
   uni.classcol = classcol; uni.sorted=sorted; uni.i_is_i=i_is_i;
   uni.monotone.resize(monotone);
   return uni;
}


tcT boolean identical(const Array<T>& first,const Array<T>& second)
{
   for(int i=0; i< first.m();i++)
      for(int j=0; j < first.n();j++)
	 if(first(i,j)!=second(i,j) && known(first(i,j)) && known(second(i,j)))
	    return(false);
   return(true);
}


tcT dataSet<T> dataSet<T>::operator * (dataSet<T>& d)
{
   int interCard=0;
   if ( dim()==d.dim() )
      for (int left = 0; left < distinct(); left++)
	 for (int right = 0; right < d.distinct(); right++)
	    interCard += identical(s(left),d.s(right));
   dataSet<T> inter(interCard,dim());
   inter.totCard=0; inter.nbDiff=0; inter.i_is_i=true; inter.sorted=sorted;
   inter.monotone.resize(monotone);
   if ( interCard )
   {
      for (int left=0; left<distinct(); left++)
	 for (int right=0; right<d.distinct(); right++)
	    if (identical(s(index(left)),d.s(d.index(right))))
	    {
	       inter.s(inter.nbDiff) = s(index(left));
	       inter.index(inter.nbDiff) = inter.nbDiff;
	       inter.mult(inter.nbDiff) = mult(index(left));
	       inter.ancest(inter.nbDiff) = ancest(index(left));
	       inter.totCard += inter.mult(inter.nbDiff++);
	    }
   }
   return inter;
}


tcT dataSet<T>& dataSet<T>::add(Array<T>& data, int label)
{
   if ( data.n()!=dim() )
      throw whereDimMismatch(
	 "dataSet<T>& dataSet<T>::add(Array<T>&,int)",
	 1,data.n(),1,dim());
   else
   {
      nbDiff++; totCard++; sorted=0;
      if ( nbDiff>m() || !i_is_i )
      {
	 int newM = m()+1;
	 resize_(newM,n()); index.resize_(newM);
	 mult.resize_(newM); ancest.resize_(newM);
      }
      int freePtr = ( i_is_i ? nbDiff-1 : m()-1 );
      s(freePtr,1) = data.s(0,1); index(nbDiff-1) = freePtr;
      mult(freePtr) = 1; ancest(freePtr) = label;
   }
   return *this;
}



tcT dataSet<T>& dataSet<T>::remove(int dIndex)
{
   if ( dIndex>=0 && dIndex<distinct() )
   {
      totCard-=mult(index(dIndex));
      if ( dIndex<distinct()-1 )
      {
	 const int nbToMove = distinct()-dIndex-1;
	 if ( i_is_i )
	 {
	    s(dIndex,nbToMove) =        _(dIndex+1,nbToMove);
	    mult.s(dIndex,nbToMove) =   mult._(dIndex+1,nbToMove);
	    ancest.s(dIndex,nbToMove) = ancest._(dIndex+1,nbToMove);
	 }
	 else
	    index.s(dIndex,nbToMove) = index._(dIndex+1,nbToMove);
      }
      nbDiff--;
   }
   return *this;
}


tcT dataSet<T>& dataSet<T>::checkMult (int& rep)
{
   if ( nbDiff < 2 ) return *this;
   rep = 0;
   int lastNew = 0;	// the first is a fortiori new
   if ( sorted )
   {
      for (int right = 1; right < distinct(); right++)
	 if ( s(index(right-1)) != s(index(right)) )
	 {
	    if ( ++lastNew < right )
	    {
	       index(lastNew) = index(right);
	       i_is_i = 0;
	    }
	 }
	 else
	 {
	    mult(index(lastNew)) += mult(index(right));
	    rep++;
	 }
   }
   else // ( !sorted )
   {
      for (int right = 1; right < distinct(); right++)
      {
	 int isNew = 1;	// let be otpimistic
	 for (int left = 0; (left <= lastNew) && isNew; left++)
	    if ( s(index(left)) == s(index(right)) )
	    {
	       isNew = 0;
	       mult(index(left)) += mult(index(right));
	       rep++;
	    }
	 if ( isNew && (++lastNew < right) )
	 {
	    index(lastNew) = index(right);
	    i_is_i = 0;
	 }
      }
   }
   nbDiff = ++lastNew;
   return *this;
}


tcT dataSet<T>& dataSet<T>::sort ()
{
   if ( nbDiff < 2 ) { sorted=true; return *this; }
   if ( !sorted )
   {
      Matrix<int> ind(index.s(0,1,0,nbDiff));
      Array<T>::sort(ind);
      if ( index.s(0,1,0,nbDiff) != ind )
      {
	 index.s(0,1,0,nbDiff) = ind;
	 i_is_i = 0;
      }
      sorted = 1;
   }
   return *this;
}


tcT dataSet<T>& dataSet<T>::reorganize(boolean reduce/**=true**/)
{
   if ( nbDiff == m() || !reduce )
   {
      Matrix<int> used(nbDiff);
      for (int i=0; i<nbDiff; i++) used(i) = ( index(i)==i );
      int head = 0; int hand;
      Matrix<T> tempRow(1,dim()); int tempMult, tempAncest;
      while ( head < nbDiff && used(head) ) head++;
      while ( head < nbDiff )
      {
	 tempRow    = s     (head);
	 tempMult   = mult  (head);
	 tempAncest = ancest(head);
	 hand = head; used(head) = 1;
	 do
	 {
	    s     (hand) = s     (index(hand));
	    mult  (hand) = mult  (index(hand));
	    ancest(hand) = ancest(index(hand));
	    hand = index(hand);
	    used(hand) = 1;
	 } while ( index(hand) != head );
	 s	  (hand) = tempRow;
	 mult  (hand) = tempMult;
	 ancest(hand) = tempAncest;
	 while ( ++head<nbDiff && used(head) ) {}
      }
      for (i=0; i<nbDiff; i++) index(i) = i;
   }
   else // ( nbDiff < m() )
   {
      Matrix<T> newD(nbDiff,dim());
      Matrix<int> newIndex(1,nbDiff);
      Matrix<int> newMult(1,nbDiff);
      Matrix<int> newAncest(1,nbDiff);
      for (int i=0; i<nbDiff; i++)
      {
	 newD.s(i) = s(index(i));
	 newIndex(i) = i;
	 newMult(i) = mult(index(i));
	 newAncest(i) = ancest(index(i));
      }
      resize(nbDiff,dim());
      s(0,nbDiff,0,dim()) = newD; index = newIndex;
      mult = newMult;	ancest = newAncest;
   }
   i_is_i = 1;
   return *this;
}


tcT void dataSet<T>::split(dataSet<T>& first, dataSet<T>& second, double rate) const
{
   if ( rate < 0.0 || rate > 1.0 )
      throw whereOutOfRange(
	 "void dataSet<T>::split(dataSet<T>&,dataSet<T>&,double rate) const",
	 "rate",int(rate),0,1);
   first.totCard = int(rate * double(card()));
   second.totCard = card() - first.totCard;
   int rest = card();
   int taken = 0;
   first.nbDiff = 0; second.nbDiff = 0;
   Matrix<int> wm(1, distinct(), 0);
   for (int row = 0; row < distinct(); row++)
   {
      for (int m = 0; m < mult(index(row)); m++)
      {
	 if ( randomR() < double(first.totCard-taken)/double(rest) )
	 { taken++; wm(row)++; }
	 rest--;
      }
      if ( wm(row) ) first.nbDiff++;
      if ( wm(row) < mult(index(row)) ) second.nbDiff++;
   }
   first.resize (first.nbDiff, dim());
   second.resize(second.nbDiff, dim());
   int ptrF = 0; int ptrS=0;
   for (row = 0; row < wm.n(); row++)
   {
      if ( wm(row) )
      {
	 first.s(ptrF) = s(index(row));
	 first.index(ptrF) = ptrF;
	 first.mult(ptrF) = wm(row);
	 first.ancest(ptrF) = ancest(index(row));
	 ptrF++;
      }
      if ( wm(row) < mult(index(row)) )
      {
	 second.s(ptrS) = s(index(row));
	 second.index(ptrS) = ptrS;
	 second.mult(ptrS) = mult(index(row))-wm(row);
	 second.ancest(ptrS) = ancest(index(row));
	 ptrS++;
      }
   }
   first.sorted=sorted; second.sorted=sorted;
   first.i_is_i=1; second.i_is_i=1;
   first.monotone.resize(monotone); second.monotone.resize(monotone);
}


// This is new procedure to write the binarized data back into a file for
// later use.     Mario

tcT void operator << (const char* s, const dataSet<T>& myself)
{
   ofstream out_file(s);
   out_file.setf(ios::showpoint);
   if ( uses_dont_know ) 
      out_file << "don't_know " << setw(3) << dont_know() << endl;
   out_file << setw(3) << "label_column  " << 1 << "\n"
	    << setw(3) << "multiplicity_column " << 2 << "\n"
	    << setw(3) << "class_column " << myself.classcol+3 << "\n" ;
   if ( myself.monotone.n() )
   {
      out_file << "Monotonicity";
      for (int i=0; i<myself.monotone.n(); i++)
	 if      ( myself.monotone(i)==negativeAttr )
	    out_file << " \\";
	 else if ( myself.monotone(i)==positiveAttr )
	    out_file << " /";
	 else // ( myself.monotone(i)==nonMonotonicAttr )
	    out_file << " ~";
      out_file << endl;
   }   
   if ( myself.card() )
   {
      const T maxim= myself.max().max();
      const T minim= myself.min().min();
      if (maxim == T(int(maxim)) && minim == T(int(minim)) && 
	  maxim<=T(9) && minim>=T(-9))
      { out_file << coefWidth(2) << coefPrecision(0); }
      else
      { out_file << coefWidth(6) << coefPrecision(2); }
      for (int row = 0; row < myself.distinct(); row++)
	 out_file      << setw(3) << myself.label(row) << "  "
		       << setw(3) << myself.multiplicity(row) << "  "
		       << coefPerLine(0)<< myself(row);
   }
   out_file.close();
}


tcT ostream& operator << (ostream& s, const dataSet<T>& myself)
{
   if ( myself.monotone.n() )
   {
      s << "              ";
      for (int var=0; var<myself.monotone.n(); var++)
	 switch ( myself.monotone(var) )
	 {
	  case positiveAttr: s<<"    /"; break;
	  case negativeAttr: s<<"    \\"; break;
	  default: s<<"    ~";
	 }
      s << endl;
   }
   s.setf(ios::showpoint);
   for (int row = 0; row < myself.distinct(); row++)
      s	<< setw(3) << row+1 << "["
	<< setw(3) << myself.multiplicity(row) << "*"
	<< setw(3) << myself.label(row) << "@] "
	<< coefWidth(5) << coefPrecision(1)
	   << myself(row);
   return s;
}

/////////////////////////////////
// Reading a dataSet on a file //
/////////////////////////////////

boolean eatWhite (ifstream& s) // return true is some white has been read
{
   boolean res=false;
   while ( s.peek()==' ' || s.peek()=='\t'  || s.peek()=='\f' 
	   || s.peek()=='\n' || s.peek()=='\r' ) 
   { s.get(); res=true; }
   return res;
}

boolean eatSpace (ifstream& s) // return true is some space has been read
{
   boolean res=false;
   while ( s.peek()==' ' || s.peek()=='\t' ) 
   { s.get(); res=true; }
   return res;
}

boolean eatLine (ifstream& s) // return true if the line read is non-empty
{
   const int maxLL = 1024; 
   char bidonS[maxLL];
   s.getline(bidonS, maxLL);
   for (int i=0; bidonS[i]!='\0'; i++)
      if ( bidonS[i]!=' ' && bidonS[i]!='\t' ) return true;
   return false;
}

enum dsKeyWord {dontKnow, labelCol, multiCol, classCol, monotonicity,
		factor, offset, noKeyWord};  

dsKeyWord readKeyWord (ifstream& s)
{
   char kw[8][25];
   strcpy(kw[dontKnow],"don't_know\0");
   strcpy(kw[labelCol],"label_column\0");
   strcpy(kw[multiCol],"multiplicity_column\0");
   strcpy(kw[classCol],"class_column\0");
   strcpy(kw[monotonicity],"Monotonicity_constraints\0");
   strcpy(kw[factor],"factors\0");
   strcpy(kw[offset],"offsets\0");
   eatWhite(s);
   for (dsKeyWord i=dontKnow; i<noKeyWord; i=dsKeyWord(int(i)+1))
      if ( s.peek() == kw[i][0] ) break;
   if ( i==noKeyWord ) return noKeyWord;
   s.get();
   for (int j=1; kw[i][j]!='\0'; j++, s.get())
      if ( s.peek()!=kw[i][j] )
	 break;
   if ( eatSpace(s) )
   {
      if ( s.peek()==':' || s.peek()=='=' ) { s.get(); eatSpace(s); }
      return i;
   }
   else
      if ( s.peek()==':' || s.peek()=='=' ) { s.get(); eatSpace(s); return i; }
      else
      {
	 char message[100];
	 sprintf(message,"unknown keyword starting like `%s'", kw[i]);
	 throw whereWhich(
	    "void operator >> (char* fileName, dataSet<T>&)", message);
      }
}


tcT void operator >> (const char* fileName, dataSet<T>& myself)
{
   ifstream in(fileName); int aux_label=1;
   if ( !in )
   {
      char message[100];
      sprintf(message,
	      "void operator >> (char* fileName, dataSet<T>&) , [fileName=%s]",
	      fileName);
      throw whereWhich(message,"File open error");
   }

   // read the header a first time
   int mult=-1; int label=-1; int classC=0;
   boolean monotone=false;
   boolean isfactor=false;
   boolean isoffset=false;
   dsKeyWord kwd;
   while ( (kwd=readKeyWord(in))!=noKeyWord )
   {
      switch( kwd )
      {
       case dontKnow: T dn; in >> dn; set_dont_know(dn); break;
       case labelCol: in >> label; --label; break;
       case multiCol: in >> mult; --mult; break;
       case classCol: in >> classC; --classC; break;
       case monotonicity: monotone=true; eatLine(in); break;
       case factor: isfactor=true; eatLine(in); break;
       case offset: isoffset=true; eatLine(in); break;
      }
   }

   // count the number of elements in a row
   int dim=0; double bidon;
   do
   {
      in >> bidon; dim++;
      eatSpace(in);
      if ( in.peek()==',' || in.peek()==';' ) { in.get(); eatSpace(in); }
   } while ( in.peek()!='\n' && in.peek()!='\f' && in.peek()!=EOF );

   // count the number of rows
   eatLine(in);
   boolean whiteLine=false; int nbData=1;
   while ( !in.eof() && !whiteLine ) 
   {
      whiteLine = !eatLine(in);
      if ( !whiteLine ) nbData++;
   }
   in.close();

   // adjust classcol according to the existence  of other columns
   {
      myself.classcol=classC;
      if (label>-1 && label<classC) myself.classcol--;
      if (mult >-1 && mult <classC) myself.classcol--;
   }

   // update the dataSet and prepare the second reading
   int aux_dim=dim;
   if (mult >-1) --dim; if (label >-1) --dim;
   if (monotone) myself.monotone.resize(dim-1);
   Matrix<double> myfactor, myoffset;
   if (isfactor) myfactor.resize(dim-1);
   if (isoffset) myoffset.resize(dim-1);
   myself.resize(nbData, dim); myself.totCard = nbData;
   myself.nbDiff = nbData;
   myself.sorted = 0; myself.i_is_i=1;

   // read the header a second time
   in.open(fileName,ios::in);
   while ( (kwd=readKeyWord(in))!=noKeyWord )
   {
      switch( kwd )
      {
       case dontKnow: 
       case labelCol: 
       case multiCol: 
       case classCol: in >> bidon; break;
       case factor  : in >> myfactor; break;
       case offset  : in >> myoffset; break;
       case monotonicity: 
	 for (int k=0; k<dim-1; k++)
	 {
	    switch(in.peek())
	    {
	     case '~': myself.monotone(k)=nonMonotonicAttr; break;
	     case '/': myself.monotone(k)=positiveAttr; break;
	     case '\\':myself.monotone(k)=negativeAttr; break;
	     default:
	       char message[100];
	       sprintf(message,
		"void operator >> (char* fileName, dataSet<T>&) , [fileName=%s]",
		    fileName);
	       throw whereWhich(message,"Syntax error: (/, \\ or ~ expected)");
	    }
	    in.get(); eatSpace(in);
	 }
	 break;
      }
   }

   for (int i=0; i<nbData; i++)
   {
      myself.index(i)=i;
      if (label<0) myself.ancest(i)= i+1;
      int attr=0; int attrEff=0;
      for(int j=0; j<aux_dim; j++)
      {
	 if      (j==label)  {in >> myself.ancest(i);}
	 else if (j==mult)   {in >> myself.mult(i);  }
	 else if (j==classC) {in >> myself(i,attr++);}
	 else
	 {
	    in >> bidon;
	    if ( isfactor ) bidon *= myfactor(attrEff);
	    if ( isoffset ) bidon += myoffset(attrEff);
	    myself(i,attr++) = T(bidon); 
	    attrEff++;
	 }
	 if (!in)
	 {
	    char message[100];
	    sprintf(message,
		 "void operator >> (char* fileName, dataSet<T>&) , [fileName=%s]",
		    fileName);
	    throw whereWhich(message,"Syntax error in file");
	 }
      }
   }
   in.close();
}