MAClassifier.cpp

/*
 *  This file is part of the AiBO+ project
 *
 *  Copyright (C) 2005-2016 Csaba Kertész (csaba.kertesz@gmail.com)
 *
 *  AiBO+ is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  AiBO+ is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Street #330, Boston, MA 02111-1307, USA.
 *
 */

#include "MAClassifier.hpp"

#include "MAClassifierPrivate.hpp"
#include "MAModel.hpp"

#include <3rdparty/archives/portable_iarchive.hpp>
#include <3rdparty/archives/portable_oarchive.hpp>
#include <MCBinaryData.hpp>
#include <MCContainers.hpp>
#include <MCDataContainer.hpp>
#include <MCLog.hpp>
#include <MCSampleStatistics.hpp>

#include <opencv/cv.h>
#include <opencv2/core/core.hpp>

#if __cplusplus >= 201103L && defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-register"
#endif
#include <Eigen/Dense>
#if __cplusplus >= 201103L && defined(__clang__)
#pragma GCC diagnostic pop
#endif
#include <serialize.h>
#include <maxent.h>
#include <lwpr.hh>
#if defined(__unix__)
#include <gp.h>
#include <gp_utils.h>
#endif
#include <pls.h>

#include <boost/archive/binary_oarchive.hpp>
#include <boost/archive/binary_iarchive.hpp>
#include <boost/iostreams/device/back_inserter.hpp>
#include <boost/iostreams/stream.hpp>
#include <boost/serialization/vector.hpp>

namespace
{
class InitOpenCVRNG
{
public:
  InitOpenCVRNG()
  {
    Reset();
  }

  void Reset()
  {
    // Note: Need to reset the random seed before doing anything with OpenCV's ML module otherwise
    // inconsistent results are calculated (at least with train_auto() in SVM)
    cv::theRNG().state = MCRand(0, 10000000);
  }
};

static InitOpenCVRNG InitOpenCVRNGWrapper;
}

#if defined(__unix__)
class GpWrapper
{
public:
  boost::scoped_ptr<libgp::GaussianProcess> GprScPtr;
};
#else
class GpWrapper
{
};
#endif


MAClassifier::MAClassifier() : Classifier(MA::DecisionTreeClassifier),
  Preprocessing(MA::FeatureStandardization), Regression(false), Trained(false), FeatureCount(0),
  LabelCount(1), SvmGamma(0.00001), SvmC(0.1), SvmP(0.01), SvmNu(0.0001), SvmAutoTrain(false),
  DlibEpsilon(0.00001), RvmSigmoidGamma(0.00001), RvmRbfGamma(0.0002),
  KrrGamma(1.5), KrrLambda(0.01), MeL1(0.00001), MeL2(0.005), LwprAlpha(0.2), PlsrComponents(0),
  KrlsTolerance(0.001), KrlsGamma(0.0002), TreeMaxDepth(20), TreeNodeSampleLimit(1),
  RtMaxForestSize(30)
{
}


MAClassifier::MAClassifier(MA::CRMethodType method, int label_count, bool regression) :
  Classifier(method), Preprocessing(MA::FeatureStandardization), Regression(regression),
  Trained(false), FeatureCount(0), LabelCount(label_count <= 0 ? 1 : label_count),
  SvmGamma(0.00001), SvmC(0.1), SvmP(1), SvmNu(0.0001), SvmAutoTrain(false),
  DlibEpsilon(0.00001), RvmSigmoidGamma(0.00001), RvmRbfGamma(0.0002), KrrGamma(1.5),
  KrrLambda(0.01), MeL1(0.00001), MeL2(0.005), LwprAlpha(0.2), PlsrComponents(0),
  KrlsTolerance(0.001), KrlsGamma(0.0002), TreeMaxDepth(20), TreeNodeSampleLimit(1),
  RtMaxForestSize(30)
{
  CreateClassifier();
}


MAClassifier::~MAClassifier()
{
}


bool MAClassifier::IsValid() const
{
  return !CachedSamples.empty();
}


MA::CRMethodType MAClassifier::GetMethodType() const
{
  return Classifier;
}


bool MAClassifier::IsRegression() const
{
  return Regression;
}


unsigned int MAClassifier::GetFeatureVectorSize() const
{
  return CachedSamples.size() == 0 ? 0 : CachedSamples[0].size();
}


MC::FloatTable& MAClassifier::GetFeatureVectors()
{
  return CachedSamples;
}


MC::FloatList& MAClassifier::GetLabels()
{
  return CachedLabels;
}


MC::FloatList MAClassifier::GetModelLabels() const
{
  MC::FloatList Vector(CachedUniqueLabels.begin(), CachedUniqueLabels.end());

  return Vector;
}


void MAClassifier::PrioritizeClasses(const MC::FloatList& labels)
{
  PrioritizedClasses = labels;
}


void MAClassifier::Reset()
{
  PreprocessedData.clear();
  CachedSamples.clear();
  CachedLabels.clear();
  CachedUniqueLabels.clear();
  if (Classifier == MA::NeuralNetworkClassifier)
  {
    NeuralNetwork->clear();
  } else
  if (Classifier == MA::NaiveBayesClassifier)
  {
    BayesClassifier->clear();
  } else
  if (Classifier == MA::KNearestNeighborClassifier)
  {
    KNearestClassifier->clear();
  } else
  if (Classifier >= MA::SvmClassifierCLinear && Classifier <= MA::SvmClassifierNuRbf)
  {
    SvmClassifier->clear();
    SvmClassifierParams->C = SvmC;
    SvmClassifierParams->p = SvmP;
    SvmClassifierParams->nu = SvmNu;
    SvmClassifierParams->gamma = SvmGamma;
  } else
  if (Classifier == MA::DecisionTreeClassifier)
  {
    DecisionTree->clear();
  } else
  if (Classifier == MA::RandomTreesClassifier)
  {
    RandomTrees->clear();
  } else
  if (Classifier == MA::ExtremeRandomTreesClassifier)
  {
    ExtremeRandomTrees->clear();
  } else
  if (Classifier == MA::GradientBoostedTreesClassifier)
  {
    GradientBoostedTrees->clear();
  } else
  if (Classifier == MA::ExpectationMaximization)
  {
    EmClassifier->Classifier->clear();
  } else
  if ((Classifier >= MA::SvmClassifierCLinearEkm && Classifier <= MA::SvmClassifierCLinearDlib) ||
      (Classifier >= MA::RvmClassifierLinear && Classifier <= MA::KrRegressionRbf) ||
      (Classifier >= MA::KrlsLinearRegression && Classifier <= MA::KrlsRbfRegression))
  {
    DlibFunctions.reset(nullptr);
  } else
  if (Classifier == MA::MaxEntropyClassifierL1 || Classifier == MA::MaxEntropyClassifierL2)
  {
    MaxEntropy.reset(nullptr);
  } else
  if (Classifier == MA::LwpRegression)
  {
    Lwpr.reset(nullptr);
  } else
#if defined(__unix__)
  if (Classifier == MA::GaussianProcessRegression)
  {
    Gpr->GprScPtr.reset(nullptr);
  } else
#endif
  if (Classifier == MA::PlsRegression)
  {
    Plsr.reset(nullptr);
    PlsrComponents = 0;
  }
  Trained = false;
  FeatureCount = 0;
}


void MAClassifier::SetParameter(MA::CRMethodParamType method_parameter, float value)
{
  if (Trained)
  {
    MC_WARNING("Custom classifier parameter can't be set after training.");
    return;
  }
  if (method_parameter == MA::SvmGammaParam)
  {
    SvmGamma = value;
    if (SvmClassifierParams.get())
      SvmClassifierParams->gamma = SvmGamma;
  } else
  if (method_parameter == MA::SvmCParam)
  {
    SvmC = value;
    if (SvmClassifierParams.get())
      SvmClassifierParams->C = SvmC;
  } else
  if (method_parameter == MA::SvmPParam)
  {
    SvmP = value;
    if (SvmClassifierParams.get())
      SvmClassifierParams->p = SvmP;
  } else
  if (method_parameter == MA::SvmNuParam)
  {
    SvmNu = value;
    if (SvmClassifierParams.get())
      SvmClassifierParams->nu = SvmNu;
  } else
  if (method_parameter == MA::SvmAutoTrainParam)
  {
    SvmAutoTrain = (bool)value;
  } else
  if (method_parameter == MA::DlibEpsilonParam)
  {
    DlibEpsilon = value;
  } else
  if (method_parameter == MA::RvmSigmoidGammaParam)
  {
    RvmSigmoidGamma = value;
  } else
  if (method_parameter == MA::RvmRbfGammaParam)
  {
    RvmRbfGamma = value;
  } else
  if (method_parameter == MA::KrrRbfGammaParam)
  {
    KrrGamma = value;
  } else
  if (method_parameter == MA::KrrLambdaParam)
  {
    KrrLambda = value;
  } else
  if (method_parameter == MA::MeL1Param)
  {
    MeL1 = value;
  } else
  if (method_parameter == MA::MeL2Param)
  {
    MeL2 = value;
  } else
  if (method_parameter == MA::LwprAlphaParam)
  {
    LwprAlpha = value;
  } else
  if (method_parameter == MA::PlsrComponentsParam)
  {
    PlsrComponents = (int)value;
    if (PlsrComponents < 0)
      PlsrComponents = 0;
  } else
  if (method_parameter == MA::KrlsToleranceParam)
  {
    KrlsTolerance = value;
  } else
  if (method_parameter == MA::KrlsGammaParam)
  {
    KrlsGamma = value;
  } else
  if (method_parameter == MA::TreeMaxDepthParam)
  {
    TreeMaxDepth = (int)value;
    if (TreeMaxDepth < 0)
      TreeMaxDepth = 20;
  } else
  if (method_parameter == MA::TreeNodeSampleLimitParam)
  {
    TreeNodeSampleLimit = (int)value;
    if (TreeNodeSampleLimit < 0)
      TreeNodeSampleLimit = 1;
  } else
  if (method_parameter == MA::RtMaxForestSizeParam)
  {
    RtMaxForestSize = (int)value;
    if (RtMaxForestSize < 0)
      RtMaxForestSize = 20;
  }
}


float MAClassifier::GetParameter(MA::CRMethodParamType method_parameter)
{
  if (method_parameter == MA::SvmGammaParam)
    return SvmGamma;
  else
  if (method_parameter == MA::SvmCParam)
    return SvmC;
  else
  if (method_parameter == MA::SvmPParam)
    return SvmP;
  else
  if (method_parameter == MA::SvmNuParam)
    return SvmNu;
  else
  if (method_parameter == MA::SvmAutoTrainParam)
    return (float)SvmAutoTrain;
  else
  if (method_parameter == MA::DlibEpsilonParam)
    return DlibEpsilon;
  else
  if (method_parameter == MA::RvmSigmoidGammaParam)
    return RvmSigmoidGamma;
  else
  if (method_parameter == MA::RvmRbfGammaParam)
    return RvmRbfGamma;
  else
  if (method_parameter == MA::KrrRbfGammaParam)
    return KrrGamma;
  else
  if (method_parameter == MA::KrrLambdaParam)
    return KrrLambda;
  else
  if (method_parameter == MA::MeL1Param)
    return MeL1;
  else
  if (method_parameter == MA::MeL2Param)
    return MeL2;
  else
  if (method_parameter == MA::LwprAlphaParam)
    return LwprAlpha;
  else
  if (method_parameter == MA::PlsrComponentsParam)
    return (float)PlsrComponents;
  else
  if (method_parameter == MA::KrlsToleranceParam)
    return KrlsTolerance;
  else
  if (method_parameter == MA::KrlsGammaParam)
    return KrlsGamma;
  else
  if (method_parameter == MA::TreeMaxDepthParam)
    return (float)TreeMaxDepth;
  else
  if (method_parameter == MA::TreeNodeSampleLimitParam)
    return (float)TreeNodeSampleLimit;
  else
  if (method_parameter == MA::RtMaxForestSizeParam)
    return (float)RtMaxForestSize;

  return MCFloatInfinity();
}


void MAClassifier::SetPreprocessingMode(MA::FeaturePreprocessingType preprocessing_mode)
{
  if (Trained)
  {
    MC_WARNING("Preprocessing mode can't be changed after training.");
    return;
  }
  Preprocessing = preprocessing_mode;
}


void MAClassifier::AddSamples(const MC::FloatTable& input_vectors, const MC::FloatList& labels)
{
  if (input_vectors.empty() || input_vectors[0].size() == 0)
  {
    MC_WARNING("No data in the input table for classifier training.");
    return;
  }
  // Check if the table is appropriate for training
  int RowSize = input_vectors[0].size();

  for (unsigned int i = 1; i < input_vectors.size(); ++i)
  {
    if ((int)input_vectors[i].size() != RowSize)
    {
      MC_WARNING("Row counts are inconsistent in the input table for classifier training.");
      return;
    }
    if (FeatureCount != 0 && input_vectors[i].size() != FeatureCount)
    {
      MC_WARNING("Feature count does not match with the expected training vector size (%d != %d).",
                 FeatureCount, input_vectors[i].size());
      return;
    }
  }
  if (input_vectors.size() != labels.size())
  {
    MC_WARNING("Label counts are inconsistent with the input table size for classifier training.");
    return;
  }
  // Remember for the feature count
  if (unlikely(FeatureCount == 0))
  {
    FeatureCount = RowSize;
  }
  MCMergeContainers(CachedSamples, input_vectors);
  MCMergeContainers(CachedLabels, labels);
  CachedUniqueLabels.insert(labels.begin(), labels.end());
  Trained = false;
  if ((Classifier >= MA::SvmClassifierCLinearEkm && Classifier <= MA::SvmClassifierCLinearDlib) ||
      (Classifier >= MA::RvmClassifierLinear && Classifier <= MA::KrRegressionRbf) ||
      (Classifier >= MA::KrlsLinearRegression && Classifier <= MA::KrlsRbfRegression))
  {
    DlibFunctions.reset(nullptr);
    return;
  } else
  if (Classifier == MA::MaxEntropyClassifierL1 || Classifier == MA::MaxEntropyClassifierL2)
  {
    MaxEntropy.reset(nullptr);
    return;
  } else
  if (Classifier == MA::LwpRegression)
  {
    Lwpr.reset(nullptr);
    return;
  } else
#if defined(__unix__)
  if (Classifier == MA::GaussianProcessRegression)
  {
    Gpr->GprScPtr.reset(nullptr);
    return;
  } else
#endif
  if (Classifier == MA::PlsRegression)
  {
    Plsr.reset(nullptr);
    return;
  }
}


MC::FloatList MAClassifier::Predict(const MC::FloatTable& input_vectors, MC::FloatList& confidences)
{
  if (input_vectors.empty())
  {
    MC_WARNING("No data in the input table for classifier prediction.");
    return MC::FloatList();
  }
  // Check if the table is appropriate for training
  int RowCount = input_vectors[0].size();

  if (RowCount == 0)
  {
    MC_WARNING("No data in the input table for classifier prediction.");
    return MC::FloatList();
  }
  for (unsigned int i = 1; i < input_vectors.size(); ++i)
  {
    if ((int)input_vectors[i].size() != RowCount)
    {
      MC_WARNING("Row counts are inconsistent in the input table for classifier prediction.");
      return MC::FloatList();
    }
    if (FeatureCount != 0 && input_vectors[i].size() != FeatureCount)
    {
      MC_WARNING("Feature count does not match with the expected (%d != %d).", FeatureCount,
                 input_vectors[i].size());
      return MC::FloatList();
    }
  }
  MC::FloatList Labels;

  for (unsigned int i = 0; i < input_vectors.size(); ++i)
  {
    Labels.push_back(Predict(input_vectors[i], confidences));
  }
  return Labels;
}


float MAClassifier::Predict(const MC::FloatList& input_vector, MC::FloatList& confidence)
{
  float Label = PredictReal(input_vector, confidence);

  if (MCIsFloatInfinity(Label) || CachedUniqueLabels.empty() ||
      MCContainerContains(CachedUniqueLabels, Label))
  {
    return Label;
  }

  float FinalResult = *CachedUniqueLabels.begin();
  float Diff = MCAbs(*CachedUniqueLabels.begin()-Label);

  for (auto& label : CachedUniqueLabels)
  {
    float CurrentDiff = MCAbs(label-Label);

    if (CurrentDiff < Diff)
    {
      Diff = CurrentDiff;
      FinalResult = label;
    }
  }
  return FinalResult;
}


float MAClassifier::PredictReal(const MC::FloatList& input_vector, MC::FloatList& confidences)
{
  MC_UNUSED(confidences)
  if (unlikely(CachedSamples.empty()))
  {
    MC_WARNING("The classifier is not trained yet.");
    return MCFloatInfinity();
  }
  if (input_vector.empty())
  {
    MC_WARNING("No data in the input vector for classifier prediction.");
    return MCFloatInfinity();
  }
  if (input_vector.size() != FeatureCount)
  {
    MC_WARNING("Feature count does not match with the expected test vector (%d != %d).",
               FeatureCount, input_vector.size());
    return MCFloatInfinity();
  }
  // Train the classifier
  if (unlikely(!Trained))
    Train();
  // Normalize the input vector if needed
  MC::FloatList NormalizedInputVector = input_vector;

  if (Preprocessing == MA::FeatureStandardization)
  {
    MCStandardizeVector<MC::FloatList, float>(NormalizedInputVector, PreprocessedData);
  } else
  if (Preprocessing == MA::FeatureRescaling)
  {
    MCRescaleVectorByMinMax<MC::FloatList, float>(NormalizedInputVector, PreprocessedData);
  }
  if (Preprocessing == MA::FeatureRescalingToVectorLength)
  {
    MCRescaleVectorByLength<MC::FloatList, float>(NormalizedInputVector);
  }
  /*
   * Do the actual predictions
   */
  if (DlibFunctions.get())
  {
    if (!Regression)
    {
      if (Classifier >= MA::SvmClassifierCLinearEkm && Classifier <= MA::SvmClassifierCLinearDlib)
      {
        DlibSampleDVectorType TestingSample;

        TestingSample.set_size(NormalizedInputVector.size(), 1);
        for (unsigned int i = 0; i < NormalizedInputVector.size(); ++i)
          TestingSample(i) = NormalizedInputVector[i];

        return DlibFunctions->ClassifierDFunction(TestingSample);
      }
      DlibSampleFVectorType TestingSample;

      TestingSample.set_size(NormalizedInputVector.size(), 1);
      for (unsigned int i = 0; i < NormalizedInputVector.size(); ++i)
        TestingSample(i) = NormalizedInputVector[i];

      return DlibFunctions->ClassifierFFunction(TestingSample);
    } else {
      DlibSampleDVectorType TestingSample;

      TestingSample.set_size(NormalizedInputVector.size(), 1);
      for (unsigned int i = 0; i < NormalizedInputVector.size(); ++i)
        TestingSample(i) = NormalizedInputVector[i];

      if (Classifier == MA::RvmClassifierLinear || Classifier == MA::KrRegressionLinear ||
          Classifier == MA::KrlsLinearRegression)
      {
        return DlibFunctions->LinearRegressionFunction(TestingSample);
      } else
      if (Classifier == MA::RvmClassifierSigmoid)
      {
        return DlibFunctions->SigmoidRegressionFunction(TestingSample);
      }
      return DlibFunctions->RbfRegressionFunction(TestingSample);
    }
  }
  if (Classifier == MA::MaxEntropyClassifierL1 || Classifier == MA::MaxEntropyClassifierL2)
  {
    ME_Sample TestingSample;

    for (unsigned int i1 = 0; i1 < NormalizedInputVector.size(); ++i1)
    {
      TestingSample.add_feature(MCToStr(i1), NormalizedInputVector[i1]);
    }
    MaxEntropy->classify(TestingSample);
    return MCStrConvert<float>(TestingSample.label);
  }
  if (Classifier == MA::LwpRegression)
  {
    doubleVec TestingSample(NormalizedInputVector.size());

    for (unsigned int i = 0; i < NormalizedInputVector.size(); ++i)
      TestingSample[i] = (double)NormalizedInputVector[i];
    return (float)Lwpr->predict(TestingSample)[0];
  }
#if defined(__unix__)
  if (Classifier == MA::GaussianProcessRegression)
  {
    double TestingSample[NormalizedInputVector.size()];

    for (unsigned int i = 0; i < NormalizedInputVector.size(); ++i)
      TestingSample[i] = (double)NormalizedInputVector[i];
    return (float)Gpr->GprScPtr->f(TestingSample);
  }
#endif
  if (Classifier == MA::PlsRegression)
  {
    Mat2D TestingSample(1, NormalizedInputVector.size());

    for (unsigned int i = 0; i < NormalizedInputVector.size(); ++i)
    {
      TestingSample(0, i) = NormalizedInputVector[i];
    }
    Mat2D Result = Plsr->fitted_values(TestingSample, PlsrComponents);

    return (float)Result(0, 0);
  }
  // Convert the feature vector to OpenCV's version
  cv::Mat TestingSample(1, NormalizedInputVector.size(), CV_32FC1, NormalizedInputVector.data());
  float Result = 0;

  // Classifier training
  if (Classifier == MA::NeuralNetworkClassifier)
  {
    cv::Mat Label;

    NeuralNetwork->predict(TestingSample, Label);
    Result = Label.at<float>(0, 0);
  } else
  if (Classifier == MA::NaiveBayesClassifier)
  {
    Result = BayesClassifier->predict(TestingSample);
  } else
  if (Classifier == MA::KNearestNeighborClassifier)
  {
    Result = KNearestClassifier->find_nearest(TestingSample, 1);
  } else
  if (Classifier >= MA::SvmClassifierCLinear && Classifier <= MA::SvmClassifierNuRbf)
  {
    Result = SvmClassifier->predict(TestingSample);
  } else
  if (Classifier == MA::DecisionTreeClassifier)
  {
    Result = (float)DecisionTree->predict(TestingSample)->value;
  } else
  if (Classifier == MA::RandomTreesClassifier)
  {
    if (Regression)
    {
      Result = RandomTrees->predict(TestingSample);
    } else {
      float Confidence = 0;

      Result = RandomTrees->predict_with_confidence(TestingSample, Confidence);
      // TODO: The confidence measure does not seem to work for random forest.
//      confidences.push_back(Confidence);
    }
  } else
  if (Classifier == MA::ExtremeRandomTreesClassifier)
  {
    Result = ExtremeRandomTrees->predict(TestingSample);
  } else
  if (Classifier == MA::GradientBoostedTreesClassifier)
  {
    Result = GradientBoostedTrees->predict(TestingSample);
  } else
  if (Classifier == MA::ExpectationMaximization)
  {
    cv::Vec2d Response;

    Response = EmClassifier->Classifier->predict(TestingSample);
    Result = EmClassifier->LabelMapping[(float)Response[1]];
    // TODO: The confidence measure is not in the range of [0; 1] and sometimes insane.
    // Recheck later when adopting to OpenCV 3.x. Don't forget about MAModel::Predict.
//    confidences.push_back((float)Response[0]);
  }
  return Result;
}


void MAClassifier::Train()
{
  if (Trained)
    return;

  // Normalize the training samples if needed
  MC::FloatTable NormalizedSamples = CachedSamples;

  PreprocessedData.clear();
  if (Preprocessing == MA::FeatureStandardization)
  {
    PreprocessedData = MCCalculateMeanStDevForTableColumns(NormalizedSamples);
    MCStandardizeTablePerColumns<float, MC::FloatList, MC::FloatTable>(NormalizedSamples, PreprocessedData);
  } else
  if (Preprocessing == MA::FeatureRescaling)
  {
    PreprocessedData = MCCalculateMinMaxForTableColumns(NormalizedSamples);
    MCRescaleTablePerColumnsByMinMax<float, MC::FloatList, MC::FloatTable>(NormalizedSamples, PreprocessedData);
  }
  if (Preprocessing == MA::FeatureRescalingToVectorLength)
  {
    MCRescaleTablePerRowByRowLength<float, MC::FloatList, MC::FloatTable>(NormalizedSamples);
  }

  /*
   * Initial training (OpenCV classifiers)
   */
  if (Classifier == MA::NeuralNetworkClassifier || Classifier == MA::NaiveBayesClassifier ||
      Classifier == MA::KNearestNeighborClassifier ||
      (Classifier >= MA::SvmClassifierCLinear && Classifier <= MA::SvmClassifierNuRbf) ||
      (Classifier >= MA::DecisionTreeClassifier && Classifier <= MA::ExpectationMaximization))
  {
    // Convert the feature vector to OpenCV's version
    cv::Mat TrainingSamples(0, FeatureCount, CV_32FC1);
    cv::Mat TrainingLabels(CachedLabels, false);
    float* Priors = nullptr;

    if (!PrioritizedClasses.empty())
    {
      MC::FloatList UniqueLabels(CachedUniqueLabels.begin(), CachedUniqueLabels.end());

      PriorityLabels = MC::FloatList(CachedUniqueLabels.size(), 0.1);
      for (unsigned int i = 0; i < PrioritizedClasses.size(); ++i)
        for (unsigned int i1 = 0; i1 < UniqueLabels.size(); ++i1)
      {
        if (PrioritizedClasses[i] == UniqueLabels[i1])
        {
          PriorityLabels[i1] = 0.15;
        }
      }
      Priors = PriorityLabels.data();
    }
    for (unsigned int i = 0; i < NormalizedSamples.size(); ++i)
    {
      cv::Mat Sample(1, FeatureCount, CV_32FC1, NormalizedSamples[i].data());

      TrainingSamples.push_back(Sample);
    }
    // Randomization before every training
    InitOpenCVRNGWrapper.Reset();
    MC_TRY_BEGIN
      // Classifier training
      if (Classifier == MA::NeuralNetworkClassifier)
      {
        cv::Mat Layers(4, 1, CV_32SC1);

        Layers.row(0) = cv::Scalar(FeatureCount); // -V525 (PVS Studio suppression)
        Layers.row(1) = cv::Scalar(FeatureCount);
        Layers.row(2) = cv::Scalar(FeatureCount);
        Layers.row(3) = cv::Scalar(1);
        NeuralNetwork->create(Layers);
        NeuralNetwork->train(TrainingSamples, TrainingLabels, cv::Mat());
      } else
      if (Classifier == MA::NaiveBayesClassifier)
      {
        BayesClassifier->train(TrainingSamples, TrainingLabels, cv::Mat(), cv::Mat(), false);
      } else
      if (Classifier == MA::KNearestNeighborClassifier)
      {
        KNearestClassifier->train(TrainingSamples, TrainingLabels, cv::Mat(), Regression, 32, false);
      } else
      if (Classifier >= MA::SvmClassifierCLinear && Classifier <= MA::SvmClassifierNuRbf)
      {
        if (SvmAutoTrain)
        {
          SvmClassifier->train_auto(TrainingSamples, TrainingLabels, cv::Mat(), cv::Mat(),
                                    *SvmClassifierParams.get());
          if (Classifier == MA::SvmClassifierCLinear || Classifier == MA::SvmClassifierCRbf)
          {
            MC_LOG("C parameter (SVM autotraining): %1.6f", SvmClassifier->get_params().C);
          }
          if (Classifier == MA::SvmClassifierNuLinear || Classifier == MA::SvmClassifierNuRbf)
            MC_LOG("ν (nu) parameter (SVM autotraining): %1.6f", SvmClassifier->get_params().nu);
          if (Classifier == MA::SvmClassifierCRbf || Classifier == MA::SvmClassifierNuRbf)
            MC_LOG("γ (gamma) parameter (SVM autotraining): %1.6f", SvmClassifier->get_params().gamma);
        } else {
          SvmClassifier->train(TrainingSamples, TrainingLabels, cv::Mat(), cv::Mat(),
                               *SvmClassifierParams.get());
        }
      } else
      if (Classifier == MA::DecisionTreeClassifier)
      {
        cv::Mat DtVarType(FeatureCount+1, 1, CV_8U);
        CvDTreeParams Params(TreeMaxDepth, // max depth
                             TreeNodeSampleLimit, // min sample count
                             0.001f, // regression accuracy: N/A here
                             true, // compute surrogate split, no missing data (variable importance calculation)
                             LabelCount <= 1 ? 2 : LabelCount, // max number of categories
                             1, // k-fold cross-validation to prune a tree
                             false, // more aggressive pruning
                             true, // truncate pruned branches
                             Priors // the array of priors
                             );

        DtVarType.setTo(cv::Scalar(CV_VAR_NUMERICAL)); // all inputs are numerical
        // this is a classification problem (i.e. predict a discrete number of class
        // outputs) so reset the last (+1) output var_type element to CV_VAR_CATEGORICAL
        if (!Regression)
          DtVarType.at<uchar>(FeatureCount, 0) = CV_VAR_CATEGORICAL;
        else
          DtVarType.at<uchar>(FeatureCount, 0) = CV_VAR_NUMERICAL;
        DecisionTree->train(TrainingSamples, CV_ROW_SAMPLE, TrainingLabels, cv::Mat(), cv::Mat(),
                            DtVarType, cv::Mat(), Params);
      } else
      if (Classifier == MA::RandomTreesClassifier || Classifier == MA::ExtremeRandomTreesClassifier)
      {
        cv::Mat RtVarType(FeatureCount+1, 1, CV_8U);
        bool VariableImportance = (Classifier == MA::RandomTreesClassifier ? true : false);
        CvRTParams Params(TreeMaxDepth, TreeNodeSampleLimit, 0.01, VariableImportance, 1024,
                          Priors, VariableImportance, 0, RtMaxForestSize, 0.01,
                          CV_TERMCRIT_ITER+CV_TERMCRIT_EPS);

        RtVarType.setTo(cv::Scalar(CV_VAR_NUMERICAL)); // all inputs are numerical
        // this is a classification problem (i.e. predict a discrete number of class
        // outputs) so reset the last (+1) output var_type element to CV_VAR_CATEGORICAL
        if (!Regression)
          RtVarType.at<uchar>(FeatureCount, 0) = CV_VAR_CATEGORICAL;
        else
          RtVarType.at<uchar>(FeatureCount, 0) = CV_VAR_NUMERICAL;

        if (Classifier == MA::RandomTreesClassifier)
        {
          RandomTrees->train(TrainingSamples, CV_ROW_SAMPLE, TrainingLabels, cv::Mat(), cv::Mat(),
                             RtVarType, cv::Mat(), Params);
        } else {
          ExtremeRandomTrees->train(TrainingSamples, CV_ROW_SAMPLE, TrainingLabels, cv::Mat(), cv::Mat(),
                                    RtVarType, cv::Mat(), Params);
        }
      } else
      if (Classifier == MA::GradientBoostedTreesClassifier)
      {
        cv::Mat GbtVarType(FeatureCount+1, 1, CV_8U);
        CvGBTreesParams Params(CvGBTrees::SQUARED_LOSS, RtMaxForestSize, 0.8, 0.01, TreeMaxDepth, false);

        GbtVarType.setTo(cv::Scalar(CV_VAR_NUMERICAL)); // all inputs are numerical
        // this is a classification problem (i.e. predict a discrete number of class
        // outputs) so reset the last (+1) output var_type element to CV_VAR_CATEGORICAL
        if (!Regression)
          GbtVarType.at<uchar>(FeatureCount, 0) = CV_VAR_CATEGORICAL;
        else
          GbtVarType.at<uchar>(FeatureCount, 0) = CV_VAR_NUMERICAL;

        GradientBoostedTrees->train(TrainingSamples, CV_ROW_SAMPLE, TrainingLabels, cv::Mat(), cv::Mat(),
                                    GbtVarType, cv::Mat(), Params);
      } else
      if (Classifier == MA::ExpectationMaximization)
      {
        // Sort out the samples into one mixture set/label
        MC::FloatList Labels(CachedUniqueLabels.begin(), CachedUniqueLabels.end());
        cv::Mat Weights(1, Labels.size(), CV_64FC1, cv::Scalar(0));
        std::vector<cv::Mat> Covs;
        cv::Mat Means(Labels.size(), FeatureCount, CV_64FC1);

        Covs.resize(Labels.size());
        EmClassifier->LabelMapping.clear();
        for (unsigned int i = 0; i < Labels.size(); ++i)
        {
          cv::Mat Samples(0, FeatureCount, CV_64FC1);

          EmClassifier->LabelMapping.insert(std::make_pair((float)i, Labels[i]));
          for (unsigned int i1 = 0; i1 < NormalizedSamples.size(); ++i1)
          {
            if (CachedLabels[i1] == Labels[i])
            {
              const cv::Mat Sample(1, FeatureCount, CV_32FC1, NormalizedSamples[i1].data());
              cv::Mat TempSample;

              Sample.convertTo(TempSample, CV_64FC1);
              Samples.push_back(TempSample);
            }
          }
          cv::calcCovarMatrix(Samples, Covs[i], Means.row(i),
                              CV_COVAR_NORMAL+CV_COVAR_ROWS+CV_COVAR_SCALE, CV_64FC1);
          if (!PrioritizedClasses.empty() && MCContainerContains(PrioritizedClasses, Labels[i]))
          {
            Weights.at<double>(i) = 0.15;
          } else {
            Weights.at<double>(i) = 0.1;
          }
        }
        // Note: Default parameter EM::COV_MAT_DIAGONAL is changed to cv::EM::COV_MAT_GENERIC
        // because of http://stackoverflow.com/questions/23485982
        EmClassifier->Classifier.reset(new cv::EM(LabelCount, cv::EM::COV_MAT_GENERIC));
        EmClassifier->Classifier->trainE(TrainingSamples, Means, Covs, Weights);
      }
    MC_CATCH_BEGIN
      MC_WARNING("Fatal error on OpenCV side while training a classifier!");
      return;
    MC_CATCH_END
    Trained = true;
    return;
  }
  /*
   * Initial training (maximum entropy)
   */
  if ((Classifier == MA::MaxEntropyClassifierL1 || Classifier == MA::MaxEntropyClassifierL2) &&
      !MaxEntropy.get())
  {
    MaxEntropy.reset(new ME_Model);
    // Build training samples
    for (unsigned int i = 0; i < NormalizedSamples.size(); ++i)
    {
      ME_Sample NewSample(MCToStr(CachedLabels[i]));

      for (unsigned int i1 = 0; i1 < NormalizedSamples[0].size(); ++i1)
      {
        NewSample.add_feature(MCToStr(i1), NormalizedSamples[i][i1]);
      }
      MaxEntropy->add_training_sample(NewSample);
    }
    // Training
    if (Classifier == MA::MaxEntropyClassifierL1)
      MaxEntropy->use_l1_regularizer((double)MeL1);
    else
    if (Classifier == MA::MaxEntropyClassifierL2)
      MaxEntropy->use_l2_regularizer((double)MeL2);
    MaxEntropy->train();
    Trained = true;
    return;
  }
  /*
   * Initial training (lwp regression)
   */
  if (Classifier == MA::LwpRegression && !Lwpr.get())
  {
    MC_TRY_BEGIN
      Lwpr.reset(new LWPR_Object(FeatureCount, 1));
    MC_CATCH_BEGIN
      printf("LWPR_Object FAILED\n");
      exit(1);
    MC_CATCH_END
    Lwpr->setInitD(50);
    /* Set init_alpha to 250 in all elements */
    Lwpr->setInitAlpha(250);
    Lwpr->wGen(LwprAlpha);
    std::vector<doubleVec> TrainingSamples;
    std::vector<doubleVec> TrainingLabels;

    // Convert the training samples/labels
    for (unsigned int i = 0; i < NormalizedSamples.size(); ++i)
    {
      TrainingSamples.push_back(doubleVec(FeatureCount));

      for (unsigned int i1 = 0; i1 < FeatureCount; ++i1)
      {
        TrainingSamples[i][i1] = (double)NormalizedSamples[i][i1];
      }
      TrainingLabels.push_back(doubleVec(1));
      TrainingLabels[i][0] = (double)CachedLabels[i];
    }
    // Training
    float Error = 1000.0;
    int Iterations = 0;

    while (Iterations < 1000 && Error > 0.00001)
    {
      Error = 0.0;
      Iterations++;
      for (unsigned int i = 0; i < NormalizedSamples.size(); ++i)
        Lwpr->update(TrainingSamples[i], TrainingLabels[i]);
      for (unsigned int i = 0; i < NormalizedSamples.size(); ++i)
      {
        float CurrentError = (float)Lwpr->predict(TrainingSamples[i])[0]-CachedLabels[i];

        Error += CurrentError*CurrentError;
      }
    }
//      printf("Iterations: %d - Final error: %1.10f\n", Iterations, (float)Error);
    Trained = true;
    return;
  }
#if defined(__unix__)
  /*
   * Initial training (Gaussian process regression)
   */
  if (Classifier == MA::GaussianProcessRegression && !Gpr->GprScPtr.get())
  {
    Gpr->GprScPtr.reset(new libgp::GaussianProcess(FeatureCount, "CovSum ( CovSEiso, CovNoise)"));
    Eigen::VectorXd params(Gpr->GprScPtr->covf().get_param_dim());

    params << 0.0, 0.0, -2.0;
    // set parameters of covariance function
    Gpr->GprScPtr->covf().set_loghyper(params);
    // Convert the training samples/labels
    for (unsigned int i = 0; i < NormalizedSamples.size(); ++i)
    {
      double TrainingSample[FeatureCount];

      for (unsigned int i1 = 0; i1 < FeatureCount; ++i1)
      {
        TrainingSample[i1] = (double)NormalizedSamples[i][i1];
      }
      Gpr->GprScPtr->add_pattern(TrainingSample, (double)CachedLabels[i]);
    }
    Trained = true;
    return;
  }
#endif
  /*
   * Initial training (pls regression)
   */
  if (Classifier == MA::PlsRegression && !Plsr.get())
  {
    Plsr.reset(new PLS_Model);
    Mat2D TrainingSamples(NormalizedSamples.size(), FeatureCount);
    Mat2D TrainingLabels(CachedLabels.size(), 1);

//      printf("Pls: Sample matrix size: %dx%d\n", NormalizedSamples.size(), FeatureCount);
    // Convert the training samples/labels
    for (unsigned int i = 0; i < NormalizedSamples.size(); ++i)
    {
      for (unsigned int i1 = 0; i1 < FeatureCount; ++i1)
      {
        TrainingSamples(i, i1) = NormalizedSamples[i][i1];
      }
      TrainingLabels(i, 0) = CachedLabels[i];
    }
    if (PlsrComponents == 0)
    {
      int LastComponentCount = -1;

      for (int i = 3; i < 100; i += 5)
      {
//          printf("Pls: Trying component count: %d\n", i);
        Plsr->initialize((int)TrainingSamples.cols(), (int)TrainingLabels.cols(), i);
        Plsr->plsr(TrainingSamples, TrainingLabels, KERNEL_TYPE2);
        Rowi Components = Plsr->loo_optimal_num_components(TrainingSamples, TrainingLabels);
        bool Consistent = true;

        for (int i1 = 1; i1 < Components.size(); ++i1)
        {
          if (Components(0) != Components(i1))
          {
            Consistent = false;
            break;
          }
        }
        if (!Consistent)
          continue;
        if ((LastComponentCount == -1 && Components(0) > 0) ||
            (LastComponentCount != -1 && Components(0) != LastComponentCount))
        {
          LastComponentCount = Components(0);
          continue;
        }
        if (LastComponentCount != -1 && Components(0) == LastComponentCount)
          break;
      }
      if (LastComponentCount == -1)
      {
        LastComponentCount = 10;
      } else {
        LastComponentCount += 2;
      }
      PlsrComponents = LastComponentCount;
//        printf("Final components: %d\n", LastComponentCount);
    }
    Plsr->initialize((int)TrainingSamples.cols(), (int)TrainingLabels.cols(), PlsrComponents);
    Plsr->plsr(TrainingSamples, TrainingLabels, KERNEL_TYPE2);
    Trained = true;
    return;
  }
  /*
   * Initial training (Dlib classifier methods)
   */
  if (!DlibFunctions.get() && !Regression)
  {
    DlibFunctions.reset(new DlibWrapper);
    if (Classifier >= MA::SvmClassifierCLinearEkm && Classifier <= MA::SvmClassifierCLinearDlib)
    {
      // Build training samples
      DlibDSampleTableType TrainingSamples;
      MC::DoubleList TrainingLabels;

      for (unsigned int i = 0; i < NormalizedSamples.size(); ++i)
      {
        DlibSampleDVectorType List;

        List.set_size(NormalizedSamples[0].size(), 1);
        for (unsigned int i1 = 0; i1 < NormalizedSamples[0].size(); ++i1)
        {
          List(i1) = NormalizedSamples[i][i1];
        }
        TrainingSamples.push_back(List);
        TrainingLabels.push_back((double)CachedLabels[i]);
      }
      DlibDMultiTrainerType MultiTrainer;

      // Training
      if (Classifier == MA::SvmClassifierCLinearEkm)
      {
        SvmCLinearEkmTrainerType SvmTrainer;

        SvmTrainer.set_epsilon(DlibEpsilon);
        SvmTrainer.set_c(SvmC);
        SvmTrainer.set_kernel(DlibDLinearKernelType());
        MultiTrainer.set_trainer(SvmTrainer);
      } else
      if (Classifier == MA::SvmClassifierCRbfEkm)
      {
        SvmCRbfEkmTrainerType SvmTrainer;

        SvmTrainer.set_epsilon(DlibEpsilon);
        SvmTrainer.set_c(SvmC);
        SvmTrainer.set_kernel(DlibDRbfKernelType(RvmRbfGamma));
        MultiTrainer.set_trainer(SvmTrainer);
      } else
      if (Classifier == MA::SvmClassifierCLinearDcd)
      {
        SvmCLinearDcdTrainerType SvmTrainer;

        SvmTrainer.set_epsilon(DlibEpsilon);
        SvmTrainer.set_c(SvmC);
        MultiTrainer.set_trainer(SvmTrainer);
      } else
      if (Classifier == MA::SvmClassifierCLinearDlib)
      {
        SvmCLinearTrainerType SvmTrainer;

        SvmTrainer.set_epsilon(DlibEpsilon);
        SvmTrainer.set_c(SvmC);
        MultiTrainer.set_trainer(SvmTrainer);
      }
      DlibFunctions->ClassifierDFunction = MultiTrainer.train(TrainingSamples, TrainingLabels);
    } else {
      // Build training samples
      DlibFSampleTableType TrainingSamples;
      MC::DoubleList TrainingLabels;

      for (unsigned int i = 0; i < NormalizedSamples.size(); ++i)
      {
        DlibSampleFVectorType List;

        List.set_size(NormalizedSamples[0].size(), 1);
        for (unsigned int i1 = 0; i1 < NormalizedSamples[0].size(); ++i1)
        {
          List(i1) = NormalizedSamples[i][i1];
        }
        TrainingSamples.push_back(List);
        TrainingLabels.push_back((double)CachedLabels[i]);
      }
      DlibFMultiTrainerType MultiTrainer;

      // Training
      if (Classifier == MA::RvmClassifierLinear)
      {
        RvmLinearTrainerType RvmTrainer;

        RvmTrainer.set_epsilon(DlibEpsilon);
        RvmTrainer.set_kernel(DlibFLinearKernelType());
        MultiTrainer.set_trainer(RvmTrainer);
      } else
      if (Classifier == MA::RvmClassifierSigmoid)
      {
        RvmSigmoidTrainerType RvmTrainer;

        RvmTrainer.set_epsilon(DlibEpsilon);
        RvmTrainer.set_kernel(DlibFSigmoidKernelType(RvmSigmoidGamma, -1.0));
        MultiTrainer.set_trainer(RvmTrainer);
      } else
      if (Classifier == MA::RvmClassifierRbf)
      {
        RvmRbfTrainerType RvmTrainer;

        RvmTrainer.set_epsilon(DlibEpsilon);
        RvmTrainer.set_kernel(DlibFRbfKernelType(RvmRbfGamma));
        MultiTrainer.set_trainer(RvmTrainer);
      } else
      if (Classifier == MA::KrRegressionLinear)
      {
        KrrLinearTrainerType KrrTrainer;

        KrrTrainer.set_lambda(KrrLambda);
        KrrTrainer.set_kernel(DlibFLinearKernelType());
        MultiTrainer.set_trainer(KrrTrainer);
      } else
      if (Classifier == MA::KrRegressionRbf)
      {
        KrrRbfTrainerType KrrTrainer;

        KrrTrainer.set_lambda(KrrLambda);
        KrrTrainer.set_kernel(DlibFRbfKernelType(KrrGamma));
        MultiTrainer.set_trainer(KrrTrainer);
      }
      DlibFunctions->ClassifierFFunction = MultiTrainer.train(TrainingSamples, TrainingLabels);
    }
    Trained = true;
    return;
  }
  /*
   * Initial training (Dlib regression methods)
   */
  if (!DlibFunctions.get())
  {
    DlibFunctions.reset(new DlibWrapper);
    // Build training samples
    DlibDSampleTableType TrainingSamples;
    MC::DoubleList TrainingLabels;

    // Kernel recursive least squares regression must be trained by sample basis
    if (Classifier != MA::KrlsLinearRegression && Classifier != MA::KrlsRbfRegression)
    {
      for (unsigned int i = 0; i < NormalizedSamples.size(); ++i)
      {
        DlibSampleDVectorType List;

        List.set_size(NormalizedSamples[0].size(), 1);
        for (unsigned int i1 = 0; i1 < NormalizedSamples[0].size(); ++i1)
        {
          List(i1) = NormalizedSamples[i][i1];
        }
        TrainingSamples.push_back(List);
        TrainingLabels.push_back((double)CachedLabels[i]);
      }
    }
    // Training
    DlibFunctions.reset(new DlibWrapper);
    if (Classifier == MA::RvmClassifierLinear)
    {
      LinearRegressionTrainerType RvmRegressionTrainer;

      RvmRegressionTrainer.set_epsilon(DlibEpsilon);
      RvmRegressionTrainer.set_kernel(DlibDLinearKernelType());
      DlibFunctions->LinearRegressionFunction = RvmRegressionTrainer.train(TrainingSamples, TrainingLabels);
    } else
    if (Classifier == MA::RvmClassifierSigmoid)
    {
      SigmoidRegressionTrainerType RvmRegressionTrainer;

      RvmRegressionTrainer.set_epsilon(DlibEpsilon);
      RvmRegressionTrainer.set_kernel(DlibDSigmoidKernelType(RvmSigmoidGamma, -1.0));
      DlibFunctions->SigmoidRegressionFunction = RvmRegressionTrainer.train(TrainingSamples, TrainingLabels);
    } else
    if (Classifier == MA::RvmClassifierRbf)
    {
      RbfRegressionTrainerType RvmRegressionTrainer;

      RvmRegressionTrainer.set_epsilon(DlibEpsilon);
      RvmRegressionTrainer.set_kernel(DlibDRbfKernelType(RvmRbfGamma));
      DlibFunctions->RbfRegressionFunction = RvmRegressionTrainer.train(TrainingSamples, TrainingLabels);
    } else
    if (Classifier == MA::KrRegressionLinear)
    {
      KrrLinearRegressionTrainerType KrrRegressionTrainer;

      KrrRegressionTrainer.set_lambda(KrrLambda);
      KrrRegressionTrainer.set_kernel(DlibDLinearKernelType());
      DlibFunctions->LinearRegressionFunction = KrrRegressionTrainer.train(TrainingSamples, TrainingLabels);
    } else
    if (Classifier == MA::KrRegressionRbf)
    {
      KrrRbfRegressionTrainerType KrrRegressionTrainer;

      KrrRegressionTrainer.set_lambda(KrrLambda);
      KrrRegressionTrainer.set_kernel(DlibDRbfKernelType(KrrGamma));
      DlibFunctions->RbfRegressionFunction = KrrRegressionTrainer.train(TrainingSamples, TrainingLabels);
    } else
    if (Classifier == MA::KrlsLinearRegression)
    {
      KrlsLinearRegressionTrainerType KrlsLinearRegressionTrainer(DlibDLinearKernelType(), KrlsTolerance);

      for (unsigned int i = 0; i < NormalizedSamples.size(); ++i)
      {
        DlibSampleDVectorType List;

        List.set_size(NormalizedSamples[0].size(), 1);
        for (unsigned int i1 = 0; i1 < NormalizedSamples[0].size(); ++i1)
        {
          List(i1) = NormalizedSamples[i][i1];
        }
        KrlsLinearRegressionTrainer.train(List, CachedLabels[i]);
      }
      DlibFunctions->LinearRegressionFunction = KrlsLinearRegressionTrainer.get_decision_function();
    } else
    if (Classifier == MA::KrlsRbfRegression)
    {
      KrlsRbfRegressionTrainerType KrlsRbfRegressionTrainer(DlibDRbfKernelType(KrlsGamma), KrlsTolerance);

      for (unsigned int i = 0; i < NormalizedSamples.size(); ++i)
      {
        DlibSampleDVectorType List;

        List.set_size(NormalizedSamples[0].size(), 1);
        for (unsigned int i1 = 0; i1 < NormalizedSamples[0].size(); ++i1)
        {
          List(i1) = NormalizedSamples[i][i1];
        }
        KrlsRbfRegressionTrainer.train(List, CachedLabels[i]);
      }
      DlibFunctions->RbfRegressionFunction = KrlsRbfRegressionTrainer.get_decision_function();
    }
    Trained = true;
    return;
  }
}


MA::CvResultsType MAClassifier::CrossValidate(int iterations, MA::ClassifierCrossValidationType cv_type,
                                              float cv_parameter, const MC::FloatTable& samples,
                                              const MC::FloatList& labels,
                                              const MC::FloatTable& validation_samples,
                                              const MC::FloatList& validation_labels,
                                              float regression_accuracy)
{
  if (cv_type == MA::RandomLabelCrossValidation && cv_parameter < 1)
  {
    MC_WARNING("At least one sample per label is needed for random label cross-validation (%d < 1)",
               cv_parameter);
    return MA::CvResultsType();
  }
  if (cv_type == MA::KFoldCrossValidation && cv_parameter < 2)
  {
    MC_WARNING("At least 2-fold cross-validation can be done (%d < 2)", cv_parameter);
    return MA::CvResultsType();
  }
  if (cv_type == MA::MonteCarloCrossValidation && (cv_parameter < 1 || cv_parameter > 99))
  {
    MC_WARNING("Monte-Carlo cross-validation must have the training set size between 1 < %d < 100)",
               cv_parameter);
    return MA::CvResultsType();
  }
  if (samples.size() != labels.size())
  {
    MC_WARNING("Label and samples counts are not consistent (%d != %d)", labels.size(), samples.size());
    return MA::CvResultsType();
  }
  MC::FloatTable* SampleFolds = new MC::FloatTable[(unsigned int)cv_parameter];
  MC::FloatList* LabelFolds = new MC::FloatList[(unsigned int)cv_parameter];
  MC::IntList* TrainingSampleIndexFolds = new MC::IntList[(unsigned int)cv_parameter];
  unsigned int MonteCarloTrainingSetSize = (int)(cv_parameter*samples.size() / 100.0);
  const MC::FloatList UniqueTestingLabels = MCUniqueItemsFromContainer<MC::FloatList>(labels);
  const MC::FloatList UniqueValidationLabels = MCUniqueItemsFromContainer<MC::FloatList>(validation_labels);
  int TestingLabelCount = UniqueTestingLabels.size();
  int ValidationLabelCount = UniqueValidationLabels.size();
  MA::FloatTableList TestingResultsTableList;
  MA::FloatTableList ValidationResultsTableList;
  MC::FloatList SampleRanks(samples.size(), 0.0);

  // Note: K-fold cross-validation needs to calculate cv_paremeter-th folds in an iteration
  for (int iter = 0; iter < iterations*(cv_type == MA::KFoldCrossValidation ? (int)cv_parameter : 1);
       ++iter)
  {
    MC::FloatTable TrainingSamples;
    MC::FloatList TrainingLabels;
    MC::FloatTable TestingSamples;
    MC::FloatList TestingLabels;
    MC::IntList Gambling;
    MC::FloatTable TestingResultsTable;
    MC::FloatTable ValidationResultsTable;
    MC::IntList TestingLabelSamples(TestingLabelCount, 0);
    MC::IntList ValidationLabelSamples(ValidationLabelCount, 0);
    MC::IntList TrainingSampleIndexes;

    // Initialize the testing results table
    for (int i = 0; i < TestingLabelCount; ++i)
      TestingResultsTable.push_back(MC::FloatList(TestingLabelCount, 0.0));
    // Initialize the validation results table
    if (!validation_samples.empty())
    {
      for (int i = 0; i < ValidationLabelCount; ++i)
        ValidationResultsTable.push_back(MC::FloatList(ValidationLabelCount, 0.0));
    }
    // Build the training/testing sets for random label cross-validation
    if (cv_type == MA::RandomLabelCrossValidation)
    {
      MA::FloatIntMap LabelsMap;

      // Fill the label map
      for (unsigned int i = 0; i < UniqueTestingLabels.size(); ++i)
      {
        LabelsMap.insert(std::make_pair(UniqueTestingLabels[i], 0));
      }
      // Gambling the indexes
      for (unsigned int i = 0; i < samples.size(); ++i)
      {
        Gambling.push_back(i);
      }
      MCRandomizeContainer<int, MC::IntList>(Gambling);
      for (unsigned int i = 0; i < samples.size(); ++i)
      {
        int CurrentIndex = Gambling[i];

        if (LabelsMap[labels[CurrentIndex]] < (int)cv_parameter)
        {
          LabelsMap[labels[CurrentIndex]]++;
          TrainingLabels.push_back(labels[CurrentIndex]);
          TrainingSamples.push_back(samples[CurrentIndex]);
          TrainingSampleIndexes.push_back(CurrentIndex);
        } else {
          TestingLabels.push_back(labels[CurrentIndex]);
          TestingSamples.push_back(samples[CurrentIndex]);
        }
      }
    }
    // Build the training/testing sets for k-fold cross-validation
    if (cv_type == MA::KFoldCrossValidation)
    {
      // Gambling the samples/labels
      if (iter % (int)cv_parameter == 0)
      {
        for (int i = 0; i < (int)cv_parameter; ++i)
        {
          SampleFolds[i].clear();
          LabelFolds[i].clear();
          TrainingSampleIndexFolds[i].clear();
        }
        for (unsigned int i = 0; i < samples.size(); ++i)
        {
          Gambling.push_back(i);
        }
        MCRandomizeContainer<int, MC::IntList>(Gambling);
        for (unsigned int i = 0; i < samples.size(); ++i)
        {
          SampleFolds[i % (int)cv_parameter].push_back(samples[Gambling[i]]);
          LabelFolds[i % (int)cv_parameter].push_back(labels[Gambling[i]]);
          TrainingSampleIndexFolds[i % (int)cv_parameter].push_back(Gambling[i]);
        }
      }
      // Apply the current fold
      for (int i = 0; i < (int)cv_parameter; ++i)
      {
        if (iter % (int)cv_parameter == i)
        {
          MCMergeContainers(TestingSamples, SampleFolds[i]);
          MCMergeContainers(TestingLabels, LabelFolds[i]);
        } else {
          MCMergeContainers(TrainingSamples, SampleFolds[i]);
          MCMergeContainers(TrainingLabels, LabelFolds[i]);
          MCMergeContainers(TrainingSampleIndexes, TrainingSampleIndexFolds[i]);
        }
      }
    }
    // Build the training/testing sets for Monte-Carlo cross-validation
    if (cv_type == MA::MonteCarloCrossValidation)
    {
      // Gambling the indexes
      for (unsigned int i = 0; i < samples.size(); ++i)
      {
        Gambling.push_back(i);
      }
      MCRandomizeContainer<int, MC::IntList>(Gambling);
      for (unsigned int i = 0; i < samples.size(); ++i)
      {
        if (i <= MonteCarloTrainingSetSize)
        {
          TrainingLabels.push_back(labels[Gambling[i]]);
          TrainingSamples.push_back(samples[Gambling[i]]);
          TrainingSampleIndexes.push_back(Gambling[i]);
        } else {
          TestingLabels.push_back(labels[Gambling[i]]);
          TestingSamples.push_back(samples[Gambling[i]]);
        }
      }
    }
    // Reset the classifier if needs be
    if (Trained)
      Reset();
    // Add samples and predict
    AddSamples(TrainingSamples, TrainingLabels);
    // Calculate the accuracy on the testing set
    MC::FloatList CurrentResults;

    if (!TestingSamples.empty())
    {
      MC::FloatList Confidence;

      CurrentResults = Predict(TestingSamples, Confidence);
      for (unsigned int i = 0; i < TestingLabels.size(); ++i)
      {
        if (!Regression)
        {
          int TestingLabelIndex = MCItemIndicesInContainer(UniqueTestingLabels, TestingLabels[i])[0];
          int PredictedLabelIndex = MCItemIndicesInContainer(UniqueTestingLabels, CurrentResults[i])[0];

          if (TestingLabelIndex >= 0)
          {
            if (PredictedLabelIndex >= 0)
            {
              TestingResultsTable[TestingLabelIndex][PredictedLabelIndex]++;
            }
            TestingLabelSamples[TestingLabelIndex]++;
          }
        } else {
          if (CurrentResults[i] >= TestingLabels[i]-regression_accuracy &&
              CurrentResults[i] <= TestingLabels[i]+regression_accuracy)
          {
            int TestingLabelIndex = MCItemIndicesInContainer(UniqueTestingLabels, TestingLabels[i])[0];
            int PredictedLabelIndex = MCItemIndicesInContainer(UniqueTestingLabels, CurrentResults[i])[0];

            if (TestingLabelIndex >= 0)
            {
              if (PredictedLabelIndex >= 0)
              {
                TestingResultsTable[TestingLabelIndex][PredictedLabelIndex]++;
              }
              TestingLabelSamples[TestingLabelIndex]++;
            }
          }
        }
      }
      // Update the sample ranks
      // 1. Calculate the final accuracy on the testing set in percentage
      float Result = MCCalculateDiagonalStatInTable<float>(TestingResultsTable, *new MCArithmeticSum<float>);

      Result = Result / CurrentResults.size()*100;
      // 2. Add the accuracy number to all used training samples as a performance measure
      for (unsigned int i = 0; i < TrainingSampleIndexes.size(); ++i)
      {
        SampleRanks[TrainingSampleIndexes[i]] += Result;
      }
    }
    // Check the hypothesis on the validation set
    if (!validation_samples.empty())
    {
      MC::FloatList Confidence;

      CurrentResults = Predict(validation_samples, Confidence);
      for (unsigned int i = 0; i < validation_labels.size(); ++i)
      {
        if (!Regression)
        {
          int ValidationLabelIndex = MCItemIndicesInContainer(UniqueValidationLabels, validation_labels[i])[0];
          int PredictedLabelIndex = MCItemIndicesInContainer(UniqueValidationLabels, CurrentResults[i])[0];

          if (ValidationLabelIndex >= 0)
          {
            if (PredictedLabelIndex >= 0)
            {
              ValidationResultsTable[ValidationLabelIndex][PredictedLabelIndex]++;
            }
            ValidationLabelSamples[ValidationLabelIndex]++;
          }
        } else {
          if (CurrentResults[i] >= validation_labels[i]-regression_accuracy &&
              CurrentResults[i] <= validation_labels[i]+regression_accuracy)
          {
            int ValidationLabelIndex = MCItemIndicesInContainer(UniqueValidationLabels, validation_labels[i])[0];
            int PredictedLabelIndex = MCItemIndicesInContainer(UniqueValidationLabels, CurrentResults[i])[0];

            if (ValidationLabelIndex >= 0)
            {
              if (PredictedLabelIndex >= 0)
              {
                ValidationResultsTable[ValidationLabelIndex][PredictedLabelIndex]++;
              }
              ValidationLabelSamples[ValidationLabelIndex]++;
            }
          }
        }
      }
    } // End of validation set evaluation
    // Calculation the results of this run
    if (!TestingSamples.empty())
    {
      for (unsigned int i = 0; i < TestingLabelSamples.size(); ++i)
      {
        if (TestingLabelSamples[i] != 0.0)
          MCMultiplyTableRow(TestingResultsTable, i, (float)100.0 / TestingLabelSamples[i]);
      }
      TestingResultsTableList.push_back(TestingResultsTable);
    }
    if (!validation_samples.empty())
    {
      for (unsigned int i = 0; i < ValidationLabelSamples.size(); ++i)
      {
        if (ValidationLabelSamples[i] != 0.0)
          MCMultiplyTableRow(ValidationResultsTable, i, (float)100.0 / ValidationLabelSamples[i]);
      }
      ValidationResultsTableList.push_back(ValidationResultsTable);
    }
  }
  // Averaging the k-fold cross-validation results
  if (cv_type == MA::KFoldCrossValidation)
  {
    MA::FloatTableList FinalTestingResults;
    MA::FloatTableList FinalValidationResults;
    MC::FloatTable CurrentTable;
    MC::IntList LabelSampleCounts;

    for (unsigned int i = 0; i < TestingResultsTableList.size(); ++i)
    {
      if (i % (int)cv_parameter == 0)
      {
        CurrentTable.clear();
        LabelSampleCounts = MC::IntList(TestingResultsTableList[0].size(), 0);
      }
      MCSumTables(CurrentTable, TestingResultsTableList[i], (float)1.0);
      // Check how many real label samples we have to average the iterations correctly
      for (unsigned int i1 = 0; i1 < TestingResultsTableList[i].size(); ++i1)
      {
        bool Empty = true;
        for (unsigned int i2 = 0; i2 < TestingResultsTableList[i][i1].size(); ++i2)
        {
          if (TestingResultsTableList[i][i1][i2] != 0.0)
          {
            Empty = false;
            break;
          }
        }
        if (!Empty)
          LabelSampleCounts[i1]++;
      }
      if (i % (int)cv_parameter == (unsigned int)cv_parameter-1)
      {
        for (unsigned int i1 = 0; i1 < CurrentTable.size(); ++i1)
          MCMultiplyTableRow(CurrentTable, i1, (float)1.0 / LabelSampleCounts[i1]);
        FinalTestingResults.push_back(CurrentTable);
      }
    }
    if (!validation_samples.empty())
    {
      for (unsigned int i = 0; i < ValidationResultsTableList.size(); ++i)
      {
        if (i % (int)cv_parameter == 0)
        {
          LabelSampleCounts = MC::IntList(ValidationResultsTableList[0].size(), 0);
          CurrentTable.clear();
        }
        MCSumTables(CurrentTable, ValidationResultsTableList[i], (float)1.0);
        // Check how many real label samples we have to average the iterations correctly
        for (unsigned int i1 = 0; i1 < ValidationResultsTableList[i].size(); ++i1)
        {
          bool Empty = true;
          for (unsigned int i2 = 0; i2 < ValidationResultsTableList[i][i1].size(); ++i2)
          {
            if (ValidationResultsTableList[i][i1][i2] != 0.0)
            {
              Empty = false;
              break;
            }
          }
          if (!Empty)
            LabelSampleCounts[i1]++;
        }
        if (i % (int)cv_parameter == (unsigned int)cv_parameter-1)
        {
          for (unsigned int i1 = 0; i1 < CurrentTable.size(); ++i1)
            MCMultiplyTableRow(CurrentTable, i1, (float)1.0 / LabelSampleCounts[i1]);
          FinalValidationResults.push_back(CurrentTable);
        }
      }
    }
    delete [] SampleFolds;
    delete [] LabelFolds;
    delete [] TrainingSampleIndexFolds;
    return MA::CvResultsType(FinalTestingResults, FinalValidationResults, SampleRanks);
  }
  delete [] SampleFolds;
  delete [] LabelFolds;
  delete [] TrainingSampleIndexFolds;
  return MA::CvResultsType(TestingResultsTableList, ValidationResultsTableList, SampleRanks);
}


MCBinaryData* MAClassifier::Encode() const
{
  // Huh, idea is from:
  // http://boost.2283326.n4.nabble.com/in-memory-stream-binary-archive-serialization-td2578880.html
  std::vector<char> TempBuffer;

  boost::iostreams::stream<boost::iostreams::back_insert_device<std::vector<char> > > OutStream(TempBuffer);
  {
    boost::archive::binary_oarchive OutputArchive(OutStream);

    OutputArchive << *this;
  }
  MCBinaryData* BinaryData = new MCBinaryData((int)TempBuffer.size());

  memcpy(BinaryData->GetData(), &TempBuffer[0], TempBuffer.size());
  return BinaryData;
}


MAClassifier* MAClassifier::Decode(const MCBinaryData& data)
{
  MAClassifier* Classifier = nullptr;

  MC_TRY_BEGIN
    boost::iostreams::basic_array_source<char> Source((char*)data.GetData(),
                                                      (std::size_t)(data.GetSize()));
    boost::iostreams::stream<boost::iostreams::basic_array_source<char> > InStream(Source);
    boost::archive::binary_iarchive InputArchive(InStream);

    Classifier = new MAClassifier();
    InputArchive >> *Classifier;
  MC_CATCH_BEGIN
    MC_WARNING("Incompatible classifier encoding");
    return nullptr;
  MC_CATCH_END
  return Classifier;
}


MAModel* MAClassifier::ExportModel() const
{
  CvStatModel* Model = nullptr;
  cv::Algorithm* Model2 = nullptr;

  if (!Trained)
  {
    MC_WARNING("Can't export a not trained classifier");
    return nullptr;
  }
  // OpenCV classifiers
  if (Classifier >= MA::SvmClassifierCLinear && Classifier <= MA::SvmClassifierNuRbf)
  {
    Model = SvmClassifier.get();
  } else
  if (Classifier == MA::NeuralNetworkClassifier)
  {
    Model = NeuralNetwork.get();
  } else
  if (Classifier == MA::NaiveBayesClassifier)
  {
    Model = BayesClassifier.get();
  } else
  if (Classifier == MA::KNearestNeighborClassifier)
  {
    Model = KNearestClassifier.get();
  } else
  if (Classifier == MA::DecisionTreeClassifier)
  {
    Model = DecisionTree.get();
  } else
  if (Classifier == MA::RandomTreesClassifier)
  {
    Model = RandomTrees.get();
  } else
  if (Classifier == MA::ExtremeRandomTreesClassifier)
  {
    Model = ExtremeRandomTrees.get();
  } else
  if (Classifier == MA::GradientBoostedTreesClassifier)
  {
    Model = GradientBoostedTrees.get();
  } else
  if (Classifier == MA::ExpectationMaximization)
  {
    Model2 = EmClassifier->Classifier.get();
  }
  if (Model || Model2)
  {
    boost::scoped_ptr<cv::FileStorage> FileStorage;

    FileStorage.reset(new cv::FileStorage(".xml", cv::FileStorage::WRITE+cv::FileStorage::MEMORY));
    if (Model)
    {
      Model->write(**FileStorage, MA::CRMethodTypeStrs[(int)Classifier].c_str());
    } else {
      cvStartWriteStruct(**FileStorage, MA::CRMethodTypeStrs[(int)Classifier].c_str(), CV_NODE_MAP);
      Model2->write(*FileStorage);
      cvEndWriteStruct(**FileStorage);
    }
    *FileStorage << "FeatureCount" << (int)FeatureCount;
    MC::FloatList UniqueLabels(CachedUniqueLabels.begin(), CachedUniqueLabels.end());

    *FileStorage << "Labels" << MCEncodeToString(UniqueLabels, true);
    if (Classifier == MA::ExpectationMaximization)
    {
      *FileStorage << "LabelMapping" << MCEncodeToString(EmClassifier->LabelMapping, true);
    }
    *FileStorage << "PreprocessingMode" << MA::FeaturePreprocessingTypeStrs[(int)Preprocessing];
    *FileStorage << "PreprocessingCoefficients" << MCEncodeToString(PreprocessedData, true);
    return new MAModel(FileStorage->releaseAndGetString());
  }
  // Dlib classifiers
  std::ostringstream OutputBuffer;

  dlib::serialize(MA::CRMethodTypeStrs[(int)Classifier], OutputBuffer);
  if (Classifier == MA::SvmClassifierCLinearEkm || Classifier == MA::SvmClassifierCRbfEkm ||
      Classifier == MA::SvmClassifierCLinearDcd || Classifier == MA::SvmClassifierCLinearDlib)
  {
    DlibDModelSerializeType SaveModel = DlibFunctions->ClassifierDFunction;

    dlib::serialize(SaveModel, OutputBuffer);
  } else
  if (Classifier == MA::RvmClassifierLinear || Classifier == MA::RvmClassifierSigmoid ||
      Classifier == MA::RvmClassifierRbf || Classifier == MA::KrRegressionLinear ||
      Classifier == MA::KrRegressionRbf)
  {
    DlibFModelSerializeType SaveModel = DlibFunctions->ClassifierFFunction;

    dlib::serialize(SaveModel, OutputBuffer);
  } else {
    MC_WARNING("Unsupported classifier for model export (%s)", MA::CRMethodTypeStrs[(int)Classifier].c_str());
    return nullptr;
  }
  dlib::serialize((int)FeatureCount, OutputBuffer);
  MC::FloatList UniqueLabels(CachedUniqueLabels.begin(), CachedUniqueLabels.end());

  dlib::serialize(UniqueLabels, OutputBuffer);
  dlib::serialize(MA::FeaturePreprocessingTypeStrs[(int)Preprocessing], OutputBuffer);
  dlib::serialize(PreprocessedData, OutputBuffer);
  return new MAModel(OutputBuffer.str());
}


void MAClassifier::CreateClassifier()
{
  if (Classifier == MA::NeuralNetworkClassifier)
  {
    if (!NeuralNetwork.get())
      NeuralNetwork.reset(new CvANN_MLP);
  } else
  if (Classifier == MA::NaiveBayesClassifier)
  {
    if (!BayesClassifier.get())
      BayesClassifier.reset(new CvNormalBayesClassifier);
  } else
  if (Classifier == MA::KNearestNeighborClassifier)
  {
    if (!KNearestClassifier.get())
      KNearestClassifier.reset(new CvKNearest);
  } else
  if (Classifier >= MA::SvmClassifierCLinear && Classifier <= MA::SvmClassifierNuRbf)
  {
    if (!SvmClassifier.get())
      SvmClassifier.reset(new CvSVM);
    if (!SvmClassifierParams.get())
      SvmClassifierParams.reset(new CvSVMParams);
    if (Classifier >= MA::SvmClassifierCLinear && Classifier <= MA::SvmClassifierCRbf)
    {
      if (!Regression)
        SvmClassifierParams->svm_type = CvSVM::C_SVC;
      else
        SvmClassifierParams->svm_type = CvSVM::EPS_SVR;
    }
    if (Classifier >= MA::SvmClassifierNuLinear && Classifier <= MA::SvmClassifierNuRbf)
    {
      if (!Regression)
        SvmClassifierParams->svm_type = CvSVM::NU_SVC;
      else
        SvmClassifierParams->svm_type = CvSVM::NU_SVR;
    }
    if (Classifier == MA::SvmClassifierCLinear || Classifier == MA::SvmClassifierNuLinear)
      SvmClassifierParams->kernel_type = CvSVM::LINEAR;
    if (Classifier == MA::SvmClassifierCRbf || Classifier == MA::SvmClassifierNuRbf)
      SvmClassifierParams->kernel_type = CvSVM::RBF;
    SvmClassifierParams->C = SvmC;
    SvmClassifierParams->p = SvmP;
    SvmClassifierParams->nu = SvmNu;
    SvmClassifierParams->gamma = SvmGamma;
  } else
  if (Classifier == MA::DecisionTreeClassifier)
  {
    if (!DecisionTree.get())
      DecisionTree.reset(new CvDTree);
  } else
  if (Classifier == MA::RandomTreesClassifier)
  {
    if (!RandomTrees.get())
      RandomTrees.reset(new MARandomTrees);
  } else
  if (Classifier == MA::ExtremeRandomTreesClassifier)
  {
    if (!ExtremeRandomTrees.get())
      ExtremeRandomTrees.reset(new CvERTrees);
  } else
  if (Classifier == MA::GradientBoostedTreesClassifier)
  {
    if (!GradientBoostedTrees.get())
      GradientBoostedTrees.reset(new CvGBTrees);
  } else
  if (Classifier == MA::ExpectationMaximization)
  {
    if (!EmClassifier.get())
    {
      EmClassifier.reset(new CvEmWrapper);
      // Note: Default parameter EM::COV_MAT_DIAGONAL is changed to cv::EM::COV_MAT_GENERIC
      // because of http://stackoverflow.com/questions/23485982
      EmClassifier->Classifier.reset(new cv::EM(LabelCount, cv::EM::COV_MAT_GENERIC));
    }
#if defined(__unix__)
  } else
  if (Classifier == MA::GaussianProcessRegression)
  {
    if (!Gpr.get())
      Gpr.reset(new GpWrapper);
#endif
  }
}


template<class Archive>
void MAClassifier::load(Archive& archive, const unsigned int version)
{
  MC_UNUSED(version);
  int TempInt = 0;

  archive & TempInt;
  Classifier = (MA::CRMethodType)TempInt;
  archive & TempInt;
  Preprocessing = (MA::FeaturePreprocessingType)TempInt;
  // cppcheck-suppress clarifyCondition
  archive & Regression;
  archive & LabelCount;
  CreateClassifier();
  archive & SvmGamma;
  archive & SvmC;
  archive & SvmP;
  archive & SvmNu;
  // cppcheck-suppress clarifyCondition
  archive & SvmAutoTrain;
  archive & DlibEpsilon;
  archive & RvmSigmoidGamma;
  archive & RvmRbfGamma;
  archive & KrrGamma;
  archive & KrrLambda;
  archive & MeL1;
  archive & MeL2;
  archive & LwprAlpha;
  archive & PlsrComponents;
  archive & KrlsTolerance;
  archive & KrlsGamma;
  archive & TreeMaxDepth;
  archive & TreeNodeSampleLimit;
  archive & RtMaxForestSize;
  // The reset applies the learning parameters
  Reset();
  // Feature count must be restored after loading and applying the learning parameters
  archive & FeatureCount;
  // Load samples and train classifier
  archive & CachedSamples;
  archive & CachedLabels;
  MC::FloatList Labels;

  archive & Labels;
  CachedUniqueLabels = std::set<float>(Labels.begin(), Labels.end());
  if (Classifier == MA::ExpectationMaximization)
  {
    archive & EmClassifier->LabelMapping;
  }
  archive & PrioritizedClasses;
  archive & PriorityLabels;
  Trained = false;
}


template<class Archive>
void MAClassifier::save(Archive& archive, const unsigned int version) const
{
  MC_UNUSED(version);
  int TempInt = (int)Classifier;

  archive & TempInt;
  TempInt = (int)Preprocessing;
  archive & TempInt;
  // cppcheck-suppress clarifyCondition
  archive & Regression;
  archive & LabelCount;
  archive & SvmGamma;
  archive & SvmC;
  archive & SvmP;
  archive & SvmNu;
  // cppcheck-suppress clarifyCondition
  archive & SvmAutoTrain;
  archive & DlibEpsilon;
  archive & RvmSigmoidGamma;
  archive & RvmRbfGamma;
  archive & KrrGamma;
  archive & KrrLambda;
  archive & MeL1;
  archive & MeL2;
  archive & LwprAlpha;
  archive & PlsrComponents;
  archive & KrlsTolerance;
  archive & KrlsGamma;
  archive & TreeMaxDepth;
  archive & TreeNodeSampleLimit;
  archive & RtMaxForestSize;
  // Note: Feature count must be restored after loading and applying the learning parameters
  archive & FeatureCount;
  archive & CachedSamples;
  archive & CachedLabels;
  MC::FloatList Labels(CachedUniqueLabels.begin(), CachedUniqueLabels.end());

  archive & Labels;
  if (Classifier == MA::ExpectationMaximization)
  {
    archive & EmClassifier->LabelMapping;
  }
  archive & PrioritizedClasses;
  archive & PriorityLabels;
}

// Explicit template instantiation
// http://stackoverflow.com/questions/2152002/how-do-i-force-a-particular-instance-of-a-c-template-to-instantiate
// Post by Alexander Poluektov
template void MAClassifier::load<boost::archive::binary_iarchive>(boost::archive::binary_iarchive&,
                                                                  const unsigned int);
template void MAClassifier::save<boost::archive::binary_oarchive>(boost::archive::binary_oarchive&,
                                                                  const unsigned int) const;
template void MAClassifier::load<eos::portable_iarchive>(eos::portable_iarchive&, const unsigned int);
template void MAClassifier::save<eos::portable_oarchive>(eos::portable_oarchive&, const unsigned int) const;