MCDataStorage.cpp

/*
 *  This file is part of the AiBO+ project
 *
 *  Copyright (C) 2005-2016 Csaba Kertész (csaba.kertesz@gmail.com)
 *
 *  AiBO+ is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  AiBO+ is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Street #330, Boston, MA 02111-1307, USA.
 *
 */

#include "MCDataStorage.hpp"

#include "MCBinaryData.hpp"
#include "MCDataContainer.hpp"
#include "MCLog.hpp"
#include "MCThreadLocalData.hpp"
#include "MCZipper.hpp"

#include <boost/archive/binary_iarchive.hpp>
#include <boost/archive/binary_oarchive.hpp>
#include <boost/iostreams/device/back_inserter.hpp>
#include <boost/iostreams/stream.hpp>

#include <fstream>

namespace
{
// Verbose mode
MCThreadLocalData<bool> Verbose(true);

void CheckStaticDataStorageVariables()
{
  if (unlikely(!Verbose.get()))
  {
    Verbose.reset(new bool);
    *Verbose = false;
  }
}
}

std::string MCDataStorage::VersionID = "DataStorage 1.00";

MCDataStorage::MCDataStorage(const std::string& name, bool portable) : Name(name),
  Portable(portable)
{
  CheckStaticDataStorageVariables();
  if (unlikely(IsVerbose()))
  {
    MC_LOG("Storage: %s - ctor", Name.c_str());
  }
}


MCDataStorage::~MCDataStorage()
{
  if (unlikely(IsVerbose()))
  {
    MC_LOG("Storage: %s - dtor", Name.c_str());
  }
  Clear();
}


bool MCDataStorage::IsPortable() const
{
  return Portable;
}


bool MCDataStorage::IsCompact() const
{
  return DataContainers.empty();
}


std::string MCDataStorage::GetName() const
{
  return Name;
}


void MCDataStorage::SetName(const std::string& name)
{
  Name = name;
}


void MCDataStorage::DumpContainerNames() const
{
  MC::StringList ContainerNames = GetContainerNames();

  printf("Containers (%d): ", (int)ContainerNames.size());
  for (unsigned int i = 0; i < ContainerNames.size(); ++i)
    printf("%s, ", ContainerNames[i].c_str());
  printf("\n");
}


bool MCDataStorage::LoadFromFile(const std::string& file_name)
{
  MC_TRY_BEGIN
    std::ifstream InputFileStream(file_name.c_str(), std::ios::binary);

    if (!InputFileStream.is_open())
    {
      MC_WARNING("Unable to open file to load binary data: %s", file_name.c_str());
      return false;
    }
    InputFileStream.seekg(0, std::ios::end);
    int FileSize = (int)InputFileStream.tellg();

    InputFileStream.seekg(0, std::ios::beg);
    LoadFromInputStream(InputFileStream, FileSize);
    InputFileStream.close();
  MC_CATCH_BEGIN
    MC_WARNING("Incompatible encoded data storage file (%s)", file_name.c_str());
    return false;
  MC_CATCH_END
  return true;
}


bool MCDataStorage::SaveToFile(const std::string& file_name)
{
  MC_TRY_BEGIN
    std::ofstream OutputFileStream(file_name.c_str(), std::ios::out | std::ios::binary);

    if (!OutputFileStream.is_open())
    {
      MC_WARNING("Unable to create file to save binary data: %s", file_name.c_str());
      return false;
    }
    SaveToOutputStream(OutputFileStream);
    OutputFileStream.flush();
    OutputFileStream.close();
  MC_CATCH_BEGIN
    MC_WARNING("Unable to save encoded data storage file (%s)", file_name.c_str());
    return false;
  MC_CATCH_END
  return true;
}


MCDataStorage* MCDataStorage::Decode(const MCBinaryData& data, bool portable)
{
  MCDataStorage* DataStorage = new MCDataStorage("", portable);

  MC_TRY_BEGIN
    boost::iostreams::basic_array_source<char> Source((char*)data.GetData(), (std::size_t)data.GetSize());
    boost::iostreams::stream<boost::iostreams::basic_array_source<char> > InStream(Source);

    DataStorage->LoadFromInputStream(InStream, data.GetSize());
  MC_CATCH_BEGIN
    MC_WARNING("Incompatible encoded data storage version");
    delete DataStorage;
    return nullptr;
  MC_CATCH_END
  return DataStorage;
}


MCBinaryData* MCDataStorage::Encode()
{
  // Huh, idea is from:
  // http://boost.2283326.n4.nabble.com/in-memory-stream-binary-archive-serialization-td2578880.html
  std::vector<char> TempBuffer;

  boost::iostreams::stream<boost::iostreams::back_insert_device<std::vector<char> > > OutStream(TempBuffer);

  SaveToOutputStream(OutStream);
  MCBinaryData* BinaryData = new MCBinaryData((int)TempBuffer.size());

  memcpy(BinaryData->GetData(), &TempBuffer[0], TempBuffer.size());
  return BinaryData;
}


void MCDataStorage::LoadFromInputStream(std::istream& input_stream, unsigned int max_bytes)
{
  eos::portable_iarchive* InputPArchive = nullptr;
  boost::archive::binary_iarchive* InputArchive = nullptr;
  std::string DataStorageID;

  if (Portable)
  {
    InputPArchive = new eos::portable_iarchive(input_stream);
    *InputPArchive & DataStorageID;
  } else {
    InputArchive = new boost::archive::binary_iarchive(input_stream);
    *InputArchive & DataStorageID;
  }
  if (DataStorageID != VersionID)
  {
    MC_WARNING("The archive is not a data storage.");
    delete InputPArchive;
    delete InputArchive;
    return;
  }
  if (InputArchive)
  {
    *InputArchive & Name;
  } else {
    *InputPArchive & Name;
  }
  Clear();
  while (input_stream.tellg() < max_bytes)
  {
    std::string ContainerName;
    MCBinaryData* ContainerData = new MCBinaryData;
    bool Compressed = false;

    if (InputArchive)
    {
      *InputArchive & ContainerName;
      // cppcheck-suppress clarifyCondition
      *InputArchive & Compressed;
      *InputArchive & *ContainerData;
    } else {
      *InputPArchive & ContainerName;
      // cppcheck-suppress clarifyCondition
      *InputPArchive & Compressed;
      *InputPArchive & *ContainerData;
    }
    if (Compressed)
    {
      MCBinaryData* OutputData = new MCBinaryData;

      // The best compromise is the zlib (fast) method because it is pretty quick for
      // data containers and it compresses best over every algorithms except bzip2.
      // See more in comments of SaveToOutputStream()
      if (MCZipper::DetectCompressionMethod(*ContainerData, MC::ZlibCompression) == MC::ZlibCompression)
      {
        MCZipper::DecompressZlib(*ContainerData, *OutputData);
      } else {
        // This is left for backward compatibility of old saved sessions (it can lead to crashes)
        MCZipper::DecompressQuickLz(*ContainerData, *OutputData, true);
      }
      // Drop the malformed containers otherwise crash is very likely later
      if (OutputData->GetSize() > 0)
        BinaryContainers.insert(std::make_pair(ContainerName, OutputData));
      delete ContainerData;
      ContainerData = nullptr;
    } else {
      BinaryContainers.insert(std::make_pair(ContainerName, ContainerData));
    }
  }
  delete InputPArchive;
  delete InputArchive;
}


void MCDataStorage::SaveToOutputStream(std::ostream& output_stream)
{
  eos::portable_oarchive* OutputPArchive = nullptr;
  boost::archive::binary_oarchive* OutputArchive = nullptr;

  if (Portable)
  {
    OutputPArchive = new eos::portable_oarchive(output_stream);
    *OutputPArchive & VersionID;
    *OutputPArchive & Name;
  } else {
    OutputArchive = new boost::archive::binary_oarchive(output_stream);
    *OutputArchive & VersionID;
    *OutputArchive & Name;
  }
  // Encode the containers again
  for (auto& container : DataContainers)
  {
    bool Encoded = false;

    for (auto& container2 : BinaryContainers)
    {
      if (container.first == container2.first)
      {
        Encoded = true;
        delete container2.second;
        container2.second = container.second->Encode();
        break;
      }
    }
    if (!Encoded)
    {
      BinaryContainers.insert(std::make_pair(container.first, container.second->Encode()));
    }
  }
  // Write data containers
  for (auto& container : BinaryContainers)
  {
    // Compress the data if a container size is bigger than 2 KB
    MCBinaryData* EncodedData = container.second;
    bool Compressing = EncodedData->GetSize() > 2000;

    if (OutputArchive)
    {
      *OutputArchive & container.first;
      // cppcheck-suppress clarifyCondition
      *OutputArchive & Compressing;
    } else {
      *OutputPArchive & container.first;
      // cppcheck-suppress clarifyCondition
      *OutputPArchive & Compressing;
    }
    if (Compressing)
    {
      MCBinaryData CompressedData;

      /*
       * The best compromise is the zlib (fast) method because it is pretty quick for
       * data containers and it compresses best over every algorithms except bzip2.
       *
       * Compression size and speed (Lz4/Medium): 7110 bytes, 526320 bytes / msec
       * Decompression speed (Lz4/Medium): 547373 bytes / msec
       * Compression size and speed (Zlib/Fast): 4443 bytes, 123282 bytes / msec
       * Decompression speed (Zlib/Fast): 234588 bytes / msec
       * Compression size and speed (Zlib/Medium): 3714 bytes, 43259 bytes / msec
       * Decompression speed (Zlib/Medium): 295345 bytes / msec
       * Compression size and speed (Zlib/Best): 3640 bytes, 36426 bytes / msec
       * Decompression speed (Zlib/Best): 235936 bytes / msec
       * Compression size and speed (QuickLZ/Medium): 5684 bytes, 37871 bytes / msec
       * Decompression speed (QuickLZ/Medium): 912288 bytes / msec
       * Compression size and speed (MiniLzo/Medium): 6076 bytes, 494614 bytes / msec
       * Decompression speed (MiniLzo/Medium): 746418 bytes / msec
       * Compression size and speed (Lz4/Medium): 6576 bytes, 366544 bytes / msec
       * Decompression speed (Lz4/Medium): 1001292 bytes / msec
       * Compression size and speed (Snappy/Medium): 8026 bytes, 331072 bytes / msec
       * Decompression speed (Snappy/Medium): 720228 bytes / msec
       * Compression size and speed (Bzip2/Fast): 2266 bytes, 1956 bytes / msec
       * Decompression speed (Bzip2/Fast): 35299 bytes / msec
       * Compression size and speed (Bzip2/Medium): 2266 bytes, 1962 bytes / msec
       * Decompression speed (Bzip2/Medium): 31848 bytes / msec
       * Compression size and speed (Bzip2/Best): 2266 bytes, 2173 bytes / msec
       * Decompression speed (Bzip2/Best): 49224 bytes / msec
       */
      MCZipper::CompressZlib(*EncodedData, CompressedData, MC::BestCompression);
      if (OutputArchive)
      {
        *OutputArchive & CompressedData;
      } else {
        *OutputPArchive & CompressedData;
      }
    } else {
      if (OutputArchive)
      {
        *OutputArchive & *EncodedData;
      } else {
        *OutputPArchive & *EncodedData;
      }
    }
  }
  delete OutputPArchive;
  delete OutputArchive;
}


MCDataContainer* MCDataStorage::DecodeContainer(const MCBinaryData& data)
{
  return MCDataContainer::Decode(data, Portable);
}


MC::StringList MCDataStorage::GetContainerNames() const
{
  MC::StringList FinalNames = MCGetContainerKeys<std::string>(DataContainers);
  MC::StringList OtherNames = MCGetContainerKeys<std::string>(BinaryContainers);

  for (auto& name : OtherNames)
  {
    if (!MCContainerContains(FinalNames, name))
    {
      FinalNames.push_back(name);
    }
  }
  // Guarantee the sorted container name list
  std::sort(FinalNames.begin(), FinalNames.end());
  return FinalNames;
}


MC::DataContainerSPtr MCDataStorage::CreateContainer(const std::string& container_name)
{
  for (auto& container : DataContainers)
  {
    if (container.first == container_name)
    {
      MC_WARNING("Can't add new container with the same name (%s)", container_name.c_str());
      return MC::DataContainerSPtr();
    }
  }
  for (auto& container : BinaryContainers)
  {
    if (container.first == container_name)
    {
      MC_WARNING("Can't add new container with the same name (%s)", container_name.c_str());
      return MC::DataContainerSPtr();
    }
  }
  MC::DataContainerSPtr NewContainer(new MCDataContainer(container_name, Portable));

  DataContainers.insert(std::make_pair(container_name, NewContainer));
  return NewContainer;
}


MC::DataContainerSPtr MCDataStorage::GetContainer(const std::string& container_name)
{
  // Get a data container
  for (auto& container : DataContainers)
  {
    if (container.first == container_name)
      return container.second;
  }
  // Decode and return a data container
  for (auto& container : BinaryContainers)
  {
    if (container.first == container_name)
    {
      MC::DataContainerSPtr DataContainer(DecodeContainer(*container.second));

      return DataContainers.insert(std::make_pair(container.first, DataContainer)).first->second;
    }
  }
  return MC::DataContainerSPtr();
}


void MCDataStorage::OptimizeCachedContainers()
{
  for (auto iter = DataContainers.begin(); iter != DataContainers.end();)
  {
    if (iter->second.use_count() == 1)
    {
      bool Encoded = false;

      for (auto& container : BinaryContainers)
      {
        if (iter->first == container.first)
        {
          Encoded = true;
          delete container.second;
          container.second = iter->second->Encode();
          break;
        }
      }
      if (!Encoded)
      {
        BinaryContainers.insert(std::make_pair(iter->first, iter->second->Encode()));
      }
      if (unlikely(IsVerbose()))
      {
        MC_LOG("Storage: %s - drop container: %s", Name.c_str(), iter->first.c_str());
      }
      // Note: Erasing from map/multimap while iterating: see http://stackoverflow.com/questions/446205
      DataContainers.erase(iter++);
    } else {
      ++iter;
    }
  }
}


void MCDataStorage::RemoveContainer(const std::string& container_name)
{
  for (auto iter = DataContainers.begin(); iter != DataContainers.end(); ++iter)
  {
    if (iter->first == container_name)
    {
      if (unlikely(IsVerbose()))
      {
        MC_LOG("Storage: %s - remove container: %s", Name.c_str(), iter->first.c_str());
      }
      DataContainers.erase(iter);
      break;
    }
  }
  DataContainers.clear();
  for (auto iter = BinaryContainers.begin(); iter != BinaryContainers.end(); ++iter)
  {
    if (iter->first == container_name)
    {
      if (unlikely(IsVerbose()))
      {
        MC_LOG("Storage: %s - remove container (binary): %s", Name.c_str(), iter->first.c_str());
      }
      delete iter->second;
      BinaryContainers.erase(iter);
      break;
    }
  }
}


void MCDataStorage::Clear()
{
  if (unlikely(IsVerbose()))
  {
    MC_LOG("Storage: %s - clear", Name.c_str());
  }
  DataContainers.clear();
  for (auto& container : BinaryContainers)
  {
    delete container.second;
  }
  BinaryContainers.clear();
}


void MCDataStorage::SetVerbose(bool new_state)
{
  CheckStaticDataStorageVariables();
  *Verbose = new_state;
}


bool MCDataStorage::IsVerbose()
{
  CheckStaticDataStorageVariables();
  return *Verbose;
}