Added buffered writing and removed text files debug info

include/DatSubsystems/DatIO.h

@@ -60,6 +60,12 @@ namespace LOTRO_DAT {
 
         DatOperationResult<> ReadSuperBlock();
 
+        //------------------------------------------------//
+        // PRIVATE READ-WRITE SECTION
+        //------------------------------------------------//
+
+        void UpdateBufferIfNeeded(long long size_to_write);
+
         //------------------------------------------------//
         // PRIVATE DEINIT SECTION
         //------------------------------------------------//
@@ -87,6 +93,8 @@ namespace LOTRO_DAT {
 
     private:
         long long actual_dat_size_;
+        long long elapsed_eof_buffer_;
+        const unsigned MAX_EOF_BUFFER = 15 * 1024 * 1024; /// Maximal size of buffer, which is written to the end of dat file to improve write speed of small fragments
     };
 }
 };

src/DatSubsystems/DatIO.cpp

@@ -6,6 +6,8 @@
 #include "DatSubsystems/DatIO.h"
 #include <algorithm>
 #include <locale>
+#include <DatSubsystems/DatIO.h>
+
 
 #ifdef WIN32
 #define fseek _fseeki64
@@ -24,7 +26,7 @@ namespace LOTRO_DAT {
      * \param[in] datFile Указатель на объект управляющего класса DatFile
      */
 
-    DatIO::DatIO(DatFile *datFile) : dat(datFile), file_handler_(nullptr), filename_("none"), actual_dat_size_(0) {
+    DatIO::DatIO(DatFile *datFile) : dat(datFile), file_handler_(nullptr), filename_("none"), actual_dat_size_(0), elapsed_eof_buffer_(0) {
     }
 
 
@@ -188,6 +190,32 @@ namespace LOTRO_DAT {
     }
 
 
+    /*!
+     * \author Gi1dor
+     * \date 10.07.2018
+     *
+     * Проверка буфера перед записью новых size_to_write байт в конец файла. Если буфер мал - то дозаписывается
+     * новый чистый блок размера MAX_EOF_BUFFER (константное поле класса DatIO);
+     *
+     * \param[in] size_to_write Размер данных в байтах, которые будут сейчас записаны в конец файла
+     *
+     */
+
+    void DatIO::UpdateBufferIfNeeded(long long size_to_write) {
+        if (elapsed_eof_buffer_ >= size_to_write) {
+            elapsed_eof_buffer_ -= size_to_write;
+            return;
+        }
+
+        BinaryData empty_buffer(std::max(MAX_EOF_BUFFER, unsigned(size_to_write)));
+
+        fseek(file_handler_, 0, SEEK_END);
+        fwrite(empty_buffer.data(), empty_buffer.size(), 1, file_handler_);
+        actual_dat_size_ += empty_buffer.size();
+        elapsed_eof_buffer_ += empty_buffer.size() - size_to_write;
+    }
+
+
     /*!
      * \author Gi1dor
      * \date 30.06.2018
@@ -207,16 +235,15 @@ namespace LOTRO_DAT {
         if (file_handler_ == nullptr)
             return DatOperationResult<>(ERROR, "IOWRITEDATA: file handler is null pointer on writing data.");
 
-        if (offset != ftell(file_handler_))
-            fseek(file_handler_, offset, SEEK_SET);
-
         if (data_offset + size > data.size())
             return DatOperationResult<>(ERROR, "IOWRITEDATA: writing more than BinaryData size.");
 
-        fwrite(data.data() + data_offset, unsigned(size), 1, file_handler_);
+        UpdateBufferIfNeeded(size);
 
-        if (offset + size > actual_dat_size_)
-            actual_dat_size_ = offset + size;
+        if (offset != ftell(file_handler_))
+            fseek(file_handler_, offset, SEEK_SET);
+
+        fwrite(data.data() + data_offset, unsigned(size), 1, file_handler_);
 
         return DatOperationResult<>(SUCCESS, "Data writing successful.");
     }
@@ -346,6 +373,7 @@ namespace LOTRO_DAT {
         fprintf(file, "fragment_journal_size = %lld\n", fragmentation_journal_size);
         fprintf(file, "root_dir_offset = %lld\n", root_directory_offset);
         fprintf(file, "free_dat_size = %lld\n", free_dat_size);
+        fprintf(file, "DatLibrary: EOF buffer size constant = %d\n", MAX_EOF_BUFFER);
     }
 
 }

src/Subfiles/TextSubFile.cpp

@@ -137,7 +137,7 @@ namespace LOTRO_DAT {
         // first 4 bytes - file_id, then 4 bytes - unknown, then 1 byte - unknown
 
         if (old_data.ToNumber<4>(8) != file_id_) {
-            std::cout << "Error: file_id mismatch. Found " << old_data.ToNumber<4>(8) << ", expected: " << file_id_
+            LOG(WARNING) << "Error: file_id mismatch. Found " << old_data.ToNumber<4>(8) << ", expected: " << file_id_
                       << "\n";
         }
 
@@ -312,13 +312,13 @@ namespace LOTRO_DAT {
                                    + u" gid:" + to_utf16(new_data.fragment_id);
 
         // Deleting '[' and ']' brackets
-        std::u16string text_data = new_data.text.substr(1, new_data.text.size() - 2) + file_data;
+        std::u16string text_data = new_data.text.substr(1, new_data.text.size() - 2);// + file_data;
 
 
-        if (file_id_ == 620757423 && new_data.fragment_id == 96627013) {
-            std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> codecvt;
-            std::cout << "AAAAAAAAAAAAA: " << codecvt.to_bytes(text_data) << std::endl;
-        }
+//        if (file_id_ == 620757423 && new_data.fragment_id == 96627013) {
+//            std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> codecvt;
+//            std::cout << "AAAAAAAAAAAAA: " << codecvt.to_bytes(text_data) << std::endl;
+//        }
         text_pieces_.clear();
 
         const std::u16string DNT = u"<--DO_NOT_TOUCH!-->";
@@ -334,14 +334,13 @@ namespace LOTRO_DAT {
 
         text_pieces_.push_back(text_data.substr(prev));
 
-        if (file_id_ == 620757423 && new_data.fragment_id == 96627013) {
-            std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> codecvt;
-            std::cout << "PIECES: SIZE = " << text_pieces_.size() << "..." << std::endl;
-            for (const auto &i : text_pieces_) {
-                std::cout << "   PIECE='" << codecvt.to_bytes(i) << "'\n";
-            }
-        }
-
+//        if (file_id_ == 620757423 && new_data.fragment_id == 96627013) {
+//            std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> codecvt;
+//            std::cout << "PIECES: SIZE = " << text_pieces_.size() << "..." << std::endl;
+//            for (const auto &i : text_pieces_) {
+//                std::cout << "   PIECE='" << codecvt.to_bytes(i) << "'\n";
+//            }
+//        }
 
         // Building BinaryData from pieces
         unsigned buffer_offset = 0;