Merge branch 'text_patches' into 'master'

Text patches to master

See merge request LotRO-dev-tools/Universal_dat_library!2

Source/BinaryData.cpp

@@ -5,6 +5,7 @@
 #include "BinaryData.h"
 #include "Common/DatException.h"
 #include "Common/ZLib/zlib.h"
+#include <algorithm>
 
 extern  "C++"
 {
@@ -16,7 +17,7 @@ namespace LOTRO_DAT {
     }
 
     BinaryData::BinaryData(const BinaryData &d) {
-		size_ = d.size_;
+        size_ = d.size_;
         data_ = new unsigned char[size_];
         memcpy(data_, d.data_, size_);
     }
@@ -28,7 +29,7 @@ namespace LOTRO_DAT {
     }
 
     BinaryData::BinaryData(unsigned int size) {
-		data_ = new unsigned char[size];
+        data_ = new unsigned char[size];
         size_ = size;
     }
 
@@ -112,20 +113,11 @@ namespace LOTRO_DAT {
     // Makes data from specified T bytes of number in raw
     template<int T>
     void BinaryData::FromNumberRAW(const long long &number) {
-        if (T < 0)
-            throw DatException("Bad BinaryData::FromNumber() - trying to make data from amount of bytes < 0");
+        if (T <= 0)
+            throw DatException("Bad BinaryData::FromNumber() - trying to make data from amount of bytes <= 0");
         try {
-            if (size_ != 0 && data_ != nullptr)
-                delete[] data_;
-
-            size_ = size_t(T);
-            data_ = new unsigned char[size_];
-
-            for (size_t i = size_ - 1; i >= 0; i--)
-            {
-                data_[i] = (unsigned char)((number >> (8 * i)) & 0xFF);
-            }
-
+            FromNumber<T>(number);
+            std::reverse(data_, data_ + size());
         } catch (...) {
             throw DatException("Bad BinaryData::ToNumber(). Error in using template function", DATA_EXCEPTION);
         }
@@ -157,21 +149,21 @@ namespace LOTRO_DAT {
         return data_;
     }
 
-	bool BinaryData::WriteToFile(const char *filename) const {
-		FILE *f;
-		fopen_s(&f, filename, "wb");
-		if (f == nullptr) {
-			throw DatException("Bad BinaryData::WriteToFile() - unable to open output file", EXPORT_EXCEPTION);
-		}
+    bool BinaryData::WriteToFile(const char *filename) const {
+        FILE *f;
+        fopen_s(&f, filename, "wb");
+        if (f == nullptr) {
+            throw DatException("Bad BinaryData::WriteToFile() - unable to open output file", EXPORT_EXCEPTION);
+        }
 
-		fwrite(data(), size(), 1, f);
-		fclose(f);
-		return true;
-	}
+        fwrite(data(), size(), 1, f);
+        fclose(f);
+        return true;
+    }
 
-	bool BinaryData::WriteToFile(const std::string &filename) const {
-		return WriteToFile(filename.c_str());
-	}
+    bool BinaryData::WriteToFile(const std::string &filename) const {
+        return WriteToFile(filename.c_str());
+    }
 
 
     void BinaryData::ReadFromFile(const char *filename) {
@@ -210,14 +202,14 @@ namespace LOTRO_DAT {
 
         uLongf new_size = max_unpacked_size;
         int res = uncompress(decompressed.data_, &new_size, data_ + offset, size_ - offset);
-		
-		if (res != 0) {
-			throw DatException("Bad BinaryData::DecompressData() - uncompress() failed!", DATA_EXCEPTION);
-		}
 
-		decompressed.size_ = (unsigned int)new_size;
-		return decompressed;
-	}
+        if (res != 0) {
+            throw DatException("Bad BinaryData::DecompressData() - uncompress() failed!", DATA_EXCEPTION);
+        }
+
+        decompressed.size_ = (unsigned int)new_size;
+        return decompressed;
+    }
 
     BinaryData BinaryData::CompressData(unsigned int offset) const {
         const unsigned max_file_size = 1024 * 1024 * 40; // Setting 40 MB as a maximum packed data;
@@ -280,6 +272,5 @@ namespace LOTRO_DAT {
     template void BinaryData::FromNumberRAW<2>(const long long &);
     template void BinaryData::FromNumberRAW<4>(const long long &);
     template void BinaryData::FromNumberRAW<8>(const long long &);
-
 }
 }

Source/DatFile.cpp

@@ -87,20 +87,15 @@ namespace LOTRO_DAT {
             return false;
         }
 
-        std::vector<SubfileData> export_data;
-
         try {
-            export_data = dictionary_[file_id]->PrepareForExport(file_data);
+            SubfileData export_data = dictionary_[file_id]->PrepareForExport(file_data);
+            export_data.binary_data.WriteToFile(path + export_data.options["ext"].as<std::string>());
         } catch (std::exception &e) {
             fprintf(stderr, "Caught %s exception.", e.what());
 
             fprintf(stderr, "Unable to extract file due to uncaught exception while preparing file for export. Passing...\n");
             return false;
         }
-
-        for (int i = 0; i < export_data.size(); ++i) {
-            export_data[i].binary_data.WriteToFile(path + "_" + std::to_string(i) + export_data[i].options["ext"].as<std::string>());
-        }
         return true;
     }
 
@@ -127,10 +122,11 @@ namespace LOTRO_DAT {
             return false;
         }
 
-        std::vector<SubfileData> export_data;
+        SubfileData export_data;
 
         try {
             export_data = dictionary_[file_id]->PrepareForExport(file_data);
+            export_data.options["did"] = dat_id_;
         } catch (std::exception &e) {
             fprintf(stderr, "Caught %s exception.", e.what());
 
@@ -138,16 +134,19 @@ namespace LOTRO_DAT {
             return false;
         }
 
-        for (int i = 0; i < export_data.size(); ++i) {
-            try {
-                db->PushFile(export_data[i]);
-            } catch (std::exception &e) {
-                fprintf(stderr, "Caught %s exception.", e.what());
-                printf("Caught %s exception.", e.what());
-                fflush(stdout);
+        if (export_data == SubfileData()) {
+            fprintf(stderr, "WARNING: file with id %lld is too small. Passing it\n", dictionary_[file_id]->file_id());
+            return true;
+        }
 
-                fprintf(stderr, "Unable to put file or it's part to database. Continuing without this part. Database may be not complete\n");
-            }
+        try {
+            db->PushFile(export_data);
+        } catch (std::exception &e) {
+            fprintf(stderr, "Caught %s exception.", e.what());
+            printf("Caught %s exception.", e.what());
+            fflush(stdout);
+
+            fprintf(stderr, "Unable to put file or it's part to database. Continuing without this part. Database may be not complete\n");
         }
         return true;
     }
@@ -232,9 +231,33 @@ namespace LOTRO_DAT {
         return true;
     }
 
-    // TODO: Write description and make asserts and IMPLEMENT
+    // TODO: Write description and make asserts
     bool DatFile::PatchAllDatabase(Database *db) {
-        return false;
+        SubfileData data;
+        try {
+            data = db->GetNextFile();
+        } catch (std::exception &e) {
+            fprintf(stderr, "Caught %s exception.\n", e.what());
+            fprintf(stderr, "DatFile::PatchAllDatabase() error! Caught exception while fetching file from database! Stopping...\n");
+            return false;
+        }
+        while (data != SubfileData()) {
+            try {
+                PatchFile(data);
+            } catch (std::exception &e) {
+                fprintf(stderr, "Caught %s exception.\n", e.what());
+                fprintf(stderr, "DatFile::PatchAllDatabase() error! Caught exception while patching file! Passing...\n");
+            }
+
+            try {
+                data = db->GetNextFile();
+            } catch (std::exception &e) {
+                fprintf(stderr, "Caught %s exception.\n", e.what());
+                fprintf(stderr, "DatFile::PatchAllDatabase() error! Caught exception while fetching file from database! Stopping...\n");
+                return false;
+            }
+        }
+        return true;
     }
 
     /// DatFile::WriteUnorderedDictionary(...);
@@ -268,9 +291,9 @@ namespace LOTRO_DAT {
         if (file->file_id() != mfile_id.ToNumber<4>(0))
             throw DatException("Bad DatFile::GetFileData() - file_id in Subfile doesn't match to file_id in DatFile.", READ_EXCEPTION);
 
-        BinaryData data((unsigned)(file->file_size()));
+        BinaryData data((unsigned)(file->file_size() + (8 - offset)));
         if (file->block_size() >= file->file_size() + 8) {
-            ReadData(data, file->file_size(), file->file_offset() + offset);
+            ReadData(data, file->file_size() + (8 - offset), file->file_offset() + offset);
             return data;
         }
 

Source/Subfile.cpp

@@ -6,6 +6,7 @@
 #include "BinaryData.h"
 #include "DatFile.h"
 #include "Common/DatException.h"
+#include "SubfileData.h"
 
 #include <algorithm>
 
@@ -94,7 +95,7 @@ namespace LOTRO_DAT {
     ///  3) field options - YAML field, which consists of some parameters of file such as file_id, extension and so on.
     /// Returns true if preparation was success. Otherwise returns false;
 
-    std::vector<SubfileData> Subfile::PrepareForExport(const BinaryData &file_data) {
+    SubfileData Subfile::PrepareForExport(const BinaryData &file_data) {
         throw DatException("Bad Subfile::PrepareForExport() - function is not implemented for this type.", EXPORT_EXCEPTION);
     }
 

Source/Subfile.h

@@ -32,7 +32,7 @@ namespace LOTRO_DAT
         virtual FILE_TYPE FileType() const;
         virtual std::string Extension() const;
 
-        virtual std::vector<SubfileData> PrepareForExport(const BinaryData &file_data);
+        virtual SubfileData PrepareForExport(const BinaryData &file_data);
 
         virtual BinaryData MakeForImport(const BinaryData &old_data, const SubfileData &data);
 

Source/Subfiles/DdsSubfile.cpp

@@ -27,7 +27,7 @@ namespace LOTRO_DAT {
         return ".dds";
     }
 
-    std::vector<SubfileData> DdsSubfile::PrepareForExport(const BinaryData &file_data) {
+    SubfileData DdsSubfile::PrepareForExport(const BinaryData &file_data) {
         BinaryData data = file_data;
         if (data.CheckCompression())
             data = data.DecompressData(4);
@@ -135,12 +135,11 @@ namespace LOTRO_DAT {
                 throw DatException("Bad DdsSubfile::PrepareAsBinary() - unknown header format.", EXPORT_EXCEPTION);
         }
 
-        std::vector<SubfileData> output(1);
-        output[0].binary_data = (ddsData);
-        output[0].text_data = (u"");
-        output[0].options["fid"] = file_id();
-        output[0].options["ext"] = Extension();
-        return output;
+        SubfileData result;
+        result.binary_data = ddsData;
+        result.options["fid"] = file_id();
+        result.options["ext"] = Extension();
+        return result;
     }
 
     BinaryData DdsSubfile::MakeForImport(const BinaryData &old_data, const SubfileData &data) {

Source/Subfiles/DdsSubfile.h

@@ -20,7 +20,7 @@ namespace LOTRO_DAT {
 
         std::string Extension() const override;
 
-        std::vector<SubfileData> PrepareForExport(const BinaryData &file_data) override;
+        SubfileData PrepareForExport(const BinaryData &file_data) override;
 
         BinaryData MakeForImport(const BinaryData &old_data, const SubfileData &data) override;
     };

Source/Subfiles/FontSubfile.cpp

@@ -26,13 +26,12 @@ namespace LOTRO_DAT {
         return ".fontbin";
     }
 
-    std::vector<SubfileData> FontSubfile::PrepareForExport(const BinaryData &file_data) {
-        std::vector<SubfileData> output(1);
-        output[0].binary_data = file_data;
-        output[0].text_data = u"";
-        output[0].options["fid"] = file_id();
-        output[0].options["ext"] = Extension();
-        return output;
+    SubfileData FontSubfile::PrepareForExport(const BinaryData &file_data) {
+        SubfileData result;
+        result.binary_data = file_data;
+        result.options["fid"] = file_id();
+        result.options["ext"] = Extension();
+        return result;
     }
 
     BinaryData FontSubfile::MakeForImport(const BinaryData &old_data, const SubfileData &data) {

Source/Subfiles/FontSubfile.h

@@ -20,7 +20,7 @@ namespace LOTRO_DAT {
 
         std::string Extension() const override;
 
-        std::vector<SubfileData> PrepareForExport(const BinaryData &file_data) override;
+        SubfileData PrepareForExport(const BinaryData &file_data) override;
 
         BinaryData MakeForImport(const BinaryData &old_data, const SubfileData &data) override;
 

Source/Subfiles/JpgSubfile.cpp

@@ -26,13 +26,12 @@ namespace LOTRO_DAT {
         return std::string(".jpg");
     }
 
-    std::vector<SubfileData> JpgSubfile::PrepareForExport(const BinaryData &file_data) {
-        std::vector<SubfileData> output(1);
-        output[0].binary_data = file_data.CutData(24);
-        output[0].text_data = u"";
-        output[0].options["fid"] = file_id();
-        output[0].options["ext"] = Extension();
-        return output;
+    SubfileData JpgSubfile::PrepareForExport(const BinaryData &file_data) {
+        SubfileData result;
+        result.binary_data = file_data.CutData(24);
+        result.options["fid"] = file_id();
+        result.options["ext"] = Extension();
+        return result;
     }
 
     BinaryData JpgSubfile::MakeForImport(const BinaryData &old_data, const SubfileData &data) {

Source/Subfiles/JpgSubfile.h

@@ -22,7 +22,7 @@ namespace LOTRO_DAT {
 
         std::string Extension() const override;
 
-        std::vector<SubfileData> PrepareForExport(const BinaryData &file_data) override;
+        SubfileData PrepareForExport(const BinaryData &file_data) override;
 
         BinaryData MakeForImport(const BinaryData &old_data, const SubfileData &data) override;
     };

Source/Subfiles/OggSubfile.cpp

@@ -26,12 +26,12 @@ namespace LOTRO_DAT {
         return ".ogg";
     }
 
-    std::vector<SubfileData> OggSubfile::PrepareForExport(const BinaryData &file_data) {
-        std::vector<SubfileData> output(1);
-        output[0].binary_data = file_data.CutData(8);
-        output[0].options["fid"] = file_id();
-        output[0].options["ext"] = Extension();
-        return output;
+    SubfileData OggSubfile::PrepareForExport(const BinaryData &file_data) {
+        SubfileData result;
+        result.binary_data = file_data.CutData(8);
+        result.options["fid"] = file_id();
+        result.options["ext"] = Extension();
+        return result;
     }
 
     BinaryData OggSubfile::MakeForImport(const BinaryData &old_data, const SubfileData &data) {

Source/Subfiles/OggSubfile.h

@@ -19,7 +19,7 @@ namespace LOTRO_DAT {
         FILE_TYPE FileType() const override;
         std::string Extension() const override;
 
-        std::vector<SubfileData> PrepareForExport(const BinaryData &file_data) override;
+        SubfileData PrepareForExport(const BinaryData &file_data) override;
 
         BinaryData MakeForImport(const BinaryData &old_data, const SubfileData &data) override;
     };

Source/Subfiles/TextSubfile.cpp

@@ -7,6 +7,44 @@
 #include "../DatFile.h"
 #include "../Common/DatException.h"
 #include "../SubfileData.h"
+#include <algorithm>
+#include <codecvt>
+#include <locale>
+
+std::u16string to_utf16(long long x) {
+    std::u16string res;
+    while (x > 0) {
+        res += u'0' + (x % 10);
+        x /= 10ll;
+    }
+    std::reverse(res.begin(), res.end());
+    return res;
+}
+
+long long from_utf16(const std::u16string &num) {
+    long long res = 0;
+    for (auto c : num) {
+        res = res * 10ll + (c - u'0');
+    }
+    return res;
+}
+
+std::string argumentsFromUtf16(const std::u16string &args) {
+    std::string res;
+
+    size_t pointer = 0;
+    while (pointer < args.length()) {
+        size_t pointer1 = args.find(u'-', pointer);
+        if (pointer1 == std::u16string::npos)
+            pointer1 = args.length();
+        if (!res.empty())
+            res += "-";
+        res += std::to_string(from_utf16(args.substr(pointer, pointer1 - pointer)));
+        pointer = pointer1 + 1;
+    }
+
+    return res;
+}
 
 namespace LOTRO_DAT {
     TextSubfile::TextSubfile() = default;
@@ -26,8 +64,8 @@ namespace LOTRO_DAT {
         return std::string(".txt");
     }
 
-    std::vector<SubfileData> TextSubfile::PrepareForExport(const BinaryData &file_data) {
-        std::vector<SubfileData> result;
+    SubfileData TextSubfile::PrepareForExport(const BinaryData &file_data) {
+        SubfileData result;
         if (file_size() <= 10) // File is empty, nothing to do;
             return result;
 
@@ -53,30 +91,150 @@ namespace LOTRO_DAT {
                 text += text_pieces[j] + u"<--DO_NOT_TOUCH!-->";
             text += text_pieces[text_pieces.size() - 1] + u"]";
 
-            std::string arguments;
+            std::u16string arguments;
             for (size_t j = 0; j + 1 < arg_references.size(); j++)
-                arguments += std::to_string(arg_references[j]) + "-";
+                arguments += to_utf16(arg_references[j]) + u"-";
             if (!arg_references.empty())
-                arguments += std::to_string(arg_references[arg_references.size() - 1]);
+                arguments += to_utf16(arg_references[arg_references.size() - 1]);
 
-            SubfileData subfile;
+            if (result.text_data.length() > 0)
+                result.text_data += u"|||";
 
-            subfile.text_data = text;
-            subfile.options["fid"] = file_id();
-            subfile.options["gid"] = fragment_id;
-            subfile.options["ext"] = Extension();
-            if (!arg_references.empty())
-                subfile.options["args"] = arguments;
-
-            result.push_back(subfile);
+            result.text_data += to_utf16(fragment_id) + u":::";
+            result.text_data += arguments + u":::";
+            result.text_data += text;
         }
+        result.options["fid"] = file_id();
+        result.options["ext"] = Extension();
         return result;
     }
 
     BinaryData TextSubfile::MakeForImport(const BinaryData &old_data, const SubfileData &data) {
-        return Subfile::MakeForImport(old_data, data);
+        std::unordered_map<long long, SubfileData> patch_fragments = ParsePatchFragments(data);
+
+        BinaryData new_data;
+
+        if (file_size() <= 10 + 8) // File is empty, nothing to do;
+            return old_data;
+
+        long long offset = 9 + 8; // first 8 bytes - file_info. After them:
+        // first 4 bytes - file_id, then 4 bytes - unknown, then 1 byte - unknown
+
+        long long text_fragment_num = old_data.ToNumber<1>(offset);
+        if ((text_fragment_num & 0x80) != 0) {
+            text_fragment_num = (((text_fragment_num ^ 0x80) << 8) | old_data.ToNumber<1>(offset + 1));
+            offset += 1;
+        }
+        offset += 1;
+
+        new_data = new_data + old_data.CutData(0, offset); // Adding file info
+
+        for (long long i = 0; i < text_fragment_num; i++) {
+            long long fragment_id = old_data.ToNumber<8>(offset);
+            offset += 8;
+
+            new_data = new_data + old_data.CutData(offset - 8, offset);
+
+            if (patch_fragments.count(fragment_id) == 0) {
+                std::cerr << "Omg, what..? " << file_id() << " " << fragment_id << " is not in patch? " << std::endl;
+                try {
+                    // Retrieving old pieces
+                    new_data = new_data + GetPieceData(old_data, offset);
+                } catch (std::exception &e) {
+                    fprintf(stderr, "Caught %s exception.\n", e.what());
+                    fprintf(stderr, "ERROR TextSubfile::MakeForImport() - unable to get piece data for file_id %lld and fragment_id %lld", file_id(), fragment_id);
+                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
+                }
+
+                try {
+                    // Retrieving old references
+                    new_data = new_data + GetArgumentReferenceData(old_data, offset);
+                } catch (std::exception &e) {
+                    fprintf(stderr, "Caught %s exception.\n", e.what());
+                    fprintf(stderr, "ERROR TextSubfile::MakeForImport() - unable to get argument reference data for file_id %lld and fragment_id %lld", file_id(), fragment_id);
+                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
+                }
+
+                try {
+                    // Retrieving old ref_strings
+                    new_data = new_data + GetArgumentStringsData(old_data, offset);
+                } catch (std::exception &e) {
+                    fprintf(stderr, "Caught %s exception.\n", e.what());
+                    fprintf(stderr, "ERROR TextSubfile::MakeForImport() - unable to get argument string for file_id %lld and fragment_id %lld", file_id(), fragment_id);
+                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
+                }
+
+            } else {
+                try {
+                    // Making and adding new pieces
+                    new_data = new_data + BuildPieces(old_data, patch_fragments[fragment_id], offset);
+                } catch (std::exception &e) {
+                    fprintf(stderr, "Caught %s exception.\n", e.what());
+                    fprintf(stderr, "ERROR TextSubfile::MakeForImport() - unable to build piece data for file_id %lld and fragment_id %lld", file_id(), fragment_id);
+                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
+                }
+
+                try {
+                    // Making and adding new references
+                    new_data = new_data + BuildArgumentReferences(old_data, patch_fragments[fragment_id], offset);
+                } catch (std::exception &e) {
+                    fprintf(stderr, "Caught %s exception.\n", e.what());
+                    fprintf(stderr, "ERROR TextSubfile::MakeForImport() - unable to build argument references data for file_id %lld and fragment_id %lld", file_id(), fragment_id);
+                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
+                }
+
+                try {
+                    // Making and adding new strings
+                    new_data = new_data + BuildArgumentStrings(old_data, patch_fragments[fragment_id], offset);
+                } catch (std::exception &e) {
+                    fprintf(stderr, "Caught %s exception.\n", e.what());
+                    fprintf(stderr, "ERROR TextSubfile::MakeForImport() - unable to build argument strings data for file_id %lld and fragment_id %lld", file_id(), fragment_id);
+                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
+                }
+            }
+        }
+        new_data = new_data + old_data.CutData(offset); // Adding elapsed file data
+        return new_data;
+    }
+
+    std::unordered_map<long long, SubfileData> TextSubfile::ParsePatchFragments(const SubfileData &data) {
+        std::unordered_map<long long, SubfileData> res;
+        std::u16string text = data.text_data;
+
+        size_t pointer = 0;
+        while (pointer < text.length()) {
+            // Parsing fragment_id
+            size_t pointer1 = text.find(u":::", pointer);
+            if (pointer1 == std::u16string::npos)
+                throw DatException("Bad TextSubfile::ParsePatchFragments() - Unable to parse fragment id! Cannot find '...' divider");
+            long long fragment_id = from_utf16(text.substr(pointer, pointer1 - pointer));
+            pointer = pointer1 + 3;
+            res[fragment_id] = SubfileData();
+            res[fragment_id].options["gid"] = fragment_id;
+
+            // Parsing arguments
+            pointer1 = text.find(u":::", pointer);
+            if (pointer1 == std::u16string::npos)
+                throw DatException("Bad TextSubfile::ParsePatchFragments() - Unable to parse arguments! Cannot find '...' divider");
+            std::u16string arguments = text.substr(pointer, pointer1 - pointer);
+            pointer = pointer1 + 3;
+            if (arguments.length() > 0) {
+                res[fragment_id].options["args"] = argumentsFromUtf16(arguments);
+            }
+
+            // Parsing text
+            pointer1 = text.find(u"|||", pointer);
+            if (pointer1 == std::u16string::npos)
+                pointer1 = text.length();
+            std::u16string text_data = text.substr(pointer, pointer1 - pointer);
+            pointer = pointer1  + 3;
+            res[fragment_id].text_data = text_data;
+        }
+        return res;
     }
 
+    // Make pieces/arguments/argument strings functions
+
     std::vector<std::u16string> TextSubfile::MakePieces(const BinaryData &data, long long &offset) {
         long long num_pieces = data.ToNumber<4>(offset);
         offset += 4;
@@ -144,4 +302,152 @@ namespace LOTRO_DAT {
         }
         return arg_strings;
     }
+
+    // Build pieces/arguments/argument strings functions from fragment SubfileData
+
+    BinaryData TextSubfile::BuildPieces(const BinaryData &data, const SubfileData &new_data, long long &offset) {
+        try {
+            // Moving &offset pointer in &data
+            GetPieceData(data, offset);
+        } catch (std::exception &e) {
+            fprintf(stderr, "Caught %s exception.\n", e.what());
+            fprintf(stderr, "ERROR TextSubfile::BuildPieces() - unable to get piece data for file_id %lld", file_id());
+            throw DatException("Bad TextSubfile::BuildPieces()", IMPORT_EXCEPTION);
+        }
+
+        // Deleting '[' and ']' brackets
+        std::u16string text_data = new_data.text_data.substr(1, new_data.text_data.size() - 2);
+        std::vector<std::u16string> pieces;
+
+        const std::u16string DNT = u"<--DO_NOT_TOUCH!-->";
+        size_t prev = 0;
+        size_t next = text_data.find(DNT, prev);
+
+        while (next != std::string::npos) {
+            std::u16string piece = text_data.substr(prev, next - prev);
+            pieces.push_back(piece);
+            prev = next + DNT.length();
+            next = text_data.find(DNT, prev);
+        }
+        std::u16string piece = text_data.substr(prev);
+        pieces.push_back(piece);
+
+        // Building BinaryData from pieces
+        BinaryData result;
+        BinaryData temp_data(4);
+
+        temp_data.FromNumber<4>(pieces.size());
+        result = result + temp_data;
+
+        for (long long i = 0; i < pieces.size(); i++) {
+            long long piece_size = pieces[i].length();
+            if (piece_size < 128) {
+                temp_data.FromNumber<1>(piece_size);
+            } else {
+                temp_data.FromNumberRAW<2>((piece_size | 32768));
+            }
+            result = result + temp_data;
+
+            for (long long j = 0; j < piece_size; j++) {
+                temp_data.FromNumber<2>(short(pieces[i][j]));
+                result = result + temp_data;
+            }
+        }
+        return result;
+    }
+
+    BinaryData TextSubfile::BuildArgumentReferences(const BinaryData &data, const SubfileData &new_data,
+                                                    long long &offset) {
+        // Moving &offset pointer in &data
+        GetArgumentReferenceData(data, offset);
+
+        // If there are no args - making 4 null-bytes and return;
+        if (!new_data.options["args"]) {
+            BinaryData result;
+            result.FromNumber<4>(0);
+            return result;
+        }
+
+        // Parsing arguments from list in options["args"]
+        std::string args_list = new_data.options["args"].as<std::string>();
+        std::vector<long long> arguments;
+
+        size_t prev = 0;
+        size_t next = args_list.find('-', prev);
+        while (next != std::string::npos) {
+            std::string argument = args_list.substr(prev, next - prev);
+            arguments.push_back(std::stoll(argument));
+            prev = next + 1;
+            next = args_list.find('-', prev);
+        }
+        std::string argument = args_list.substr(prev);
+        arguments.push_back(std::stoll(argument));
+
+        BinaryData result;
+        BinaryData temp_data(4);
+        temp_data.FromNumber<4>(arguments.size());
+        result = result + temp_data;
+        for (auto arg_reference : arguments) {
+            temp_data.FromNumber<4>(arg_reference);
+            result = result + temp_data;
+        }
+        return result;
+    }
+
+    BinaryData TextSubfile::BuildArgumentStrings(const BinaryData &data, const SubfileData &new_data,
+                                                 long long &offset) {
+        return GetArgumentStringsData(data, offset);
+    }
+
+    // Get BinaryData contents of pieces/arguments/argument strings
+
+    BinaryData TextSubfile::GetPieceData(const BinaryData &data, long long &offset) const {
+        long long old_offset = offset;
+
+        long long num_pieces = data.ToNumber<4>(offset);
+        offset += 4;
+
+        for (long long j = 0; j < num_pieces; j++) {
+            long long piece_size = data.ToNumber<1>(offset);
+            if ((piece_size & 128) != 0) {
+                piece_size = (((piece_size ^ 128) << 8) | data.ToNumber<1>(offset + 1));
+                offset += 1;
+            }
+            offset += 1;
+            offset += piece_size * 2;
+        }
+        return data.CutData(old_offset, offset);
+    }
+
+    BinaryData TextSubfile::GetArgumentReferenceData(const BinaryData &data, long long &offset) const {
+        long long old_offset = offset;
+        long long num_references = data.ToNumber<4>(offset);
+        offset += 4;
+        offset += 4 * num_references;
+        return data.CutData(old_offset, offset);
+    }
+
+    BinaryData TextSubfile::GetArgumentStringsData(const BinaryData &data, long long &offset) const {
+        long long old_offset = offset;
+
+        long long num_arg_strings = data.ToNumber<1>(offset);
+        offset += 1;
+
+        for (long long j = 0; j < num_arg_strings; j++) {
+            long long num_args = data.ToNumber<4>(offset);
+            offset += 4;
+
+            for (long long k = 0; k < num_args; k++) {
+                long long string_size = data.ToNumber<1>(offset);
+                if ((string_size & 0x80) != 0) {
+                    string_size = (((string_size ^ 0x80) << 8) | data.ToNumber<1>(offset + 1));
+                    offset += 1;
+                }
+                offset += 1;
+                offset += string_size * 2;
+            }
+        }
+        return data.CutData(old_offset, offset);
+    }
+
 };

Source/Subfiles/TextSubfile.h

@@ -7,6 +7,7 @@
 
 #include "../Subfile.h"
 #include <vector>
+#include <unordered_map>
 
 namespace LOTRO_DAT {
     class TextSubfile : public Subfile {
@@ -21,7 +22,7 @@ namespace LOTRO_DAT {
 
         std::string Extension() const override;
 
-        std::vector<SubfileData> PrepareForExport(const BinaryData &file_data) override;
+        SubfileData PrepareForExport(const BinaryData &file_data) override;
 
         BinaryData MakeForImport(const BinaryData &old_data, const SubfileData &data) override;
 
@@ -31,6 +32,22 @@ namespace LOTRO_DAT {
         std::vector<long long> MakeArgumentReferences(const BinaryData &data, long long &offset);
 
         std::vector<std::vector<BinaryData>> MakeArgumentStrings(const BinaryData &data, long long &offset);
+
+        std::vector<SubfileData> FileFragments;
+
+        std::unordered_map<long long, SubfileData> ParsePatchFragments(const SubfileData &data);
+
+        BinaryData BuildPieces(const BinaryData &data, const SubfileData &new_data, long long &offset);
+
+        BinaryData BuildArgumentReferences(const BinaryData &data, const SubfileData &new_data, long long &offset);
+
+        BinaryData BuildArgumentStrings(const BinaryData &data, const SubfileData &new_data, long long &offset);
+
+        BinaryData GetPieceData(const BinaryData &data, long long &offset) const;
+
+        BinaryData GetArgumentReferenceData(const BinaryData &data, long long &offset) const;
+
+        BinaryData GetArgumentStringsData(const BinaryData &data, long long &offset) const;
     };
 };
 

Source/Subfiles/UnknownSubfile.cpp

@@ -26,11 +26,11 @@ namespace LOTRO_DAT {
         return std::string(".unknown");
     }
 
-    std::vector<SubfileData> UnknownSubfile::PrepareForExport(const BinaryData &file_data) {
-        std::vector<SubfileData> result(1);
-        result[0].binary_data = file_data;
-        result[0].options["fid"] = file_id();
-        result[0].options["ext"] = Extension();
+    SubfileData UnknownSubfile::PrepareForExport(const BinaryData &file_data) {
+        SubfileData result;
+        result.binary_data = file_data;
+        result.options["fid"] = file_id();
+        result.options["ext"] = Extension();
         return result;
     }
 

Source/Subfiles/UnknownSubfile.h

@@ -19,7 +19,7 @@ namespace LOTRO_DAT {
         FILE_TYPE FileType() const override;
         std::string Extension() const override;
 
-        std::vector<SubfileData> PrepareForExport(const BinaryData &file_data) override;
+        SubfileData PrepareForExport(const BinaryData &file_data) override;
 
         BinaryData MakeForImport(const BinaryData &old_data, const SubfileData &data) override;
 

Source/Subfiles/WavSubfile.cpp

@@ -26,11 +26,11 @@ namespace LOTRO_DAT {
         return ".wav";
     }
 
-    std::vector<SubfileData> WavSubfile::PrepareForExport(const BinaryData &file_data) {
-        std::vector<SubfileData> result(1);
-        result[0].binary_data = file_data.CutData(8);
-        result[0].options["fid"] = file_id();
-        result[0].options["ext"] = Extension();
+    SubfileData WavSubfile::PrepareForExport(const BinaryData &file_data) {
+        SubfileData result;
+        result.binary_data = file_data.CutData(8);
+        result.options["fid"] = file_id();
+        result.options["ext"] = Extension();
         return result;
     }
 

Source/Subfiles/WavSubfile.h

@@ -21,7 +21,7 @@ namespace LOTRO_DAT {
 
         std::string Extension() const override;
 
-        std::vector<SubfileData> PrepareForExport(const BinaryData &file_data) override;
+        SubfileData PrepareForExport(const BinaryData &file_data) override;
 
         BinaryData MakeForImport(const BinaryData &old_data, const SubfileData &data) override;
 

Source/Tests/extract_test.cpp

@@ -22,20 +22,21 @@ const std::string filename = "client_local_English.dat";
 // Change these variables to true if you want export catecory to files.
 const bool exportImagesToFiles = false;
 const bool exportFontsToFiles = false;
-const bool exportSoundsToFiles = true;
+const bool exportSoundsToFiles = false;
 const bool exportTexturesToFiles = false;
 const bool exportUnknownToFiles = false;
 
 // Change these variables to true if you want export catecory to databases.
-const bool exportTextsToDb = false;
-const bool exportImagesToDb = false;
-const bool exportFontsToDb = false;
+const bool exportTextsToDb = true;
+const bool exportImagesToDb = true;
+const bool exportFontsToDb = true;
 const bool exportSoundsToDb = true;
-const bool exportTexturesToDb = false;
+const bool exportTexturesToDb = true;
 const bool exportUnknownToDb = false;
 // There is no need to change anything else
 
 int main() {
+
     const clock_t begin_time = clock();
 
     mkdir("Extracted data", 744);
@@ -51,49 +52,50 @@ int main() {
 
     mkdir(output_dir.c_str(), 744);
     freopen((output_dir + std::string("errors.log")).c_str(), "w", stderr);
-
+    setbuf(stdout, NULL);
+    setbuf(stderr, NULL);
+    setvbuf (stdout, NULL, _IONBF, BUFSIZ);
+    setvbuf (stderr, NULL, _IONBF, BUFSIZ);
     try {
         std::cout << "Starting search...\n" << std::flush;
         DatFile a((path + filename).c_str(), 0);
         std::cout << "Total files found: " << a.files_number() << std::endl << std::flush;
 
-        std::cout << "Writing unordered dictionary:\n";
-        a.WriteUnorderedDictionary(output_dir);
-
         std::cout << "Beginning unpacking... Please, wait for some minutes."
         "\nMaybe it's a good idea to have a cup of tea, while unpacker is working...\n" << std::flush;
 
-
+        Database *out = new Database("out.db");
         if (exportImagesToDb) {
-            Database *images = new Database(output_dir + std::string("Images.db"));
+            Database *images = out;//new Database(output_dir + std::string("Images.db"));
             std::cout << "Extracted " << a.ExtractAllFilesByType(JPG, images) << " .jpg files to Images.db" << std::endl << std::flush;
-            delete images;
+//            delete images;
         }
 
         if (exportSoundsToDb) {
-            Database *sounds = new Database(output_dir + std::string("Sounds.db"));
+            Database *sounds = out;//new Database(output_dir + std::string("Sounds.db"));
             std::cout << "Extracted " << a.ExtractAllFilesByType(WAV, sounds) << " .wav files to Sounds.db" << std::endl << std::flush;
             std::cout << "Extracted " << a.ExtractAllFilesByType(OGG, sounds) << " .ogg files to Sounds.db" << std::endl << std::flush;
-            delete sounds;
+//            delete sounds;
         }
 
         if (exportTextsToDb) {
-            Database *texts = new Database(output_dir + std::string("Texts.db"));
+            Database *texts = out;//new Database(output_dir + std::string("Texts.db"));
             std::cout << "Extracted " << a.ExtractAllFilesByType(TEXT, texts) << " text files to Texts.db" << std::endl << std::flush;
-            delete texts;
+//            delete texts;
         }
 
         if (exportFontsToDb) {
-            Database *fonts = new Database(output_dir + std::string("Fonts.db"));
+            Database *fonts = out;//new Database(output_dir + std::string("Fonts.db"));
             std::cout << "Extracted " << a.ExtractAllFilesByType(FONT, fonts) << " font files to Fonts.db" << std::endl << std::flush;
-            delete fonts;
+//            delete fonts;
         }
 
         if (exportTexturesToDb) {
-            Database *textures = new Database(output_dir + std::string("Textures.db"));
+            Database *textures = out;//new Database(output_dir + std::string("Textures.db"));
             std::cout << "Extracted " << a.ExtractAllFilesByType(DDS, textures) << " .dds files to Textures.db" << std::endl << std::flush;
-            delete textures;
+//            delete textures;
         }
+        delete out;
 
         if (exportUnknownToDb) {
             Database *unknown = new Database(output_dir + std::string("Unknown.db"));

Source/Tests/patch_test.cpp

@@ -23,9 +23,14 @@ int main() {
     const clock_t begin_time = clock();
 
     freopen("errors.log", "w", stderr);
+    //freopen("stdout.log", "w", stdout);
+    setbuf(stdout, NULL);
+    setbuf(stderr, NULL);
 
+    setvbuf (stdout, NULL, _IONBF, BUFSIZ);
+    setvbuf (stderr, NULL, _IONBF, BUFSIZ);
     Database db;
-    db.InitDatabase("sounds.db");
+    db.InitDatabase("out.db");
 
     DatFile *a;
 
@@ -34,24 +39,7 @@ int main() {
         std::cout << "Starting magic...\n";
         std::cout << "Files number: " << a->files_number() << std::endl;
         a->WriteUnorderedDictionary("");
-
-        SubfileData data;
-//        data.binary_data.ReadFromFile("167855806.ogg");
-//        data.options["fid"] = "167855806";
-//        data.options["ext"] = ".ogg";
-//
-//        auto i = a->dictionary_[167855806];
-//        BinaryData new_data = i->MakeForImport(a->GetFileData(i), data);
-//        new_data.WriteToFile("test_new_file");
-//
-//        BinaryData old_data = a->GetFileData(i);
-//        old_data.WriteToFile("test_old_file");
-
-        data = db.GetNextFile();
-        while (data != SubfileData()) {
-            a->PatchFile(data);
-            data = db.GetNextFile();
-        }
+        a->PatchAllDatabase(&db);
     } catch (std::exception &e) {
         fprintf(stderr, "Caught %s exception.", e.what());
         printf("Caught %s exception.", e.what());