Implemented text patching. Bug fixes.

Source/BinaryData.cpp

@@ -5,6 +5,7 @@
 #include "BinaryData.h"
 #include "Common/DatException.h"
 #include "Common/ZLib/zlib.h"
+#include <algorithm>
 
 extern  "C++"
 {
@@ -16,7 +17,7 @@ namespace LOTRO_DAT {
     }
 
     BinaryData::BinaryData(const BinaryData &d) {
-		size_ = d.size_;
+        size_ = d.size_;
         data_ = new unsigned char[size_];
         memcpy(data_, d.data_, size_);
     }
@@ -28,7 +29,7 @@ namespace LOTRO_DAT {
     }
 
     BinaryData::BinaryData(unsigned int size) {
-		data_ = new unsigned char[size];
+        data_ = new unsigned char[size];
         size_ = size;
     }
 
@@ -112,20 +113,11 @@ namespace LOTRO_DAT {
     // Makes data from specified T bytes of number in raw
     template<int T>
     void BinaryData::FromNumberRAW(const long long &number) {
-        if (T < 0)
-            throw DatException("Bad BinaryData::FromNumber() - trying to make data from amount of bytes < 0");
+        if (T <= 0)
+            throw DatException("Bad BinaryData::FromNumber() - trying to make data from amount of bytes <= 0");
         try {
-            if (size_ != 0 && data_ != nullptr)
-                delete[] data_;
-
-            size_ = size_t(T);
-            data_ = new unsigned char[size_];
-
-            for (size_t i = size_ - 1; i >= 0; i--)
-            {
-                data_[i] = (unsigned char)((number >> (8 * i)) & 0xFF);
-            }
-
+            FromNumber<T>(number);
+            std::reverse(data_, data_ + size());
         } catch (...) {
             throw DatException("Bad BinaryData::ToNumber(). Error in using template function", DATA_EXCEPTION);
         }
@@ -157,21 +149,21 @@ namespace LOTRO_DAT {
         return data_;
     }
 
-	bool BinaryData::WriteToFile(const char *filename) const {
-		FILE *f;
-		fopen_s(&f, filename, "wb");
-		if (f == nullptr) {
-			throw DatException("Bad BinaryData::WriteToFile() - unable to open output file", EXPORT_EXCEPTION);
-		}
+    bool BinaryData::WriteToFile(const char *filename) const {
+        FILE *f;
+        fopen_s(&f, filename, "wb");
+        if (f == nullptr) {
+            throw DatException("Bad BinaryData::WriteToFile() - unable to open output file", EXPORT_EXCEPTION);
+        }
 
-		fwrite(data(), size(), 1, f);
-		fclose(f);
-		return true;
-	}
+        fwrite(data(), size(), 1, f);
+        fclose(f);
+        return true;
+    }
 
-	bool BinaryData::WriteToFile(const std::string &filename) const {
-		return WriteToFile(filename.c_str());
-	}
+    bool BinaryData::WriteToFile(const std::string &filename) const {
+        return WriteToFile(filename.c_str());
+    }
 
 
     void BinaryData::ReadFromFile(const char *filename) {
@@ -210,14 +202,14 @@ namespace LOTRO_DAT {
 
         uLongf new_size = max_unpacked_size;
         int res = uncompress(decompressed.data_, &new_size, data_ + offset, size_ - offset);
-		
-		if (res != 0) {
-			throw DatException("Bad BinaryData::DecompressData() - uncompress() failed!", DATA_EXCEPTION);
-		}
 
-		decompressed.size_ = (unsigned int)new_size;
-		return decompressed;
-	}
+        if (res != 0) {
+            throw DatException("Bad BinaryData::DecompressData() - uncompress() failed!", DATA_EXCEPTION);
+        }
+
+        decompressed.size_ = (unsigned int)new_size;
+        return decompressed;
+    }
 
     BinaryData BinaryData::CompressData(unsigned int offset) const {
         const unsigned max_file_size = 1024 * 1024 * 40; // Setting 40 MB as a maximum packed data;
@@ -280,6 +272,5 @@ namespace LOTRO_DAT {
     template void BinaryData::FromNumberRAW<2>(const long long &);
     template void BinaryData::FromNumberRAW<4>(const long long &);
     template void BinaryData::FromNumberRAW<8>(const long long &);
-
 }
 }

Source/DatFile.cpp

@@ -126,6 +126,7 @@ namespace LOTRO_DAT {
 
         try {
             export_data = dictionary_[file_id]->PrepareForExport(file_data);
+            export_data.options["did"] = dat_id_;
         } catch (std::exception &e) {
             fprintf(stderr, "Caught %s exception.", e.what());
 
@@ -133,6 +134,11 @@ namespace LOTRO_DAT {
             return false;
         }
 
+        if (export_data == SubfileData()) {
+            fprintf(stderr, "WARNING: file with id %lld is too small. Passing it\n", dictionary_[file_id]->file_id());
+            return true;
+        }
+
         try {
             db->PushFile(export_data);
         } catch (std::exception &e) {
@@ -225,9 +231,35 @@ namespace LOTRO_DAT {
         return true;
     }
 
-    // TODO: Write description and make asserts and IMPLEMENT
+    // TODO: Write description and make asserts
     bool DatFile::PatchAllDatabase(Database *db) {
-        return false;
+        //GetFileData(dictionary_[621006304]).WriteToFile("kek");
+
+        SubfileData data;
+        try {
+            data = db->GetNextFile();
+        } catch (std::exception &e) {
+            fprintf(stderr, "Caught %s exception.\n", e.what());
+            fprintf(stderr, "DatFile::PatchAllDatabase() error! Caught exception while fetching file from database! Stopping...\n");
+            return false;
+        }
+        while (data != SubfileData()) {
+            try {
+                PatchFile(data);
+            } catch (std::exception &e) {
+                fprintf(stderr, "Caught %s exception.\n", e.what());
+                fprintf(stderr, "DatFile::PatchAllDatabase() error! Caught exception while patching file! Passing...\n");
+            }
+
+            try {
+                data = db->GetNextFile();
+            } catch (std::exception &e) {
+                fprintf(stderr, "Caught %s exception.\n", e.what());
+                fprintf(stderr, "DatFile::PatchAllDatabase() error! Caught exception while fetching file from database! Stopping...\n");
+                return false;
+            }
+        }
+        return true;
     }
 
     /// DatFile::WriteUnorderedDictionary(...);
@@ -261,9 +293,9 @@ namespace LOTRO_DAT {
         if (file->file_id() != mfile_id.ToNumber<4>(0))
             throw DatException("Bad DatFile::GetFileData() - file_id in Subfile doesn't match to file_id in DatFile.", READ_EXCEPTION);
 
-        BinaryData data((unsigned)(file->file_size()));
+        BinaryData data((unsigned)(file->file_size() + (8 - offset)));
         if (file->block_size() >= file->file_size() + 8) {
-            ReadData(data, file->file_size(), file->file_offset() + offset);
+            ReadData(data, file->file_size() + (8 - offset), file->file_offset() + offset);
             return data;
         }
 

Source/Subfiles/TextSubfile.cpp

@@ -8,18 +8,48 @@
 #include "../Common/DatException.h"
 #include "../SubfileData.h"
 #include <algorithm>
+#include <codecvt>
+#include <locale>
 
-std::u16string to_utf16(long long x)
-{
+std::u16string to_utf16(long long x) {
     std::u16string res;
     while (x > 0) {
-        res += u'0' + short(x % 10);
+        res += u'0' + (x % 10);
         x /= 10ll;
     }
     std::reverse(res.begin(), res.end());
     return res;
 }
 
+long long from_utf16(const std::u16string &num) {
+    long long res = 0;
+    for (auto c : num) {
+        res = res * 10ll + (c - u'0');
+    }
+    std::cout << "fragment_id: " << res << std::endl;
+    return res;
+}
+
+std::string argumentsFromUtf16(const std::u16string &args) {
+    std::string res;
+
+    size_t pointer = 0;
+    while (pointer < args.length()) {
+        size_t pointer1 = args.find(u'-', pointer);
+        if (pointer1 == std::u16string::npos)
+            pointer1 = args.length();
+        if (!res.empty())
+            res += "-";
+        res += std::to_string(from_utf16(args.substr(pointer, pointer1 - pointer)));
+        pointer = pointer1 + 1;
+    }
+
+    std::wstring_convert<std::codecvt_utf8_utf16<char16_t>,char16_t> codecvt;
+    std::cout << "Arguments old: "<< codecvt.to_bytes(args) << '\n' ;
+    std::cout << "Arguments new: "<< res << '\n' ;
+    return res;
+}
+
 namespace LOTRO_DAT {
     TextSubfile::TextSubfile() = default;
 
@@ -84,18 +114,13 @@ namespace LOTRO_DAT {
     }
 
     BinaryData TextSubfile::MakeForImport(const BinaryData &old_data, const SubfileData &data) {
-        BinaryData new_data;
-
-        // Creating map for fast access to patch fragment by it's id
-        std::unordered_map<long long, long long> fragment_vector_dictionary;
-
-        std::cerr << "Fragments for " << file_id() << std::endl;
-        for (int i = 0; i < FileFragments.size(); i++)
-            fragment_vector_dictionary[FileFragments[i].options["gid"].as<long long>()] = i;
+        std::unordered_map<long long, SubfileData> patch_fragments = ParsePatchFragments(data);
+        std::cout << "Made fragments. Fragments are: " << std::flush;
+        for (auto &i : patch_fragments)
+            std::cout << i.first << " " << std::flush;
+        std::cout << std::endl;
 
-        for (auto i : fragment_vector_dictionary)
-            std::cerr << i.first << " ";
-        std::cerr << std::endl;
+        BinaryData new_data;
 
         if (file_size() <= 10 + 8) // File is empty, nothing to do;
             return old_data;
@@ -118,22 +143,115 @@ namespace LOTRO_DAT {
 
             new_data = new_data + old_data.CutData(offset - 8, offset);
 
-            if (fragment_vector_dictionary.count(fragment_id) == 0) {
+            if (patch_fragments.count(fragment_id) == 0) {
                 std::cerr << "Omg, what..? " << file_id() << " " << fragment_id << " is not in patch? " << std::endl;
-                new_data = new_data + GetPieceData(old_data, offset); // Retrieving old pieces
-                new_data = new_data + GetArgumentReferenceData(old_data, offset); // Retrieving old references
-                new_data = new_data + GetArgumentStringsData(old_data, offset); // Retrieving old ref_strings
+                try {
+                    // Retrieving old pieces
+                    new_data = new_data + GetPieceData(old_data, offset);
+                } catch (std::exception &e) {
+                    fprintf(stderr, "Caught %s exception.\n", e.what());
+                    fprintf(stderr, "ERROR TextSubfile::MakeForImport() - unable to get piece data for file_id %lld and fragment_id %lld", file_id(), fragment_id);
+                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
+                }
+
+                try {
+                    // Retrieving old references
+                    new_data = new_data + GetArgumentReferenceData(old_data, offset);
+                } catch (std::exception &e) {
+                    fprintf(stderr, "Caught %s exception.\n", e.what());
+                    fprintf(stderr, "ERROR TextSubfile::MakeForImport() - unable to get argument reference data for file_id %lld and fragment_id %lld", file_id(), fragment_id);
+                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
+                }
+
+                try {
+                    // Retrieving old ref_strings
+                    new_data = new_data + GetArgumentStringsData(old_data, offset);
+                } catch (std::exception &e) {
+                    fprintf(stderr, "Caught %s exception.\n", e.what());
+                    fprintf(stderr, "ERROR TextSubfile::MakeForImport() - unable to get argument string for file_id %lld and fragment_id %lld", file_id(), fragment_id);
+                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
+                }
+
             } else {
-                SubfileData fragment_data = FileFragments[fragment_vector_dictionary[fragment_id]];
-                new_data = new_data + BuildPieces(old_data, fragment_data, offset); // Making and adding new pieces
-                new_data = new_data + BuildArgumentReferences(old_data, fragment_data, offset); // Making and adding new references
-                new_data = new_data + BuildArgumentStrings(old_data, fragment_data, offset); // Making and adding new strings
+                try {
+                    // Making and adding new pieces
+                    new_data = new_data + BuildPieces(old_data, patch_fragments[fragment_id], offset);
+                } catch (std::exception &e) {
+                    fprintf(stderr, "Caught %s exception.\n", e.what());
+                    fprintf(stderr, "ERROR TextSubfile::MakeForImport() - unable to build piece data for file_id %lld and fragment_id %lld", file_id(), fragment_id);
+                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
+                }
+
+                try {
+                    // Making and adding new references
+                    new_data = new_data + BuildArgumentReferences(old_data, patch_fragments[fragment_id], offset);
+                } catch (std::exception &e) {
+                    fprintf(stderr, "Caught %s exception.\n", e.what());
+                    fprintf(stderr, "ERROR TextSubfile::MakeForImport() - unable to build argument references data for file_id %lld and fragment_id %lld", file_id(), fragment_id);
+                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
+                }
+
+                try {
+                    // Making and adding new strings
+                    new_data = new_data + BuildArgumentStrings(old_data, patch_fragments[fragment_id], offset);
+                } catch (std::exception &e) {
+                    fprintf(stderr, "Caught %s exception.\n", e.what());
+                    fprintf(stderr, "ERROR TextSubfile::MakeForImport() - unable to build argument strings data for file_id %lld and fragment_id %lld", file_id(), fragment_id);
+                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
+                }
             }
         }
         new_data = new_data + old_data.CutData(offset); // Adding elapsed file data
         return new_data;
     }
 
+    std::unordered_map<long long, SubfileData> TextSubfile::ParsePatchFragments(const SubfileData &data) {
+        std::unordered_map<long long, SubfileData> res;
+        std::u16string text = data.text_data;
+
+        size_t pointer = 0;
+        while (pointer < text.length()) {
+            // Parsing fragment_id
+            size_t pointer1 = text.find(u":::", pointer);
+            if (pointer1 == std::u16string::npos)
+                throw DatException("Bad TextSubfile::ParsePatchFragments() - Unable to parse fragment id! Cannot find '...' divider");
+            long long fragment_id = from_utf16(text.substr(pointer, pointer1 - pointer));
+            pointer = pointer1 + 3;
+            res[fragment_id] = SubfileData();
+            res[fragment_id].options["gid"] = fragment_id;
+
+            // Parsing arguments
+            pointer1 = text.find(u":::", pointer);
+            if (pointer1 == std::u16string::npos)
+                throw DatException("Bad TextSubfile::ParsePatchFragments() - Unable to parse arguments! Cannot find '...' divider");
+            std::u16string arguments = text.substr(pointer, pointer1 - pointer);
+            pointer = pointer1 + 3;
+            if (arguments.length() > 0) {
+                res[fragment_id].options["args"] = argumentsFromUtf16(arguments);
+            }
+
+            // Parsing text
+            pointer1 = text.find(u"|||", pointer);
+            if (pointer1 == std::u16string::npos)
+                pointer1 = text.length();
+            std::u16string text_data = text.substr(pointer, pointer1 - pointer);
+            pointer = pointer1  + 3;
+            res[fragment_id].text_data = text_data;
+
+
+            std::wstring_convert<std::codecvt_utf8_utf16<char16_t>,char16_t> codecvt;
+            std::cout << "Fragment id: " << res[fragment_id].options["gid"].as<long long>() << "\n";
+            if (res[fragment_id].options["args"])
+                std::cout << "Args: " << res[fragment_id].options["args"].as<std::string>() << "\n";
+            else
+                std::cout << "Args: none \n";
+            std::cout << "Text: " << codecvt.to_bytes(res[fragment_id].text_data) << '\n' ;
+        }
+        return res;
+    }
+
+    // Make pieces/arguments/argument strings functions
+
     std::vector<std::u16string> TextSubfile::MakePieces(const BinaryData &data, long long &offset) {
         long long num_pieces = data.ToNumber<4>(offset);
         offset += 4;
@@ -202,14 +320,25 @@ namespace LOTRO_DAT {
         return arg_strings;
     }
 
+    // Build pieces/arguments/argument strings functions from fragment SubfileData
+
     BinaryData TextSubfile::BuildPieces(const BinaryData &data, const SubfileData &new_data, long long &offset) {
-        // Moving &offset pointer in &data
-        GetPieceData(data, offset);
+        try {
+            // Moving &offset pointer in &data
+            GetPieceData(data, offset);
+        } catch (std::exception &e) {
+            fprintf(stderr, "Caught %s exception.\n", e.what());
+            fprintf(stderr, "ERROR TextSubfile::BuildPieces() - unable to get piece data for file_id %lld", file_id());
+            throw DatException("Bad TextSubfile::BuildPieces()", IMPORT_EXCEPTION);
+        }
 
         // Deleting '[' and ']' brackets
         std::u16string text_data = new_data.text_data.substr(1, new_data.text_data.size() - 2);
         std::vector<std::u16string> pieces;
 
+        std::wstring_convert<std::codecvt_utf8_utf16<char16_t>,char16_t> codecvt;
+        std::cout << "AAAAA0: " << codecvt.to_bytes(text_data) << '\n' ;
+
         const std::u16string DNT = u"<--DO_NOT_TOUCH!-->";
         size_t prev = 0;
         size_t next = text_data.find(DNT, prev);
@@ -223,6 +352,8 @@ namespace LOTRO_DAT {
         std::u16string piece = text_data.substr(prev);
         pieces.push_back(piece);
 
+        std::cout << "AAAAA: " << codecvt.to_bytes(piece) << '\n' ;
+
         // Building BinaryData from pieces
         BinaryData result;
         BinaryData temp_data(4);
@@ -272,6 +403,8 @@ namespace LOTRO_DAT {
             next = args_list.find('-', prev);
         }
         std::string argument = args_list.substr(prev);
+        std::cout << "DDD: Argument is " << argument << std::endl;
+        std::cout << "DDD: Arguments are" << new_data.options["args"].as<std::string>() << std::endl;
         arguments.push_back(std::stoll(argument));
 
         BinaryData result;
@@ -290,6 +423,8 @@ namespace LOTRO_DAT {
         return GetArgumentStringsData(data, offset);
     }
 
+    // Get BinaryData contents of pieces/arguments/argument strings
+
     BinaryData TextSubfile::GetPieceData(const BinaryData &data, long long &offset) const {
         long long old_offset = offset;
 
@@ -305,6 +440,7 @@ namespace LOTRO_DAT {
             offset += 1;
             offset += piece_size * 2;
         }
+        std::cout << "DDDDDDDDDDDD: " << old_offset << " " << offset << std::endl;
         return data.CutData(old_offset, offset);
     }
 

Source/Subfiles/TextSubfile.h

@@ -7,6 +7,7 @@
 
 #include "../Subfile.h"
 #include <vector>
+#include <unordered_map>
 
 namespace LOTRO_DAT {
     class TextSubfile : public Subfile {
@@ -34,6 +35,8 @@ namespace LOTRO_DAT {
 
         std::vector<SubfileData> FileFragments;
 
+        std::unordered_map<long long, SubfileData> ParsePatchFragments(const SubfileData &data);
+
         BinaryData BuildPieces(const BinaryData &data, const SubfileData &new_data, long long &offset);
 
         BinaryData BuildArgumentReferences(const BinaryData &data, const SubfileData &new_data, long long &offset);

Source/Tests/extract_test.cpp

@@ -28,14 +28,15 @@ const bool exportUnknownToFiles = false;
 
 // Change these variables to true if you want export catecory to databases.
 const bool exportTextsToDb = true;
-const bool exportImagesToDb = true;
-const bool exportFontsToDb = true;
-const bool exportSoundsToDb = true;
-const bool exportTexturesToDb = true;
+const bool exportImagesToDb = false;
+const bool exportFontsToDb = false;
+const bool exportSoundsToDb = false;
+const bool exportTexturesToDb = false;
 const bool exportUnknownToDb = false;
 // There is no need to change anything else
 
 int main() {
+
     const clock_t begin_time = clock();
 
     mkdir("Extracted data", 744);
@@ -51,7 +52,10 @@ int main() {
 
     mkdir(output_dir.c_str(), 744);
     freopen((output_dir + std::string("errors.log")).c_str(), "w", stderr);
-
+    setbuf(stdout, NULL);
+    setbuf(stderr, NULL);
+    setvbuf (stdout, NULL, _IONBF, BUFSIZ);
+    setvbuf (stderr, NULL, _IONBF, BUFSIZ);
     try {
         std::cout << "Starting search...\n" << std::flush;
         DatFile a((path + filename).c_str(), 0);

Source/Tests/patch_test.cpp

@@ -23,9 +23,14 @@ int main() {
     const clock_t begin_time = clock();
 
     freopen("errors.log", "w", stderr);
+    freopen("stdout.log", "w", stdout);
+    setbuf(stdout, NULL);
+    setbuf(stderr, NULL);
 
+    setvbuf (stdout, NULL, _IONBF, BUFSIZ);
+    setvbuf (stderr, NULL, _IONBF, BUFSIZ);
     Database db;
-    db.InitDatabase("sounds.db");
+    db.InitDatabase("out.db");
 
     DatFile *a;
 
@@ -34,8 +39,9 @@ int main() {
         std::cout << "Starting magic...\n";
         std::cout << "Files number: " << a->files_number() << std::endl;
         a->WriteUnorderedDictionary("");
+        a->PatchAllDatabase(&db);
 
-        SubfileData data;
+//        SubfileData data;
 //        data.binary_data.ReadFromFile("167855806.ogg");
 //        data.options["fid"] = "167855806";
 //        data.options["ext"] = ".ogg";
@@ -46,12 +52,6 @@ int main() {
 //
 //        BinaryData old_data = a->GetFileData(i);
 //        old_data.WriteToFile("test_old_file");
-
-        data = db.GetNextFile();
-        while (data != SubfileData()) {
-            a->PatchFile(data);
-            data = db.GetNextFile();
-        }
     } catch (std::exception &e) {
         fprintf(stderr, "Caught %s exception.", e.what());
         printf("Caught %s exception.", e.what());