Written basic import functions

Added functions for managing with "fragmentation journal" - special dictionary, where free parts of .dat file are stored. Implemented and slightly tested importing and patching images to .dat file. This is non-stable commit, needs to be tested and refactored

Source/BinaryData.cpp

@@ -39,8 +39,10 @@ namespace LOTRO_DAT {
 
     BinaryData BinaryData::operator +(const BinaryData &b) {
         BinaryData res(size_ + b.size());
-        memcpy(res.data_, data_, size_);
-        memcpy(res.data_ + size_, b.data_, b.size_);
+        if (size_ > 0)
+            memcpy(res.data_, data_, size_);
+        if (b.size() > 0)
+            memcpy(res.data_ + size_, b.data_, b.size_);
         return res;
     }
 
@@ -86,8 +88,7 @@ namespace LOTRO_DAT {
         if (T < 0)
             throw DatException("Bad BinaryData::FromNumber() - trying to make data from amount of bytes < 0");
         try {
-            if (size_ != 0 && data_ != nullptr)
-                delete[] data_;
+            delete[] data_;
 
             size_ = size_t(T);
             data_ = new unsigned char[size_];
@@ -166,6 +167,29 @@ namespace LOTRO_DAT {
 		return WriteToFile(filename.c_str());
 	}
 
+
+    void BinaryData::ReadFromFile(const char *filename) {
+        FILE *f;
+        fopen_s(&f, filename, "rb");
+        if (f == nullptr) {
+            throw DatException("Bad BinaryData::WriteToFile() - unable to open output file", EXPORT_EXCEPTION);
+        }
+
+        _fseeki64(f, 0, SEEK_END);
+        long long file_size = ftell(f);
+        _fseeki64(f, 0, SEEK_SET);
+
+        BinaryData temp_data = BinaryData(unsigned(file_size));
+
+        fread(temp_data.data_, temp_data.size_, 1, f);
+        *this = temp_data;
+        fclose(f);
+    }
+
+    void BinaryData::ReadFromFile(const std::string &filename) {
+        ReadFromFile(filename.c_str());
+    }
+
     bool BinaryData::CheckCompression() const {
         if (size() < 10)
             return false;
@@ -217,6 +241,7 @@ namespace LOTRO_DAT {
         return newdata;
     }
 
+
     template long long BinaryData::ToNumber<1>(long long const&) const;
     template long long BinaryData::ToNumber<2>(long long const&) const;
     template long long BinaryData::ToNumber<4>(long long const&) const;

Source/BinaryData.h

@@ -47,6 +47,10 @@ namespace LOTRO_DAT
 		bool WriteToFile(const char *filename) const;
 		bool WriteToFile(const std::string &filename) const;
 
+        void ReadFromFile(const char *filename);
+
+        void ReadFromFile(const std::string &filename);
+
         bool CheckCompression() const;
 		BinaryData DecompressData(unsigned int offset = 0) const;
         BinaryData CompressData(unsigned int offset = 0) const;

Source/Common/DatException.h

@@ -27,7 +27,7 @@ namespace LOTRO_DAT
         UNKNOWN_EXCEPTION
     };
 
-    class DatException : std::exception {
+    class DatException : public std::exception {
     public:
         DatException()
                 : msg_(const_cast<char *>("Unknown DatException has been raised!"))

Source/DatFile.cpp

@@ -29,7 +29,8 @@ namespace LOTRO_DAT {
 
         try {
             MakeDictionary();
-        } catch (...) {
+        } catch (std::exception &e) {
+            fprintf(stderr, "Caught %s exception.", e.what());
             fprintf(stderr, "Unable to make dictionary!! Unable to init DatFile!!!");
             return;
         }
@@ -44,7 +45,9 @@ namespace LOTRO_DAT {
         if (patched_) {
             std::cout << "There are some updated files. Rewriting dictionary..." << std::endl << std::flush;
             UpdateHeader();
+            std::cout << "Updated header..." << std::endl << std::flush;
             UpdateSubdirectories();
+            std::cout << "Updated subdirectories..." << std::endl << std::flush;
         }
 
         if (file_handler_ != nullptr)
@@ -61,7 +64,7 @@ namespace LOTRO_DAT {
     /// Returns true, if file was successfully extracted;
     /// Throws DatException() if undefined behaviour happened
 
-    bool DatFile::ExtractFile(long long file_id, const std::string path) {
+    bool DatFile::ExtractFile(long long file_id, const std::string &path) {
         if (dat_state_ != READY) {
             throw DatException("Bad DatFile::ExtractFile() - invalid DatFile state!", EXPORT_EXCEPTION);
         }
@@ -69,7 +72,9 @@ namespace LOTRO_DAT {
 
         try {
             file_data = GetFileData(dictionary_[file_id], 8);
-        } catch (...) {
+        } catch (std::exception &e) {
+            fprintf(stderr, "Caught %s exception.", e.what());
+
             fprintf(stderr, "Unable to extract file due to uncaught exception while getting file data. Passing...\n");
             return false;
         }
@@ -81,7 +86,9 @@ namespace LOTRO_DAT {
 
         try {
             dictionary_[file_id]->PrepareForExport(file_data, export_size, binary_data, text_data, options);
-        } catch (...) {
+        } catch (std::exception &e) {
+            fprintf(stderr, "Caught %s exception.", e.what());
+
             fprintf(stderr, "Unable to extract file due to uncaught exception while preparing file for export. Passing...\n");
             return false;
         }
@@ -108,7 +115,9 @@ namespace LOTRO_DAT {
 
         try {
             file_data = GetFileData(dictionary_[file_id], 8);
-        } catch (...) {
+        } catch (std::exception &e) {
+            fprintf(stderr, "Caught %s exception.", e.what());
+
             fprintf(stderr, "Unable to extract file due to uncaught exception while getting file data. Passing...\n");
             return false;
         }
@@ -120,7 +129,9 @@ namespace LOTRO_DAT {
 
         try {
             dictionary_[file_id]->PrepareForExport(file_data, export_size, binary_data, text_data, options);
-        } catch (...) {
+        } catch (std::exception &e) {
+            fprintf(stderr, "Caught %s exception.", e.what());
+
             fprintf(stderr, "Unable to extract file due to uncaught exception while preparing file for export. Passing...\n");
             return false;
         }
@@ -130,7 +141,11 @@ namespace LOTRO_DAT {
             option << options[i];
             try {
                 db->PushFile(binary_data[i], text_data[i], option.str());
-            } catch (...) {
+            } catch (std::exception &e) {
+                fprintf(stderr, "Caught %s exception.", e.what());
+                printf("Caught %s exception.", e.what());
+                fflush(stdout);
+
                 fprintf(stderr, "Unable to put file or it's part to database. Continuing without this part. Database may be not complete\n");
             }
         }
@@ -179,6 +194,39 @@ namespace LOTRO_DAT {
         return success;
     }
 
+    /// TODO: Write desctiption
+    bool DatFile::PatchFile(const char *filename, YAML::Node options, long long dat_id) {
+        if (dat_id != dat_id_)
+            return false;
+
+        BinaryData data;
+        data.ReadFromFile(filename);
+
+        auto file_id = options["fid"].as<long long>();
+        BinaryData old_data = GetFileData(dictionary_[file_id]);
+        data = dictionary_[file_id]->MakeForImport(old_data, data, u"", options);
+
+        try {
+            ApplyFilePatch(dictionary_[file_id], data);
+        } catch (std::exception &e) {
+            fprintf(stderr, "Caught %s exception.", e.what());
+
+            fprintf(stderr,
+                    "Some errors happened while patching file with id = %lld. Continuing process without this file..\n"
+                            "WARNING: DAT FILE CAN BE CORRUPTED!\n", file_id);
+            printf("Some errors happened while patching file with id = %lld. Continuing process without this file..\n"
+                           "WARNING: DAT FILE CAN BE CORRUPTED!\n", file_id);
+            fflush(stdout);
+            return false;
+        }
+        return true;
+    }
+
+    /// TODO: Write description
+    bool DatFile::PatchDatabase(Database *db) {
+        return false;
+    }
+
     /// DatFile::WriteUnorderedDictionary(...);
     /// Prints list of all found files with some information about them to file.
     /// Gets std::string path - path to directory, where the file will be written with name "dict.txt"
@@ -330,7 +378,9 @@ namespace LOTRO_DAT {
             throw DatException("Bad DatFile::MakeDictionary() - DatFile isn't in valid state!", INIT_EXCEPTION);
         try {
             root_directory_->MakeDictionary(dictionary_);
-        } catch (...) {
+        } catch (std::exception &e) {
+            fprintf(stderr, "Caught %s exception.", e.what());
+
             fprintf(stderr, "Bad DatFile::MakeDictionary() - File is corrupted?\n");
             return;
         }
@@ -374,22 +424,51 @@ namespace LOTRO_DAT {
     /// Special functions used by patch process.
     /// Shouldn't be used by any external class.
 
-    void DatFile::ApplyFilePatch(const Subfile *file, const BinaryData &data) {
+    void DatFile::ApplyFilePatch(Subfile *file, const BinaryData &data) {
+        if (patched_list.count(file->file_id()) != 0) {
+            return;
+        }
+
         auto journal = GetFragmentationJournal();
 
-        // TODO: write content
+        file->file_size_ = data.size() - 8;
+        file->fragments_count_ = 0;
+
+        if (data.size() > file->block_size()) {
+            file->file_offset_ = journal[0].second;
+            file->block_size_ = data.size();
+            journal[0].second += data.size();
 
+            if (journal[0].first < data.size()) {
+                UpdateFileSize();
+                journal[0].first = 0;
+                std::cout << "File size updated - new size is " << file_size_ << std::endl;
+            } else {
+                journal[0].first -= data.size();
+            }
+        }
+
+        BinaryData fragments_count(4);
+        fragments_count.FromNumber<4>(file->fragments_count_);
+
+        BinaryData file_data = fragments_count + data.CutData(4);
+
+        if (file->file_id() != file_data.ToNumber<4>(8))
+            throw DatException("Bad DatFile::ApplyFilePatch() - Created data's file_id doesn't match to original! "
+                                       "Patch wasn't written to .dat file");
+
+        WriteData(file_data, file_data.size(), file->file_offset());
+        patched_list[file->file_id()] = new BinaryData(file->MakeHeaderData());
         UpdateFragmentationJournal(journal);
-        patched_ = true;
     }
 
     void DatFile::UpdateSubdirectories() {
-        root_directory_ -> UpdateDirectories();
-        return;
+        root_directory_->UpdateDirectories(patched_list);
     }
 
     std::vector<std::pair<long long, long long> > DatFile::GetFragmentationJournal() {
         BinaryData data(8);
+        std::cout << "journal offset: " << fragmentation_journal_offset_ << std::endl;
         ReadData(data, 8, fragmentation_journal_offset_ + 8);
         std::vector<std::pair<long long, long long> > result;
         result.emplace_back(std::make_pair(data.ToNumber<4>(0), data.ToNumber<4>(4)));
@@ -405,6 +484,9 @@ namespace LOTRO_DAT {
         data.FromNumber<4>(constant2_);
         WriteData(data, 4, 0x140);
 
+        data.FromNumber<4>(file_size_);
+        WriteData(data, 4, 0x148);
+
         data.FromNumber<4>(version1_);
         WriteData(data, 4, 0x14C);
 
@@ -412,14 +494,14 @@ namespace LOTRO_DAT {
         WriteData(data, 4, 0x150);
 
         data.FromNumber<4>(fragmentation_journal_offset_);
-        WriteData(data, 4, 0x150);
+        WriteData(data, 4, 0x154);
 
         data.FromNumber<4>(root_directory_offset_);
-        WriteData(data, 4, 0x150);
+        WriteData(data, 4, 0x160);
     }
 
     void DatFile::UpdateFragmentationJournal(const std::vector<std::pair<long long, long long> > &journal) {
-        for (int i = 0; i < journal.size(); i++) {
+        for (unsigned i = 0; i < journal.size(); i++) {
             long long size = journal[i].first;
             long long offset = journal[i].second;
 
@@ -431,5 +513,10 @@ namespace LOTRO_DAT {
             WriteData(data, 4, fragmentation_journal_offset_ + 8 * (i + 1) + 4);
         }
     }
+
+    void DatFile::UpdateFileSize() {
+        _fseeki64(file_handler_, 0, SEEK_END);
+        file_size_ = _ftelli64(file_handler_);
+    }
 }
 }

Source/DatFile.h

@@ -16,6 +16,7 @@
 #include <unordered_map>
 #include <set>
 #include <vector>
+#include <yaml-cpp/node/node.h>
 #include "Database.h"
 
 // Dat file names definitions
@@ -61,12 +62,16 @@ namespace LOTRO_DAT
         explicit DatFile(const char* filename, int dat_id);
         ~DatFile();
 
-        bool ExtractFile(long long file_id, const std::string path = "");
+        bool ExtractFile(long long file_id, const std::string &path = "");
         bool ExtractFile(long long file_id, Database *db);
 
         int ExtractAllFilesByType(FILE_TYPE type, std::string path = "");
         int ExtractAllFilesByType(FILE_TYPE type, Database *db);
 
+        bool PatchFile(const char *filename, YAML::Node options, long long dat_id);
+
+        bool PatchDatabase(Database *db);
+
         void WriteUnorderedDictionary(std::string path) const;
 
         long long constant1() const;
@@ -79,6 +84,8 @@ namespace LOTRO_DAT
 
         BinaryData GetFileData(const Subfile* file, long long offset = 0);
 
+        void UpdateSubdirectories();
+
     private:
         FILE *file_handler_;
         SubDirectory *root_directory_;
@@ -92,12 +99,15 @@ namespace LOTRO_DAT
         void ReadData(BinaryData &data, long long size, long long offset = 0, long long data_offset = 0);
         void WriteData(const BinaryData &data, long long size, long long offset = 0, long long data_offset = 0);
 
-        void ApplyFilePatch(const Subfile* file, const BinaryData &data);
-        void UpdateSubdirectories();
+        void ApplyFilePatch(Subfile *file, const BinaryData &data);
         std::vector<std::pair<long long, long long> > GetFragmentationJournal();
         void UpdateHeader();
         void UpdateFragmentationJournal(const std::vector<std::pair<long long, long long> > &journal);
 
+        void UpdateFileSize();
+
+        std::unordered_map<long long, BinaryData *> patched_list;
+
         long long constant1_;
         long long constant2_;
         long long file_size_;

Source/LotroDatPatcher.h

@@ -4,3 +4,6 @@
 
 #include "DatFile.h"
 #include "Database.h"
+#include "BinaryData.h"
+#include <yaml-cpp/yaml.h>
+#include "Common\ZLib\zlib.h"

Source/SubDirectory.cpp

@@ -25,7 +25,9 @@ namespace LOTRO_DAT {
             dat_(dat), offset_(offset), max_subdirs_(max_subdirs) {
         try {
             MakeSubDirectories();
-        }catch (...) {
+        } catch (std::exception &e) {
+            fprintf(stderr, "Caught %s exception.", e.what());
+            
             fprintf(stderr, "Unable to initialize directory at offset %lld. Initializing it as empty directory...\n",
                     offset);
             subdirs_.clear();
@@ -35,7 +37,9 @@ namespace LOTRO_DAT {
 
         try {
             MakeSubFiles();
-        }catch (...) {
+        } catch (std::exception &e) {
+            fprintf(stderr, "Caught %s exception.", e.what());
+            
             fprintf(stderr, "Unable to initialize directory at offset %lld. Initializing it as empty directory...\n",
                     offset);
             subdirs_.clear();
@@ -66,7 +70,9 @@ namespace LOTRO_DAT {
                         )
                 );
 
-            }catch (...) {
+            } catch (std::exception &e) {
+                fprintf(stderr, "Caught %s exception.", e.what());
+            
                 fprintf(stderr, "Making SubDirectory at offset %lld failed, continuing\n", data.ToNumber<4>(i + 4));
             }
         }
@@ -79,7 +85,7 @@ namespace LOTRO_DAT {
         for (unsigned int i = 0; i < 61 * 32; i += 32) {
             if (data.ToNumber<4>(i + 8) < 0x32 || data.ToNumber<4>(i + 12) < 0x32)
                 continue;
-            subfiles_.emplace_back(
+            subfiles_.push_back(
                     MakeSubfile(
                             data.ToNumber<4>(i), // fragments_count
                             data.ToNumber<4>(i + 4), // unknown1
@@ -104,12 +110,15 @@ namespace LOTRO_DAT {
             i.MakeDictionary(dict);
     }
 
-    void SubDirectory::UpdateDirectories() {
+    void SubDirectory::UpdateDirectories(std::unordered_map<long long, BinaryData *> &patched_files) {
         for (unsigned i = 0; i < subfiles_.size(); i++) {
-            dat_->WriteData(subfiles_[i]->MakeHeaderData(), 32, offset_ + 63ll * 8ll + i * 32);
+            if (patched_files.count(subfiles_[i]->file_id()) != 0) {
+                dat_->WriteData(*patched_files[subfiles_[i]->file_id()], 32, offset_ + 63ll * 8ll + i * 32);
+            }
         }
-        for (unsigned i = 0; i < subdirs_.size(); i++)
-            subdirs_[i].UpdateDirectories();
+
+        for (SubDirectory &i : subdirs_)
+            i.UpdateDirectories(patched_files);
     }
 
     Subfile *SubDirectory::MakeSubfile(long long fragments_count, long long unknown1, long long file_id,

Source/SubDirectory.h

@@ -25,7 +25,8 @@ namespace LOTRO_DAT
         SubDirectory();
         SubDirectory(long long offset, DatFile *dat, long long max_subdirs = 63);
         void MakeDictionary(std::unordered_map<long long, Subfile*> &dict);
-        void UpdateDirectories();
+
+        void UpdateDirectories(std::unordered_map<long long, BinaryData *> &patched_files);
 
     private:
         void MakeSubDirectories();

Source/Subfile.cpp

@@ -15,7 +15,7 @@ const long long MAXSIZE = 50ll * 1024ll * 1024ll; // Setting maximal file size 5
                                                   // of Subfiles and Subdirectories search in DatFile.
 namespace LOTRO_DAT {
 
-    Subfile::Subfile() {}
+    Subfile::Subfile() = default;
 
     Subfile::Subfile(DatFile *dat, long long fragments_count, long long unknown1, long long file_id,
                                 long long file_offset,
@@ -108,7 +108,7 @@ namespace LOTRO_DAT {
     }
 
     BinaryData Subfile::MakeHeaderData() {
-        BinaryData header(32);
+        BinaryData header;
         BinaryData data(4);
         data.FromNumber<4>(fragments_count_);
         header = header + data;
@@ -126,6 +126,7 @@ namespace LOTRO_DAT {
         header = header + data;
         data.FromNumber<4>(block_size_);
         header = header + data;
+        return header;
     }
 
 };

Source/Subfile.h

@@ -20,6 +20,7 @@ namespace LOTRO_DAT
 
     class Subfile
     {
+        friend class DatFile;
     public:
         Subfile();
         Subfile(DatFile *dat, long long fragments_count, long long unknown1, long long file_id, long long file_offset,

Source/Subfiles/JpgSubfile.cpp

@@ -8,7 +8,7 @@
 #include "../Common/DatException.h"
 
 namespace LOTRO_DAT {
-    JpgSubfile::JpgSubfile() {}
+    JpgSubfile::JpgSubfile() = default;
 
     JpgSubfile::JpgSubfile(DatFile *dat, long long fragments_count, long long unknown1,
                              long long file_id, long long file_offset, long long file_size,
@@ -42,6 +42,8 @@ namespace LOTRO_DAT {
             !options["fid"] || options["fid"].as<long long>() != file_id()) {
             throw DatException("Bad DdsSubfile::MakeForImport() - invalid options data!", IMPORT_EXCEPTION);
         }
-        return old_data.CutData(0, 40) + binary_data;
+        BinaryData file_size;
+        file_size.FromNumber<4>(binary_data.size());
+        return old_data.CutData(0, 28) + file_size + binary_data;
     }
 };

Source/Subfiles/TextSubfile.cpp

@@ -67,11 +67,11 @@ namespace LOTRO_DAT {
             text_data.emplace_back(text);
             options.emplace_back(YAML::Node());
 
-            options[export_size]["fid"] = file_id();
-            options[export_size]["gid"] = fragment_id;
-            options[export_size]["ext"] = Extension();
+            options[unsigned(export_size)]["fid"] = file_id();
+            options[unsigned(export_size)]["gid"] = fragment_id;
+            options[unsigned(export_size)]["ext"] = Extension();
             if (!arg_references.empty())
-                options[export_size]["args"] = arguments;
+                options[unsigned(export_size)]["args"] = arguments;
 
             ++export_size;
         }
@@ -101,7 +101,8 @@ namespace LOTRO_DAT {
             std::u16string piece;
 
             for (long long k = 0; k < piece_size; k++) {
-                char16_t c = char16_t(((short(piece_data[2 * k + 1])) << 8) | (short(piece_data[2 * k])));
+                char16_t c = char16_t(
+                        ((short(piece_data[2 * unsigned(k) + 1])) << 8) | (short(piece_data[2 * unsigned(k)])));
                 piece += c;
             }
 
@@ -143,7 +144,7 @@ namespace LOTRO_DAT {
                 }
                 offset += 1;
 
-                arg_strings[j].emplace_back(data.CutData(offset, offset + string_size * 2));
+                arg_strings[unsigned(j)].emplace_back(data.CutData(offset, offset + string_size * 2));
                 offset += string_size * 2;
             }
         }

Source/Tests/extract_test.cpp

@@ -20,10 +20,10 @@ const std::string path = "";//"E:\\SteamLibrary\\steamapps\\common\\";//Lord Of
 const std::string filename = "client_local_English.dat";
 
 // Change these variables to true if you want export catecory to files.
-const bool exportImagesToFiles = false;
+const bool exportImagesToFiles = true;
 const bool exportFontsToFiles = false;
 const bool exportSoundsToFiles = false;
-const bool exportTexturesToFiles = true;
+const bool exportTexturesToFiles = false;
 const bool exportUnknownToFiles = false;
 
 // Change these variables to true if you want export catecory to databases.
@@ -42,7 +42,7 @@ int main() {
 
     std::time_t result = std::time(nullptr);
     char *ttime = std::asctime(std::localtime(&result));
-    char *out_time = new char[25];
+    auto *out_time = new char[25];
     memcpy(out_time, ttime, 24);
     out_time[24] = '\0';
 
@@ -126,7 +126,8 @@ int main() {
             mkdir((output_dir + "unknown").c_str(), 744);
             std::cout << "Extracted " << a.ExtractAllFilesByType(FONT, output_dir + "unknown\\") << " unknown files to directory" << std::endl << std::flush;
         }
-    } catch (...) {
+    } catch (std::exception &e) {
+        fprintf(stderr, "Caught %s exception.", e.what());
         printf("Some critical errors occured. Need to stop execution. See information in errors.log file\n");
         fprintf(stderr, "Some critical errors occured. Need to stop execution now...\n");
     }

Source/Tests/patch_test.cpp

@@ -7,11 +7,11 @@
 #include <algorithm>
 
 #ifdef WIN32
-#include <direct.h>
 #define mkdir(dir, mode) _mkdir(dir)
 #endif
 
 #include "../LotroDatPatcher.h"
+
 using namespace LOTRO_DAT;
 
 // Change these 2 variables to your path and name of .dat file
@@ -24,12 +24,28 @@ const int file_id = 0;
 int main() {
     const clock_t begin_time = clock();
 
-    freopen("errors.log", "w", stderr);
+    BinaryData t;
+    t.ReadFromFile(patch_filename);
+    t.WriteToFile("kek.jpg");
+
+    //freopen("errors.log", "w", stderr);
     try {
         std::cout << "Starting magic...\n";
         DatFile a((path + filename).c_str(), 0);
-        std::cout << "Magic number: " << a.files_number() << std::endl;
-    } catch (...) {
+        std::cout << "Files number: " << a.files_number() << std::endl;
+        a.WriteUnorderedDictionary("");
+
+        YAML::Node options;
+        options["fid"] = "1090551300";
+        options["ext"] = ".jpg";
+        a.PatchFile(patch_filename.c_str(), options, 0);
+        //a.PatchFile(patch_filename.c_str(), options, 0);
+        a.UpdateSubdirectories();
+    } catch (std::exception &e) {
+        fprintf(stderr, "Caught %s exception.", e.what());
+        printf("Caught %s exception.", e.what());
+        fflush(stdout);
+
         printf("Some critical errors occured. Need to stop execution. See information in errors.log file");
         fprintf(stderr, "Some critical errors occured. Need to stop execution now...");
     }