4-th part of refactoring

Almost completed division between files, partly implemented new extracting
files interface in DatFile class

CMakeLists.txt

@@ -66,19 +66,23 @@ set(SOURCE_FILES
 add_library(LotroDat_static STATIC ${SOURCE_FILES})
 target_link_libraries(LotroDat_static ${CMAKE_SOURCE_DIR}/Common/ZLib/libzlibstatic.a)
 target_link_libraries(LotroDat_static ${CMAKE_SOURCE_DIR}/Common/Yaml/libyaml-cpp.a)
+target_link_libraries(LotroDat_static -static-libgcc -static-libstdc++ -Wl,-Bstatic -lstdc++ -lpthread -Wl,-Bdynamic)
 
 # SHARED LIBRARY
 add_library(LotroDat SHARED ${SOURCE_FILES})
 target_link_libraries(LotroDat ${CMAKE_SOURCE_DIR}/Common/ZLib/libzlibstatic.a)
-target_link_libraries(LotroDat_static ${CMAKE_SOURCE_DIR}/Common/Yaml/libyaml-cpp.a)
+target_link_libraries(LotroDat ${CMAKE_SOURCE_DIR}/Common/Yaml/libyaml-cpp.a)
+target_link_libraries(LotroDat -static-libgcc -static-libstdc++ -Wl,-Bstatic -lstdc++ -lpthread -Wl,-Bdynamic)
 
 # FOR EXTRACTOR USE NEXT LINE
 add_executable(LotRO_dat_extract_tester ${SOURCE_FILES} ${CMAKE_SOURCE_DIR}/Tests/extract_test.cpp)
 target_link_libraries(LotRO_dat_extract_tester ${CMAKE_SOURCE_DIR}/Common/ZLib/libzlibstatic.a)
-target_link_libraries(LotroDat_static ${CMAKE_SOURCE_DIR}/Common/Yaml/libyaml-cpp.a)
+target_link_libraries(LotRO_dat_extract_tester ${CMAKE_SOURCE_DIR}/Common/Yaml/libyaml-cpp.a)
+target_link_libraries(LotRO_dat_extract_tester -static-libgcc -static-libstdc++ -Wl,-Bstatic -lstdc++ -lpthread -Wl,-Bdynamic)
 
 # FOR PATCHER USE NEXT LINE
 add_executable(LotRO_dat_pat_tester ${SOURCE_FILES} ${CMAKE_SOURCE_DIR}/Tests/patch_test.cpp)
 target_link_libraries(LotRO_dat_pat_tester ${CMAKE_SOURCE_DIR}/Common/Zlib/libzlibstatic.a)
-target_link_libraries(LotroDat_static ${CMAKE_SOURCE_DIR}/Common/Yaml/libyaml-cpp.a)
+target_link_libraries(LotRO_dat_pat_tester ${CMAKE_SOURCE_DIR}/Common/Yaml/libyaml-cpp.a)
+target_link_libraries(LotRO_dat_pat_tester -static-libgcc -static-libstdc++ -Wl,-Bstatic -lstdc++ -lpthread -Wl,-Bdynamic)
 

Source/DatFile.cpp

@@ -8,6 +8,7 @@
 #include "Common/DatException.h"
 #include "SubDirectory.h"
 #include "Subfile.h"
+#include "yaml-cpp/yaml.h"
 
 #include <locale>
 
@@ -21,13 +22,10 @@ namespace LOTRO_DAT {
     DatFile::DatFile(const char *filename, int dat_id) {
         dat_id_ = dat_id;
         dat_state_ = CLOSED;
-        printf("state1\n");
         OpenDatFile(filename);
-        printf("state2\n");
         ReadSuperBlock();
-        printf("state3\n");
         MakeDirectories();
-        printf("end\n");
+
         try {
             MakeDictionary();
         } catch (...) {
@@ -60,7 +58,16 @@ namespace LOTRO_DAT {
         if (dat_state_ != READY) {
             throw DatException("Bad DatFile::ExtractFile() - invalid DatFile state!", EXPORT_EXCEPTION);
         }
-        return dictionary_[file_id]->ExportFile(path.c_str());
+        BinaryData file_data = GetFileData(dictionary_[file_id], 8);
+        long long export_size = 0;
+        std::vector<BinaryData> binary_data;
+        std::vector<std::u16string> text_data;
+        std::vector<YAML::Node> options;
+        dictionary_[file_id]->PrepareForExport(file_data, export_size, binary_data, text_data, options);
+
+        for (int i = 0; i < export_size; ++i) {
+            binary_data[i].WriteToFile(path + "_" + std::to_string(i) + options[i]["extension"].as<std::string>());
+        }
     }
 
     /// Extracts file with file_id to database "db".
@@ -74,7 +81,14 @@ namespace LOTRO_DAT {
         if (dat_state_ != READY) {
             throw DatException("Bad DatFile::ExtractFile() - invalid DatFile state!", EXPORT_EXCEPTION);
         }
-        return dictionary_[file_id]->ExportFile(db);
+
+        BinaryData file_data = GetFileData(dictionary_[file_id], 8);
+        long long export_size = 0;
+        std::vector<BinaryData> binary_data;
+        std::vector<std::u16string> text_data;
+        std::vector<YAML::Node> options;
+        dictionary_[file_id]->PrepareForExport(file_data, export_size, binary_data, text_data, options);
+        // TODO: Complete this function
     }
 
     /// Extracts all files with specific type to "path + type + file_id + extension" files;
@@ -91,9 +105,9 @@ namespace LOTRO_DAT {
 
         int success = 0;
         for (auto i : dictionary_) {
-            FILE_TYPE ext = i.second->ext();
+            FILE_TYPE ext = i.second->FileType();
             if (ext == type) {
-                success += i.second->ExportFile((path + std::to_string(i.second->file_id())).c_str());
+                success += ExtractFile(i.second->file_id(), (path + std::to_string(i.second->file_id())));
             }
         }
         return success;
@@ -111,55 +125,14 @@ namespace LOTRO_DAT {
 
         int success = 0;
         for (auto i : dictionary_) {
-            FILE_TYPE ext = i.second->ext();
+            FILE_TYPE ext = i.second->FileType();
             if (ext == type) {
-                success += i.second->ExportFile(db);
+                success += ExtractFile(i.second->file_id(), db);
             }
         }
         return success;
     }
 
-    /// Patches .dat with text file with specific file_id, gossip_id and values, equal to values in text database.
-    /// std::string text contains of text data, surrounded by '[' and ']', where <--DO_NOT_TOUCH!--> represents
-    ///     position of variables in order, described in args_order.
-    /// std::string args_order contains of numbers, divided by '-'. Ex. "1-2-3-4". There should be not less numbers, than
-    ///     <--DO_NOT_TOUCH!--> pieces in text variable/
-    /// std::string args contains of numbers, divided by ' '. These numbers are references to variables. There should
-    ///     be not less references, than the biggest number in args_order
-    /// Returns true if text was succesfully patched.
-    /// Throws DatException() if undefined behaviour happened;
-
-    bool DatFile::PatchTextFile(long long file_id, long long gossip_id, std::string text, std::string args_order,
-                                std::string args) {
-        return false;
-    }
-
-    /// Patches .dat with text file with specific file_id and gossip_id.
-    /// All text file data is got from database.
-    /// Returns true if text file was successfully patched.
-    /// Throws DatException() if undefined behaviour happened;
-
-    bool DatFile::PatchTextFile(long long file_id, long long gossip_id, Database *db) {
-        return false;
-    }
-
-    /// Patches .dat with binary file with specific file_id.
-    /// All file data is got from file in "file_path".
-    /// Returns true if file was successfully patched.
-    /// Throws DatException() if undefined behaviour happened;
-
-    bool DatFile::PatchBinaryFile(long long file_id, std::string file_path) {
-        return false;
-    }
-
-    /// Patches .dat with binary file with specific file_id.
-    /// All file data is got from Database "db".
-    /// Returns true if file was successfully patched.
-    /// Throws DatException() if undefined behaviour happened;
-
-    bool DatFile::PatchBinaryFile(long long file_id, Database *db) {
-        return false;
-    }
 
     void DatFile::OpenDatFile(const char *dat_name) {
         if (dat_state_ != CLOSED)
@@ -291,16 +264,8 @@ namespace LOTRO_DAT {
         fopen_s(&f, (path + "dict.txt").c_str(), "w");
         fprintf(f, "file_id offset size size2 extension\n");
         for (auto i : dictionary_) {
-            std::string extension = "unk";
-            FILE_TYPE ext = i.second->ext();
-            if (ext == TEXT) extension = "txt";
-            if (ext == JPG) extension = "jpg";
-            if (ext == DDS) extension = "dds";
-            if (ext == WAV) extension = "wav";
-            if (ext == OGG) extension = "ogg";
-            if (ext == FONT) extension = "font";
             fprintf(f, "%lld %lld %lld %lld %s\n", i.second->file_id(), i.second->file_offset(), i.second->file_size(),
-                    i.second->block_size(), extension.c_str());
+                    i.second->block_size(), i.second->Extension());
         }
         fclose(f);
     }
@@ -308,5 +273,40 @@ namespace LOTRO_DAT {
     long long DatFile::files_number() const {
         return dictionary_.size();
     }
+
+    BinaryData DatFile::GetFileData(const Subfile *file, long long int offset = 0) {
+        BinaryData mfile_id(4);
+        ReadData(mfile_id, 4, file->file_offset() + 8);
+        if (file->file_id() != mfile_id.ToNumber<4>(0))
+            throw DatException("Bad DatFile::GetFileData() - file_id in Subfile doesn't match to file_id in DatFile.", READ_EXCEPTION);
+
+        BinaryData data((unsigned)(file->file_size()));
+        if (file->block_size() >= file->file_size() + 8) {
+            ReadData(data, file->file_size(), file->file_offset() + offset);
+            return data;
+        }
+
+        BinaryData fragments_count(4);
+        ReadData(fragments_count, 4, file->file_offset());
+
+        long long fragments_number = fragments_count.ToNumber<4>(0);
+
+        long long current_block_size = file->block_size() - offset - 8 * fragments_number;
+
+        ReadData(data, current_block_size , file->file_offset() + offset);
+
+        BinaryData FragmentsDictionary(8 * unsigned(fragments_number));
+        ReadData(FragmentsDictionary, 8 * unsigned(fragments_number), file->file_offset() + file->block_size() - 8 * fragments_number);
+
+
+        for (long long i = 0; i < fragments_number; i++) {
+            long long fragment_size = FragmentsDictionary.ToNumber<4>(8 * i);
+            long long fragment_offset = FragmentsDictionary.ToNumber<4>(8 * i + 4);
+            ReadData(data, std::min(fragment_size, file->file_size() - current_block_size), fragment_offset, current_block_size );
+            current_block_size += fragment_size;
+        }
+
+        return data;
+    }
 }
 }

Source/DatFile.h

@@ -55,7 +55,6 @@ namespace LOTRO_DAT
     class DatFile
     {
         friend class SubDirectory;
-        friend class Subfile;
     public:
         DatFile();
         explicit DatFile(const char* filename, int dat_id);
@@ -67,12 +66,6 @@ namespace LOTRO_DAT
         int ExtractAllFilesByType(FILE_TYPE type, std::string path = "");
         int ExtractAllFilesByType(FILE_TYPE type, Database *db);
 
-        bool PatchTextFile(long long file_id, long long gossip_id, std::string text, std::string args_order = "", std::string args = "");
-        bool PatchTextFile(long long file_id, long long gossip_id, Database *db);
-
-        bool PatchBinaryFile(long long file_id, std::string file_path);
-        bool PatchBinaryFile(long long file_id, Database *db);
-
         void WriteUnorderedDictionary(std::string path) const;
 
         long long constant1() const;
@@ -83,7 +76,7 @@ namespace LOTRO_DAT
 
         long long files_number() const;
 
-        void ReadData(BinaryData &data, long long size, long long offset = 0, long long data_offset = 0);
+        BinaryData GetFileData(const Subfile* file, long long offset = 0);
 
     private:
         FILE *file_handler_;
@@ -95,6 +88,7 @@ namespace LOTRO_DAT
         void MakeDirectories();
         void MakeDictionary();
 
+        void ReadData(BinaryData &data, long long size, long long offset = 0, long long data_offset = 0);
         void WriteData(const BinaryData &data, long long size, long long offset = 0, long long data_offset = 0);
 
         long long constant1_;

Source/SubDirectory.cpp

@@ -112,19 +112,19 @@ namespace LOTRO_DAT {
 
         switch (type) {
             case TEXT:
-                return dynamic_cast<Subfile *>(new TextSubfile(fragments_count, unknown1, file_id, file_offset, file_size, timestamp, version, block_size));
+                return dynamic_cast<Subfile *>(new TextSubfile(dat_, fragments_count, unknown1, file_id, file_offset, file_size, timestamp, version, block_size));
             case JPG:
-                return dynamic_cast<Subfile *>(new JpgSubfile(fragments_count, unknown1, file_id, file_offset, file_size, timestamp, version, block_size));
+                return dynamic_cast<Subfile *>(new JpgSubfile(dat_, fragments_count, unknown1, file_id, file_offset, file_size, timestamp, version, block_size));
             case DDS:
-                return dynamic_cast<Subfile *>(new DdsSubfile(fragments_count, unknown1, file_id, file_offset, file_size, timestamp, version, block_size));
+                return dynamic_cast<Subfile *>(new DdsSubfile(dat_, fragments_count, unknown1, file_id, file_offset, file_size, timestamp, version, block_size));
             case WAV:
-                return dynamic_cast<Subfile *>(new WavSubfile(fragments_count, unknown1, file_id, file_offset, file_size, timestamp, version, block_size));
+                return dynamic_cast<Subfile *>(new WavSubfile(dat_, fragments_count, unknown1, file_id, file_offset, file_size, timestamp, version, block_size));
             case OGG:
-                return dynamic_cast<Subfile *>(new OggSubfile(fragments_count, unknown1, file_id, file_offset, file_size, timestamp, version, block_size));
+                return dynamic_cast<Subfile *>(new OggSubfile(dat_, fragments_count, unknown1, file_id, file_offset, file_size, timestamp, version, block_size));
             case FONT:
-                return dynamic_cast<Subfile *>(new FontSubfile(fragments_count, unknown1, file_id, file_offset, file_size, timestamp, version, block_size));
+                return dynamic_cast<Subfile *>(new FontSubfile(dat_, fragments_count, unknown1, file_id, file_offset, file_size, timestamp, version, block_size));
             default:
-                return dynamic_cast<Subfile *>(new UnknownSubfile(fragments_count, unknown1, file_id, file_offset, file_size, timestamp, version, block_size));
+                return dynamic_cast<Subfile *>(new UnknownSubfile(dat_, fragments_count, unknown1, file_id, file_offset, file_size, timestamp, version, block_size));
         }
     }
 

Source/Subfile.cpp

@@ -6,15 +6,15 @@
 #include "BinaryData.h"
 #include "DatFile.h"
 #include "Common/DatException.h"
-#include "Database.h"
-#include "Common/CommonFunctions.h"
 
 #include <algorithm>
 
 const long long MAXSIZE = 50ll * 1024ll * 1024ll; // Setting maximal file size 50 MB; Files with size more than 50 mb
                                                   // will be recognized as incorrect and passed.
-                                                  // This should be done because
+                                                  // This should be done because of not completely correct implementation
+                                                  // of Subfiles and Subdirectories search in DatFile.
 namespace LOTRO_DAT {
+
     Subfile::Subfile() {}
 
     Subfile::Subfile(DatFile *dat, long long fragments_count, long long unknown1, long long file_id,
@@ -29,6 +29,8 @@ namespace LOTRO_DAT {
                                SUBFILE_EXCEPTION);
     }
 
+    /// Typical getters of private fields, if there's need for getting information about SubFile from outside class.
+
     long long Subfile::fragments_count() const {
         return fragments_count_;
     }
@@ -61,19 +63,45 @@ namespace LOTRO_DAT {
         return block_size_;
     }
 
+    /// bool Subfile::FileType(...);
+    /// Virtual function, can (and should) be redefined in child class, otherwise an exception will be thrown while exporting/importing file.
+    /// Returns enum FILE_TYPE value, which is declared in DatFile.h
+
     FILE_TYPE Subfile::FileType() const {
         throw DatException("Bad Subfile::FileType() - function is not implemented for this type.", SUBFILE_EXCEPTION);
     }
 
+    /// std::string Subfile::Extension()
+    /// Virtual function, can (and should) be redefined in child class, otherwise an exception will be thrown while exporting/importing file.
+    /// Returns std::string with extension, beggined with '.', ex. ".jpg", ".txt" and so on;
+
     std::string Subfile::Extension() const {
         throw DatException("Bad Subfile::Extension() - function is not implemented for this type.", SUBFILE_EXCEPTION);
     }
 
+    /// bool Subfile::PrepareForExport(...);
+    /// Virtual function, can be redefined in child class, otherwise an exception will be thrown while exporting file.
+    /// Takes constant BinaryData& file_data, which contains all file data in .dat file, except first 8 bytes before file_id.
+    /// Takes references to export_size - amount of exported files/strings, and content of exported data such as:
+    ///  1) field binary_data - exported as RAW data
+    ///  2) field text_data - UTF-16 text, exporting in UTF-8
+    ///  3) field options - YAML field, which consists of some parameters of file such as file_id, extension and so on.
+    /// Returns true if preparation was success. Otherwise returns false;
+
     bool Subfile::PrepareForExport(const BinaryData &file_data, long long &export_size, std::vector<BinaryData> &binary_data,
                                         std::vector<std::u16string> &text_data, std::vector<YAML::Node> &options) {
         throw DatException("Bad Subfile::PrepareForExport() - function is not implemented for this type.", EXPORT_EXCEPTION);
     }
 
+
+    /// BinaryData Subfile::PrepareForImport(...);
+    /// Virtual function, can be redefined in child class, otherwise an exception will be thrown while importing file.
+    /// Takes constant BinaryData& file_data, which contains all file data in .dat file, including first 8 bytes befire file_id.
+    ///  1) const field binary_data - exported as RAW data
+    ///  2) const field text_data - UTF-16 text, exporting in UTF-8
+    ///  3) const field options - YAML field, which consists of some parameters of file such as file_id, extension and so on.
+    /// Returns BinaryData - bytes array, prepared for writing in .dat file
+
     BinaryData Subfile::MakeForImport(const BinaryData &old_data, const BinaryData &binary_data, const std::u16string &text_data,
                            const YAML::Node &options) {
         throw DatException("Bad Subfile::MakeForImport() - function is not implemented for this type.", IMPORT_EXCEPTION);

Source/Subfile.h

@@ -8,7 +8,6 @@
 #include <string>
 #include <vector>
 #include "yaml-cpp/yaml.h"
-#include "BinaryData.h"
 
 extern "C++"
 {
@@ -23,7 +22,7 @@ namespace LOTRO_DAT
     {
     public:
         Subfile();
-        Subfile::Subfile(DatFile *dat, long long fragments_count, long long unknown1, long long file_id, long long file_offset,
+        Subfile(DatFile *dat, long long fragments_count, long long unknown1, long long file_id, long long file_offset,
                                     long long file_size, long long timestamp, long long version, long long block_size);
 
         virtual FILE_TYPE FileType() const;

Source/Subfiles/DdsSubfile.cpp

@@ -16,9 +16,6 @@ namespace LOTRO_DAT {
                                       long long timestamp,
                                       long long version, long long block_size)
             : Subfile(dat, fragments_count, unknown1, file_id, file_offset, file_size, timestamp, version, block_size) {
-        BinaryData header(20);
-        dat_->ReadData(header, 10, file_offset_ + 8);
-        compressed_ = header.CheckCompression();
     }
 
     FILE_TYPE DdsSubfile::FileType() const {
@@ -32,7 +29,7 @@ namespace LOTRO_DAT {
     bool DdsSubfile::PrepareForExport(const BinaryData &file_data, long long &export_size, std::vector<BinaryData> &binary_data,
                                       std::vector<std::u16string> &text_data, std::vector<YAML::Node> &options) {
         BinaryData data = file_data;
-        if (compressed_)
+        if (data.CheckCompression())
             data = data.DecompressData(4);
 
         BinaryData ddsData(data.size() - 24 + 128);
@@ -154,7 +151,7 @@ namespace LOTRO_DAT {
             !options["file_id"] || options["file_id"].as<long long>() != file_id()) {
             throw DatException("Bad DdsSubfile::MakeForImport() - invalid options data!", IMPORT_EXCEPTION);
         }
-        if (compressed_)
+        if (old_data.CheckCompression())
             return old_data.CutData(0, 12) + (old_data.CutData(12, 16) + binary_data.CutData(128)).CompressData();
         else
             return old_data.CutData(0, 16) + binary_data.CutData(128);

Source/Subfiles/DdsSubfile.h

@@ -8,7 +8,7 @@
 #include "../Subfile.h"
 
 namespace LOTRO_DAT {
-    class DdsSubfile : Subfile {
+    class DdsSubfile : public Subfile {
     public:
         DdsSubfile();
         DdsSubfile(DatFile *dat, long long fragments_count, long long unknown1, long long file_id, long long file_offset,
@@ -22,9 +22,6 @@ namespace LOTRO_DAT {
 
         BinaryData MakeForImport(const BinaryData &old_data, const BinaryData &binary_data,
                                  const std::u16string &text_data, const YAML::Node &options) override;
-
-    private:
-        bool compressed_;
     };
 };
 

Source/Subfiles/FontSubfile.h

@@ -9,7 +9,8 @@
 #include "../Subfile.h"
 
 namespace LOTRO_DAT {
-    class FontSubfile : Subfile {
+    class FontSubfile : public Subfile {
+    public:
         FontSubfile();
         FontSubfile(DatFile *dat, long long fragments_count, long long unknown1, long long file_id, long long file_offset,
                    long long file_size, long long timestamp, long long version, long long block_size);

Source/Subfiles/JpgSubfile.h

@@ -9,7 +9,8 @@
 #include "../Subfile.h"
 
 namespace LOTRO_DAT {
-    class JpgSubfile : Subfile {
+    class JpgSubfile : public Subfile {
+    public:
         JpgSubfile();
         JpgSubfile(DatFile *dat, long long fragments_count, long long unknown1, long long file_id, long long file_offset,
                    long long file_size, long long timestamp, long long version, long long block_size);

Source/Subfiles/OggSubfile.h

@@ -8,10 +8,11 @@
 #include "../Subfile.h"
 
 namespace LOTRO_DAT {
-    class OggSubfile : Subfile {
+    class OggSubfile : public Subfile {
+    public:
         OggSubfile();
         OggSubfile(DatFile *dat, long long fragments_count, long long unknown1, long long file_id, long long file_offset,
-                   long long file_size, long long timestamp, long long version, long long block_size) override;
+                   long long file_size, long long timestamp, long long version, long long block_size);
 
         FILE_TYPE FileType() const override;
         std::string Extension() const override;

Source/Subfiles/TextSubfile.h

@@ -9,7 +9,7 @@
 #include <vector>
 
 namespace LOTRO_DAT {
-    class TextSubfile : Subfile {
+    class TextSubfile : public Subfile {
     public:
         TextSubfile();
 

Source/Subfiles/UnknownSubfile.h

@@ -8,7 +8,8 @@
 #include "../Subfile.h"
 
 namespace LOTRO_DAT {
-    class UnknownSubfile : Subfile {
+    class UnknownSubfile : public Subfile {
+    public:
         UnknownSubfile();
         UnknownSubfile(DatFile *dat, long long fragments_count, long long unknown1, long long file_id, long long file_offset,
                    long long file_size, long long timestamp, long long version, long long block_size);

Source/Subfiles/WavSubfile.h

@@ -8,7 +8,8 @@
 #include "../Subfile.h"
 
 namespace LOTRO_DAT {
-    class WavSubfile : Subfile {
+    class WavSubfile : public Subfile {
+    public:
         WavSubfile();
         WavSubfile(DatFile *dat, long long fragments_count, long long unknown1, long long file_id, long long file_offset,
                    long long file_size, long long timestamp, long long version, long long block_size);