Fixed and documented DatExporter module. Updated LotRO_dat_extractor.exe

CMakeLists.txt

@@ -5,7 +5,7 @@ set(CMAKE_CXX_STANDARD 14)
 set(PROJECT_BINARY_DIR bin)
 set(PROJECT_VERSION 5.2.0)
 
-SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} -O3 -Wall -Wextra")
+SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} -Wall -Wextra")
 SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}")
 
 if (MSVS)

include/DatOperationResult.h

@@ -71,12 +71,11 @@ namespace LOTRO_DAT {
         DatOperationResult(DatOperationResult<Output> &&other) noexcept
                 : __DatOperationResult_base(), value(std::move(other.value)) {}
 
-        template<typename OutputRef>
-        DatOperationResult(OutputRef &&output_, RESULT result_,
-                           const std::string &msg_ = std::string("No message provided")) : __DatOperationResult_base(
-                result_, msg_), value(std::forward<Output>(output_)) {}
+        DatOperationResult(Output &&output_, RESULT result_, const std::string &msg_ = std::string("No message provided")) : __DatOperationResult_base(
+                result_, msg_), value(output_) {}
 
-        operator Output() const { return value; }
+        DatOperationResult(const Output &output_, RESULT result_, const std::string &msg_ = std::string("No message provided")) : __DatOperationResult_base(
+                result_, msg_), value(output_) {}
 
         Output value;
     };

include/DatSubsystems/DatFileSystem.h

@@ -60,7 +60,7 @@ namespace LOTRO_DAT {
 
         DatOperationResult<> DeInit();
 
-        DatOperationResult<> PerformOperationOnAllFiles(std::function<void (std::shared_ptr<SubFile>&)> function);
+        DatOperationResult<> PerformOperationOnAllFiles(const std::function<void (std::shared_ptr<SubFile>&)>& function);
 
     private:
 

include/SubfileData.h

@@ -13,7 +13,7 @@ namespace LOTRO_DAT {
     struct SubfileData {
     public:
         SubfileData() {
-            binary_data = BinaryData();
+            binary_data = BinaryData(0);
             text_data = std::u16string();
             options = YAML::Node();
         }

src/DatFile.cpp

@@ -26,7 +26,6 @@ namespace LOTRO_DAT {
 
     DatFile::DatFile() {
         el::Configurations defaultConf;
-        el::Loggers::removeFlag(el::LoggingFlag::CreateLoggerAutomatically);
         el::Loggers::addFlag(el::LoggingFlag::NewLineForContainer);
         el::Loggers::addFlag(el::LoggingFlag::LogDetailedCrashReason);
         el::Loggers::addFlag(el::LoggingFlag::ImmediateFlush);
@@ -38,13 +37,13 @@ namespace LOTRO_DAT {
 
 
         defaultConf.setGlobally(el::ConfigurationType::ToFile, "true");
-        defaultConf.setGlobally(el::ConfigurationType::Filename, "dat_library.log");
+        //defaultConf.setGlobally(el::ConfigurationType::Filename, "dat_library.log");
         defaultConf.setGlobally(el::ConfigurationType::ToStandardOutput, "false");
         defaultConf.setGlobally(el::ConfigurationType::PerformanceTracking, "true");
         defaultConf.setGlobally(el::ConfigurationType::MaxLogFileSize, "5242880"); // 5MB
 
         defaultConf.set(el::Level::Debug, el::ConfigurationType::Enabled, "false");
-        defaultConf.set(el::Level::Debug, el::ConfigurationType::Filename, "logs/dat_library_debug.log");
+        defaultConf.set(el::Level::Debug, el::ConfigurationType::Filename, "dat_library_debug.log");
 
         el::Loggers::reconfigureAllLoggers(defaultConf);
 
@@ -124,6 +123,7 @@ namespace LOTRO_DAT {
         fileSystem_->PrintInformaion(out);
         localeManager_->PrintInformaion(out);
 
+        fclose(out);
         return DatOperationResult<>(SUCCESS);
     }
 
@@ -143,6 +143,8 @@ namespace LOTRO_DAT {
         if (operation.result != SUCCESS)
             return DatOperationResult<>(ERROR, "DATDEINIT: Error, cannot deinitialize. msg: " + operation.msg);
 
+        initialized_ = false;
+
         return DatOperationResult<>();
     }
 

src/DatSubsystems/DatExporter.cpp

@@ -13,13 +13,27 @@ namespace LOTRO_DAT {
     DatExporter::DatExporter(DatFile *datFilePtr) : dat(datFilePtr) {
     }
 
+    /*!
+     * \author Gi1dor
+     * \date 05.07.2018
+     * Извлечение в папку всех файлов, имеющих заданный тип
+     * \param[in] type Тип файлов
+     * \param[in] output_directory_path путь к папке, куда будут записаны файлы. Имя экспортируемого файла - file_id.ext, где file_id - id файла, ext - его расширение
+     *
+     * \returns DatOperationResult.value - количество извлечённых файлов
+     */
+
+
     DatOperationResult<int> DatExporter::ExtractAllFilesByType(const FILE_TYPE &type, std::string output_directory_path) {
         int success_exported = 0;
 
         auto operation = dat->getFileSystem().PerformOperationOnAllFiles([&success_exported, this, type, &output_directory_path](std::shared_ptr<SubFile>& file) -> void {
-            if (file->FileType() != type || !dat->getFileSystem().CheckCorrectSubfile(file))
+            if (file->FileType() != type)
+                return;
+            SubfileData export_file = file->PrepareForExport(dat->getFileSystem().GetFileData(file->file_id(), 8).value);
+
+            if (export_file.Empty())
                 return;
-            SubfileData export_file = file->PrepareForExport(dat->getFileSystem().GetFileData(file->file_id(), 8));
 
             bool result;
 
@@ -34,6 +48,15 @@ namespace LOTRO_DAT {
         return DatOperationResult<int>(success_exported, SUCCESS);
     }
 
+
+    /*!
+     * \author Gi1dor
+     * \date 05.07.2018
+     * Извлечение файла по его id
+     * \param[in] file_id id файла
+     * \param[in] output_filename - имя файла, куда будет записаны данные.
+     */
+
     DatOperationResult<> DatExporter::ExtractFileById(long long file_id, std::string output_filename) {
         auto operation_GetFileData = dat->getFileSystem().GetFileData(file_id, 8);
         if (operation_GetFileData.result == ERROR)
@@ -44,27 +67,45 @@ namespace LOTRO_DAT {
         if (operation_GetFileData.result == ERROR)
             return DatOperationResult<>(ERROR, "EXTRACTFILEBYID: Can't get pointer for id = " + std::to_string(file_id));
 
-        SubfileData export_data = operation_GetFilePointer.value->PrepareForExport(operation_GetFileData.value);
+        std::shared_ptr<SubFile> file = operation_GetFilePointer.value;
+
+        SubfileData export_data = file->PrepareForExport(operation_GetFileData.value);
 
-        if (export_data.Empty())
-            return DatOperationResult<>(ERROR, "EXTRACTFILEBYID: Export data is empty for id = " + std::to_string(file_id));
+        bool result;
 
-        if (!export_data.binary_data.WriteToFile(output_filename + export_data.options["ext"].as<std::string>()))
-            return DatOperationResult<>(ERROR, "EXTRACTFILEBYID: Cannot write to file" + output_filename);
+        if (file->FileType() == TEXT)
+            result = WriteStringToFile(export_data.text_data, output_filename  + file->Extension()).result;
+        else
+            result = export_data.binary_data.WriteToFile(output_filename + file->Extension());
+
+        if (!result)
+            return DatOperationResult<>(ERROR, "EXTRACTFILEBYID: Cannot write to file" + output_filename + file->Extension());
 
         return DatOperationResult<>(SUCCESS);
     }
 
+
+    /*!
+     * \author Gi1dor
+     * \date 05.07.2018
+     * Извлечение в базу данных всех файлов, имеющих заданный тип
+     * \param[in] type Тип файлов
+     * \param[in] db Указатель на объект базы данных
+     *
+     * \returns DatOperationResult.value - количество извлечённых файлов
+     */
+
+
     DatOperationResult<int> DatExporter::ExtractAllFilesByType(const FILE_TYPE &type, Database *db) {
         if (!db)
             return DatOperationResult<int>(0, ERROR, "EXTRACTALLBYTYPETODB: database is nullptr");
 
         int success_exported = 0;
         auto operation = dat->getFileSystem().PerformOperationOnAllFiles([&success_exported, this, type, db](std::shared_ptr<SubFile>& file) {
-            if (!dat->getFileSystem().CheckCorrectSubfile(file))
+            if (file->FileType() != type)
                 return;
 
-            SubfileData export_file = file->PrepareForExport(dat->getFileSystem().GetFileData(file->file_id(), 8));
+            SubfileData export_file = file->PrepareForExport(dat->getFileSystem().GetFileData(file->file_id(), 8).value);
 
             bool result = db->PushFile(export_file);
 
@@ -74,13 +115,25 @@ namespace LOTRO_DAT {
         return DatOperationResult<int>(success_exported, SUCCESS);
     }
 
+
+    /*!
+     * \author Gi1dor
+     * \date 05.07.2018
+     * Запись строки в формате UTF-16 в файл
+     * \param[in] str Строка для записи в файл
+     * \param[in] path Путь к файлу, в который строка будет записана
+     *
+     * \warning Путь к файлу должен быть корректным (папки должны существовать). Если и сам файл существует, то он будет перезаписан
+     */
+
     DatOperationResult<> DatExporter::WriteStringToFile(const std::u16string &str, const std::string &path) {
-        FILE* file = fopen(path.c_str(), "w");
-        if (!file)
+        std::basic_ofstream<char16_t> output_stream(path, std::ios::out);
+        if (!output_stream.is_open())
             return DatOperationResult<>(ERROR, "WRITESTRINGTOFILE: cant open file " + path);
 
-        fprintf(file, "%s", (char *) str.c_str());
-        fclose(file);
+        output_stream << str;
+        output_stream.close();
+
         return DatOperationResult<>();
     }
 

src/DatSubsystems/DatFileSystem.cpp

@@ -149,6 +149,8 @@ namespace LOTRO_DAT {
         if (!dat)
             return DatOperationResult<bool>(false, ERROR,
                                             "DATFSCORRECTSUBFILE: no connection with Dat (dat is nullptr)");
+        if (file->file_size() < 16)
+            return DatOperationResult<bool>(false, SUCCESS);
 
         BinaryData mfile_id(20);
         auto operation = dat->getIO().ReadData(mfile_id, 20, file->file_offset() + 8);
@@ -171,8 +173,9 @@ namespace LOTRO_DAT {
             return DatOperationResult<>(ERROR, "DATFSCOMMITDIRS: no connection with Dat (dat is nullptr)");
 
         for (auto file_id : subfile_pending_update) {
-            if (dictionary_[file_id] == nullptr)
+            if (dictionary_.count(file_id) == 0)
                 continue;
+
             auto operation = CheckCorrectSubfile(dictionary_[file_id]);
             if (operation.result == ERROR) {
                 LOG(ERROR) << "Check subfile correctness failed. Error message: " << operation.msg;
@@ -239,6 +242,7 @@ namespace LOTRO_DAT {
 
         if (operation.result == ERROR)
             return DatOperationResult<>(ERROR, "DEINITFS: unable to commit directories.");
+
         return DatOperationResult<>(SUCCESS);
     }
 
@@ -312,11 +316,17 @@ namespace LOTRO_DAT {
 
         auto initialised_file = SubFile::MakeSubfile(*dat, file);
 
-        if (dictionary_.count(file_id) > 0 && CheckCorrectSubfile(initialised_file)) {
+        if (!initialised_file)
+            return DatOperationResult<>(ERROR, "DATFSINITSUBFILE: initialised subfile pointer is empty");
+
+        if (!CheckCorrectSubfile(initialised_file).value)
+            return DatOperationResult<>(ERROR, "DATFSINITSUBFILE: initialised file " + std::to_string(file.file_id()) + "is incorrect");
+
+        if (dictionary_.count(file_id) > 0) {
             LOG(WARNING) << "Dublicate files id = " << file_id << "dictionary offsets = "
                          << dictionary_[file_id]->dictionary_offset() << " and "
                          << initialised_file->dictionary_offset();
-            if (CheckCorrectSubfile(dictionary_[file_id]))
+            if (CheckCorrectSubfile(dictionary_[file_id]).value)
                 LOG(ERROR) << "    FILE IN DICTIONARY IS ALREADY CORRECT!";
 
             dictionary_[file_id] = initialised_file;
@@ -364,7 +374,6 @@ namespace LOTRO_DAT {
             return DatOperationResult<>(ERROR, "DATFSINITALLFILES: no connection with Dat (dat is nullptr)");
 
         InitAllDirectories();
-        std::cout << "DONE ALL DIRS" << std::endl;
 
         while (!subfile_init_queue_.empty()) {
             SubFile file = *subfile_init_queue_.begin();
@@ -474,17 +483,15 @@ namespace LOTRO_DAT {
     /*!
      * \author Gi1dor
      * \date 03.07.2018
-     * Выполнение функции function для всех обнаруженных файлов в dat контейнере.
+     * Выполнение функции function для всех файлов в dat контейнере.
      *
-     * \param[in] function Функтор, принимающий в качестве аргумента указатель на объект файла std::shared_ptr<SubFile>
+     * \param[in] function Функтор, принимающий в качестве аргумента ссылку на объект-указатель std::shared_ptr<SubFile>
      */
 
-    DatOperationResult<> DatFileSystem::PerformOperationOnAllFiles(std::function<void (std::shared_ptr<SubFile>&)> function) {
+    DatOperationResult<> DatFileSystem::PerformOperationOnAllFiles(const std::function<void (std::shared_ptr<SubFile>&)>& function) {
         InitAllFiles();
-        for (const auto &file_pair: dictionary_) {
-            auto file = file_pair.second;
-            function(file);
-        }
+        for (auto file_pair: dictionary_)
+            function(file_pair.second);
         return DatOperationResult<>();
     }
 }

src/DatSubsystems/DatIO.cpp

@@ -24,7 +24,7 @@ namespace LOTRO_DAT {
      * \param[in] datFile Указатель на объект управляющего класса DatFile
      */
 
-    DatIO::DatIO(DatFile *datFile) : dat(datFile), file_handler_(nullptr), filename_(), actual_dat_size_(0) {
+    DatIO::DatIO(DatFile *datFile) : dat(datFile), file_handler_(nullptr), filename_("none"), actual_dat_size_(0) {
     }
 
 
@@ -316,6 +316,8 @@ namespace LOTRO_DAT {
      */
 
     DatOperationResult<std::string> DatIO::GetFilename() {
+        if (filename_ == "none")
+            return DatOperationResult<std::string>("", ERROR, "DATIOGETFILENAME: not initialised");
         return DatOperationResult<std::string>(filename_, SUCCESS);
     }
 
@@ -324,7 +326,7 @@ namespace LOTRO_DAT {
      * \date 30.06.2018
      * Выводит информацию о состоянии модуля в файл
      * \param[in] file указатель на объект файла, в конец которого будет напечатана информация
-     * \warning Файл file должен быть открыт для записи
+     * \warning Файл file должен быть открыт для записи. После завершения работы функции файл остаётся открытым
      */
 
     void DatIO::PrintInformaion(FILE *file) {

src/DatSubsystems/DatLocaleManager.cpp

@@ -23,7 +23,7 @@ namespace LOTRO_DAT {
         dat->getIO().ReadData(locale_offset_data, 4, 300);
         long long locale_offset = locale_offset_data.ToNumber<4>(0);
 
-        if (locale_offset == 0 || locale_offset + 8 >= dat->getIO().GetActualDatSize()) {
+        if (locale_offset == 0 || locale_offset + 8 >= dat->getIO().GetActualDatSize().value) {
             LOG(INFO) << "Dictionary offset is empty or incorrect. Passing.";
             return DatOperationResult<>();
         }
@@ -38,9 +38,8 @@ namespace LOTRO_DAT {
         LOG(INFO) << "Dictionary size is " << dict_size << ". Version is " << dict_version << ". Localed .dat size = " << dat->getIO().file_size;
 
         if (dict_version != 101) {
-            LOG(WARNING) << "DICTIONARY IS OLD! Version = " << dict_version;
             dat->getIO().WriteData(BinaryData::FromNumber<4>(0), 4, 300);
-            return DatOperationResult<>(ERROR, "Version of locales' dictionary is old (ver = " + std::to_string(dict_version) + "), cannot work with this type of .dat file.");
+            return DatOperationResult<>(SUCCESS, "Version of locales' dictionary is old (ver = " + std::to_string(dict_version) + "), cannot initialize locales.");
         }
 
         BinaryData dicts_data = BinaryData((unsigned)dict_size);
@@ -132,6 +131,11 @@ namespace LOTRO_DAT {
         if (!dat)
             return DatOperationResult<>(ERROR, "LOCALEDEINIT: no connection with Dat (dat is nullptr)");
 
+        if (patch_dict_.empty()) {
+            dat->getIO().WriteData(BinaryData::FromNumber<4>(0), 4, 300);
+            return DatOperationResult<>(SUCCESS);
+        }
+
         BinaryData binary_data = BinaryData(4 + 4 + 4 + 14 + 15 + 4
                                             + 4 + (32 + 4) * orig_dict_.size()
                                             + 4 + (32 + 4) * patch_dict_.size()

src/Examples/extractor_example.cpp

@@ -15,6 +15,7 @@ using namespace LOTRO_DAT;
 using namespace std;
 
 // Change these variables to true if you want export category to files.
+bool exportTextsToFiles = false;
 bool exportImagesToFiles = false;
 bool exportFontsToFiles = false;
 bool exportSoundsToFiles = false;
@@ -31,7 +32,7 @@ bool exportUnknownToDb = false;
 // There is no need to change anything else
 
 int main() {
-    std::cout << "Gi1dor's LotRO .dat extractor ver. 5.2.0" << std::endl;
+    std::cout << "Gi1dor's LotRO .dat extractor ver. 7.0.0" << std::endl;
 
     std::cout << "Hello! I'm a basic shell version of .dat file extractor. I can open .dat file directly, "
             "if you write path to it (with name of file) in file \"dat_file_path.txt\"\n";
@@ -66,7 +67,7 @@ int main() {
     std::cout << "Great! File initialised successfully!\n";
     std::cout << "Gathering file information...";
     file.GatherInformation("dat_info.log");
-    
+
     int cmd = 0;
     while (true) {
         std::cout << "Please, choose, what should I do. I can extract .dat file to files (press 1), "
@@ -79,6 +80,7 @@ int main() {
             std::string tmp;
             std::getline(std::cin, tmp);
         }
+
         if (cmd == -1) {
             std::cout << "Exiting. Thanks for using me!\n";
             break;
@@ -96,8 +98,8 @@ int main() {
             out_time[24] = '\0';
 
             std::string filename = file.getIO().GetFilename().value;
-            
-            std::string output_dir = std::string("Extracted data\\") + filename.substr(filename.length() - 16, 16) + " " + std::string(out_time) + "\\";
+            std::cout << "FILENAME = " << filename << std::endl;
+            std::string output_dir = std::string("Extracted data\\") + filename.substr(std::max(filename.length() - 16, 0u), 16) + " " + std::string(out_time) + "\\";
             std::replace(output_dir.begin(), output_dir.end(), ':', '-');
 
             mkdir(output_dir.c_str(), 744);
@@ -165,7 +167,10 @@ int main() {
                 std::cout << "Extracted " << file.getExporter().ExtractAllFilesByType(UNKNOWN, output_dir + "unknown\\").value << " unknown files to directory" << std::endl << std::flush;
             }
 
-            file.Deinitialize();
+            if (exportTextsToFiles) {
+                mkdir((output_dir + "texts").c_str(), 744);
+                std::cout << "Extracted " << file.getExporter().ExtractAllFilesByType(TEXT, output_dir + "texts\\").value << " text files to directory" << std::endl << std::flush;
+            }
 
             fprintf(stdout, "Spent %f seconds on running unpacker! Thank you for your patience!\n",
                     float(clock() - begin_time) / CLOCKS_PER_SEC);
@@ -195,7 +200,8 @@ int main() {
 
         if (cmd == 3) {
             std::cout << "Current parameters:\n";
-            std::cout << "(01) Export texts to database - " << (exportTextsToDb ? "true\n" : "false\n");
+            std::cout << "(01) Export texts to files - " << (exportTextsToFiles ? "true\n" : "false\n");
+            std::cout << "(02) Export texts to database - " << (exportTextsToDb ? "true\n" : "false\n");
 
             std::cout << "(11) Export images to files - " << (exportImagesToFiles ? "true\n" : "false\n");
             std::cout << "(12) Export images to database - " << (exportImagesToDb ? "true\n" : "false\n");
@@ -226,6 +232,9 @@ int main() {
             } else {
                 switch (number) {
                     case 01:
+                        exportTextsToFiles = !exportTextsToFiles;
+                        break;
+                    case 02:
                         exportTextsToDb = !exportTextsToDb;
                         break;
                     case 11:
@@ -264,6 +273,7 @@ int main() {
             }
         }
     }
+
     file.Deinitialize();
     system("pause");
     return 0;