Bug fixes. Added exporting texts (only to db), fonts and unknowns. Speed improvements

CHANGELOG

@@ -0,0 +1,5 @@
+----------------------------------------------------------------------
+Version 0.0.1
+    * First version, development state
+----------------------------------------------------------------------
+    

CMakeLists.txt

@@ -15,6 +15,7 @@ set(SOURCE_FILES
         SubDirectory.cpp SubDirectory.h
 
         sqlite/sqlite3.c sqlite/sqlite3.h
+		CommonFunctions.h
         )
 
 #add_library(LotRO_dat_Patcher SHARED ${SOURCE_FILES})

CommonFunctions.h

@@ -0,0 +1,39 @@
+#ifndef COMMON_FUNCTIONS_H
+#define COMMON_FUNCTIONS_H
+
+#include <string>
+#include <uchar.h>
+#include <codecvt>
+#include <locale>
+
+inline long long mstrlen(const char16_t *s) {
+	return sizeof(s) / sizeof(*s);
+}
+
+inline std::string u16stringToBytes(const std::u16string& str)
+{
+#if defined(_MSC_VER)
+	auto p = reinterpret_cast<unsigned short const*>(str.data());
+	auto out = std::wstring_convert<std::codecvt_utf16<unsigned short>, unsigned short>().to_bytes(p, p + str.size());
+	return out;
+#else
+	std::wstring_convert<std::codecvt_utf16<char16_t>, char16_t> conv;
+	return conv.to_bytes(str);
+#endif
+}
+
+inline std::u16string bytesToU16string(const std::string& str)
+{
+#if defined(_MSC_VER)
+	std::u16string out;
+	auto s = std::wstring_convert<std::codecvt_utf16<unsigned short>, unsigned short>().from_bytes(str);
+	auto p = reinterpret_cast<wchar_t const*>(s.data());
+	out.assign(p, p + s.size());
+	return out;
+#else
+	std::wstring_convert<std::codecvt_utf16<char16_t>, char16_t> conv;
+	return conv.from_bytes(str);
+#endif
+}
+
+#endif

Database.cpp

@@ -1,10 +1,14 @@
 //
 // Created by Иван_Архипов on 17.11.2017.
 //
+#define UNICODE
+#define _UNICODE 
 
 #include "Database.h"
 #include "DatException.h"
 #include "BinaryData.h"
+#include "CommonFunctions.h"
+
 #include <cstring>
 
 namespace LOTRO_DAT {
@@ -14,11 +18,6 @@ namespace LOTRO_DAT {
         query_size_ = 0;
     }
 
-    Database::Database(const char *filename) {
-        query_size_ = 0;
-        InitDatabase(filename);
-    }
-
     Database::Database(const std::string &filename) {
         query_size_ = 0;
         InitDatabase(filename.c_str());
@@ -36,49 +35,52 @@ namespace LOTRO_DAT {
         }
     }
 
-    void Database::InitDatabase(const char *filename) {
-        if (sqlite3_open(filename, &db_) != SQLITE_OK) {
+    void Database::InitDatabase(const std::string &filename) {
+        if (sqlite3_open(filename.c_str(), &db_) != SQLITE_OK) {
             sqlite3_close(db_);
             throw DatException("Bad Database::InitDatabase() - sqlite3_open returned an error..."
                     , DATABASE_EXCEPTION);
         }
 
+		ExecSql("PRAGMA synchronous = OFF");
+		ExecSql("PRAGMA count_changes = OFF");
+		ExecSql("PRAGMA journal_mode = MEMORY");
+		ExecSql("PRAGMA temp_store = MEMORY");
+		//ExecSql("PRAGMA encoding = \"UTF-8\";");
+
         ExecSql(CreateBinaryTableCommand_);
         ExecSql(CreateTextTableCommand_);
         ExecSql(CreateMetadataTableCommand_);
 
-        ExecSql("PRAGMA synchronous = OFF");
-        ExecSql("PRAGMA count_changes = OFF");
-        ExecSql("PRAGMA journal_mode = MEMORY");
-        ExecSql("PRAGMA temp_store = MEMORY");
-
-        sqlite3_prepare_v2(db_, InsertBinaryCommand_, strlen(InsertBinaryCommand_), &insert_binary_, nullptr);
-        sqlite3_prepare_v2(db_, InsertTextCommand_, strlen(InsertTextCommand_), &insert_text_, nullptr);
+		sqlite3_prepare_v2(db_, InsertBinaryCommand_.c_str(), InsertBinaryCommand_.length(), &insert_binary_, nullptr);
+        sqlite3_prepare_v2(db_, InsertTextCommand_.c_str(), InsertTextCommand_.length(), &insert_text_, nullptr);
 
         //sqlite3_prepare_v2(db_, GetTextCommand, 50ll * 1024ll * 1024ll, &get_text_, nullptr);
         //sqlite3_prepare_v2(db_, GetBinaryCommand, 50ll * 1024ll * 1024ll, &get_binary_, nullptr);
 
     }
 
-    void Database::ExecSql(const char *sql) {
-        char *error;
-        if (sqlite3_exec(db_, sql, nullptr, nullptr, &error) != SQLITE_OK) {
-            fprintf(stderr, "SQLite3 error: %s\n", sqlite3_errmsg(db_));
-            throw DatException((std::string("Bad Database::ExecSql() - unable to perform request")
-                                + std::string(sql)).c_str(), DATABASE_EXCEPTION);
-        }
-    }
+	void Database::ExecSql(const std::string &sql) {
+		char *error;
+		if (sqlite3_exec(db_, sql.c_str(), nullptr, nullptr, &error) != SQLITE_OK) {
+			fprintf(stderr, "SQLite3 error: %s\n", sqlite3_errmsg(db_));
+			throw DatException((std::string("Bad Database::ExecSql() - unable to perform request")
+				+ std::string(sql)).c_str(), DATABASE_EXCEPTION);
+		}
+	}
+
 
-    void Database::PushTextFile(long long file_id, long long gossip_id, char *text, char *args, int dat_id) {
+    void Database::PushTextFile(long long file_id, long long gossip_id, const char16_t *text, const char *args, int dat_id) {
         if (query_size_ == 0)
             ExecSql("BEGIN TRANSACTION");
+        query_size_++;
 
         sqlite3_bind_int(insert_text_, 1, (int)file_id);
         sqlite3_bind_int(insert_text_, 2, (int)gossip_id);
-        sqlite3_bind_text(insert_text_, 3, text, sizeof(text), SQLITE_TRANSIENT);
-        sqlite3_bind_text(insert_text_, 4, args, sizeof(args), SQLITE_TRANSIENT);
+        int a = sqlite3_bind_text16(insert_text_, 3, text, -1, SQLITE_TRANSIENT);
+        int b = sqlite3_bind_text(insert_text_, 4, args, -1, SQLITE_TRANSIENT);
 
-        if (sqlite3_step(insert_text_) != SQLITE_OK) {
+        if (sqlite3_step(insert_text_) != SQLITE_DONE) {
             fprintf(stderr, "SQLite3 error: %s\n", sqlite3_errmsg(db_));
             throw DatException((std::string("Bad Database::PushTextFile() - unable to perform push operation")).c_str(),
                                DATABASE_EXCEPTION);
@@ -86,7 +88,6 @@ namespace LOTRO_DAT {
 
         sqlite3_reset(insert_text_);
 
-        query_size_++;
         if (query_size_ >= QUERY_MAX_SIZE) {
             ExecSql("COMMIT");
             query_size_ = 0;
@@ -98,18 +99,18 @@ namespace LOTRO_DAT {
     void Database::PushBinaryFile(long long file_id, const BinaryData &data) {
         if (query_size_ == 0)
             ExecSql("BEGIN TRANSACTION");
+		query_size_++;
 
         sqlite3_bind_int(insert_binary_, 1, (int)file_id);
         sqlite3_bind_blob(insert_binary_, 2, data.data(), data.size(), SQLITE_TRANSIENT);
 
-        if (sqlite3_step(insert_binary_) != SQLITE_DONE) {
+		if (sqlite3_step(insert_binary_) != SQLITE_DONE) {
             fprintf(stderr, "SQLite3 error: %s\n", sqlite3_errmsg(db_));
             throw DatException(std::string("Bad Database::PushTextFile() - unable to perform operation").c_str()
                     , DATABASE_EXCEPTION);
         }
         sqlite3_reset(insert_binary_);
 
-        query_size_++;
         if (query_size_ >= QUERY_MAX_SIZE) {
             ExecSql("COMMIT");
             query_size_ = 0;

Database.h

@@ -17,14 +17,13 @@ namespace LOTRO_DAT
     class Database {
     public:
         Database();
-        explicit Database(const char *filename);
         explicit Database(const std::string &filename);
 
         ~Database();
 
-        void InitDatabase(const char* filename);
+        void InitDatabase(const std::string &filename);
 
-        void PushTextFile(long long file_id, long long gossip_id, char* text, char* args, int dat_id = 0);
+        void PushTextFile(long long file_id, long long gossip_id, const char16_t *text, const char *args, int dat_id = 0);
         //BinaryData& GetTextFile(long long file_id);
 
         void PushBinaryFile(long long file_id, const BinaryData& data);
@@ -33,7 +32,7 @@ namespace LOTRO_DAT
         void AddMetadata();
 
     private:
-        void ExecSql(const char *sql);
+        void ExecSql(const std::string &sql);
 
         sqlite3* db_;
         sqlite3_stmt* insert_text_;
@@ -43,21 +42,21 @@ namespace LOTRO_DAT
         sqlite3_stmt* get_binary_;
 
         unsigned long long query_size_;
-        const unsigned long long QUERY_MAX_SIZE = 256000;
+        const unsigned long long QUERY_MAX_SIZE = 210241024;
 
-        const char* CreateBinaryTableCommand_ = "CREATE TABLE IF NOT EXISTS `binary_data` ( "
+        const std::string CreateBinaryTableCommand_ = "CREATE TABLE IF NOT EXISTS `binary_data` ( "
             "`file_id` INTEGER NOT NULL DEFAULT '0', "
             "`data` BLOB, "
             "PRIMARY KEY (`file_id`));";
 
-        const char* CreateTextTableCommand_ = "CREATE TABLE IF NOT EXISTS `text_data` ( "
+        const std::string CreateTextTableCommand_ = "CREATE TABLE IF NOT EXISTS `text_data` ( "
             "`file_id` INTEGER NOT NULL DEFAULT '0', "
             "`gossip_id` INTEGER NOT NULL DEFAULT '0', "
             "`content` TEXT, "
             "`args` TEXT, "
             "PRIMARY KEY (`file_id`, `gossip_id`));";
 
-        const char* CreateMetadataTableCommand_ = "CREATE TABLE IF NOT EXISTS `meta_data` ( "
+        const std::string CreateMetadataTableCommand_ = "CREATE TABLE IF NOT EXISTS `meta_data` ( "
             "`name` TEXT NOT NULL, "
             "`description` TEXT NOT NULL, "
             "`date` TEXT NOT NULL, "
@@ -67,11 +66,11 @@ namespace LOTRO_DAT
             "`content_type` TEXT NOT NULL, "
             "PRIMARY KEY (`version`));";
 
-        const char* InsertTextCommand_ = "INSERT INTO text_data "
+        const std::string InsertTextCommand_ = "INSERT INTO text_data "
                 "(file_id, gossip_id, content, args) "
                 "VALUES (?, ?, ?, ?); ";
 
-        const char* InsertBinaryCommand_ = "INSERT INTO binary_data "
+        const std::string InsertBinaryCommand_ = "INSERT INTO binary_data "
                 "(file_id, data) "
                 "VALUES (?, ?); ";
 

LotroDatPatcher.h

@@ -9,4 +9,6 @@
 #include "Subfile.h"
 
 #include "DatException.h"
-#include "SubDirectory.h"
+#include "SubDirectory.h"
+
+#include "CommonFunctions.h"

SubDirectory.cpp

@@ -13,12 +13,12 @@ LOTRO_DAT::SubDirectory::SubDirectory() {
     offset_ = 0;
 }
 
-LOTRO_DAT::SubDirectory::SubDirectory(long long offset, DatFile *dat) :
-        offset_(offset), dat_(dat) {
+LOTRO_DAT::SubDirectory::SubDirectory(long long offset, DatFile *dat, long long max_subdirs) :
+        offset_(offset), dat_(dat), max_subdirs_(max_subdirs) {
     try {
         MakeSubDirectories();
     } catch (...) {
-		fprintf(stderr, "Unable to initialize directory at offset %lld. Initializing as empty directory...\n", offset);
+		fprintf(stderr, "Unable to initialize directory at offset %lld. Initializing it as empty directory...\n", offset);
 		subdirs_.clear();
 		subfiles_.clear();
 		return;
@@ -27,7 +27,7 @@ LOTRO_DAT::SubDirectory::SubDirectory(long long offset, DatFile *dat) :
     try {
         MakeSubFiles();
 	} catch (...) {
-		fprintf(stderr, "Unable to initialize directory at offset %lld. Initializing as empty directory...\n", offset);
+		fprintf(stderr, "Unable to initialize directory at offset %lld. Initializing it as empty directory...\n", offset);
 		subdirs_.clear();
 		subfiles_.clear();
 		return;
@@ -45,7 +45,7 @@ void LOTRO_DAT::SubDirectory::MakeSubDirectories() {
     }
 
     for (unsigned int i = 8; i < 63 * 8; i += 8) {
-        if (data.ToNumber<4>(i) == 0)
+        if (data.ToNumber<4>(i) == 0 || data.ToNumber<4>(i + 4) == 0)
             break;
 
         try {
@@ -87,10 +87,10 @@ void LOTRO_DAT::SubDirectory::MakeSubFiles() {
 
 void LOTRO_DAT::SubDirectory::MakeDictionary(std::unordered_map<long long, Subfile *> &dict) {
     for (Subfile &i : subfiles_) {
-        //if (dict.count(i.file_id()) != 0 && dict[i.file_id()]->file_offset() != i.file_offset())
-        //    throw DatException("Bad SubDirectory::MakeDictionary() - found 2 files with different offset!!!", SUBFILE_EXCEPTION);
-        if (dict.count(i.file_id()) == 0 || dict[i.file_id()]->timestamp() < i.timestamp()
-            || (dict[i.file_id()]->timestamp() == i.timestamp() && dict[i.file_id()]->block_size() < i.block_size()))
+            if (i.file_id() == 621002463)
+            fprintf(stderr, "YAY!!!! offset = %lld\n", i.file_offset());
+
+        if (dict.count(i.file_id()) == 0 || dict[i.file_id()]->timestamp() < i.timestamp())
 			dict[i.file_id()] = &i;
     }
     for (SubDirectory &i : subdirs_)

SubDirectory.h

@@ -22,7 +22,7 @@ namespace LOTRO_DAT
     {
     public:
         SubDirectory();
-        SubDirectory(long long offset, DatFile *dat);
+        SubDirectory(long long offset, DatFile *dat, long long max_subdirs = 63);
         void MakeDictionary(std::unordered_map<long long, Subfile*> &dict);
 
     private:
@@ -30,7 +30,8 @@ namespace LOTRO_DAT
         void MakeSubFiles();
 
         DatFile *dat_;
-        long long  offset_;
+        long long offset_;
+        long long max_subdirs_;
         std::vector<SubDirectory> subdirs_;
         std::vector<Subfile> subfiles_;
     };

Subfile.cpp

@@ -7,11 +7,18 @@
 #include "DatFile.h"
 #include "DatException.h"
 #include "Database.h"
+#include "CommonFunctions.h"
 
-#include <iostream>
+#include <algorithm>
+
+#ifdef WIN32
+#include <direct.h>
+#define mkdir(dir, mode) _mkdir(dir)
+#endif
 
 const long long MAXSIZE = 50ll * 1024ll * 1024ll; // Setting maximal file size 50 MB; Files with size more than 50 mb
                                                   // will be recognized as incorrect and passed.
+                                                  // This should be done because
 
 LOTRO_DAT::Subfile::Subfile() {
 	ext_ = UNKNOWN;
@@ -174,7 +181,99 @@ bool LOTRO_DAT::Subfile::ExportFile(const char *filename, Database *db) const {
 }
 
 bool LOTRO_DAT::Subfile::ExportFileAsTXT(const char *filename, Database *db) const {
-    throw DatException("ExportAsTXT function is not implemented yet", EXPORT_EXCEPTION);
+    if (db == nullptr) {
+        return false;
+    }
+
+    if (file_size() == 10) // File is empty, nothing to do;
+        return false;
+
+    BinaryData data = GetFileData();
+
+    long long offset = 9; // first 4 bytes - file_id, then 4 bytes - unknown, then 1 byte - unknown
+
+	long long text_fragment_num = data.ToNumber<1>(offset);
+	if ((text_fragment_num & 0x80) != 0) {
+		text_fragment_num = (((text_fragment_num ^ 0x80) << 8) | data.ToNumber<1>(offset + 1));
+		offset += 1;
+	}
+	offset += 1;
+
+    for (long long i = 0; i < text_fragment_num; i++) {
+        long long fragment_id = data.ToNumber<8>(offset);
+        offset += 8;
+
+        long long num_pieces = data.ToNumber<4>(offset);
+        offset += 4;
+
+        std::vector<std::u16string> text_pieces;
+        std::vector<long long> arg_references;
+        std::vector<std::vector<BinaryData> > arg_strings;
+
+        for (long long j = 0; j < num_pieces; j++) {
+            long long piece_size = data.ToNumber<1>(offset);
+            if ((piece_size & 128) != 0) {
+                piece_size = (((piece_size ^ 128) << 8) | data.ToNumber<1>(offset + 1));
+                offset += 1;
+            }
+            offset += 1;
+
+            BinaryData piece_data = data.CutData(offset, offset + piece_size * 2);
+            std::u16string piece;
+
+            for (long long k = 0; k < piece_size; k++) {
+                char16_t c = char16_t(((short(piece_data[2 * k + 1])) << 8) | (short(piece_data[2 * k])));
+                piece += c;
+            }
+
+            text_pieces.push_back(piece);
+            offset += piece_size * 2;
+        }
+
+        long long num_references = data.ToNumber<4>(offset);
+        offset += 4;
+
+        for (long long j = 0; j < num_references; j++) {
+            arg_references.emplace_back(data.ToNumber<4>(offset));
+            offset += 4;
+        }
+
+        long long num_arg_strings = data.ToNumber<1>(offset);
+        offset += 1;
+
+        for (long long j = 0; j < num_arg_strings; j++) {
+            fprintf(stderr, "Hey!!!\n");
+            long long num_args = data.ToNumber<4>(offset);
+            offset += 4;
+
+            arg_strings.emplace_back();
+            for (long long k = 0; k < num_args; k++) {
+                long long string_size = data.ToNumber<1>(offset);
+                if ((string_size & 0x80) != 0) {
+                    string_size = (((string_size ^ 0x80) << 8) | data.ToNumber<1>(offset + 1));
+                    offset += 1;
+                }
+                offset += 1;
+
+                arg_strings[j].emplace_back(data.CutData(offset, offset + string_size * 2));
+                offset += string_size * 2;
+            }
+        }
+
+        std::u16string text = u"[";
+        for (int i = 0; i + 1 < text_pieces.size(); i++)
+            text += text_pieces[i] + u"<--DO_NOT_TOUCH!-->";
+        text += text_pieces[text_pieces.size() - 1] + u"]";
+
+        std::string arguments;
+        for (int i = 0; i + 1 < arg_references.size(); i++)
+            arguments += std::to_string(arg_references[i]) + "-";
+        if (arg_references.size() >= 1)
+            arguments += std::to_string(arg_references[arg_references.size() - 1]);
+
+        db->PushTextFile(file_id(), fragment_id, text.c_str(), arguments.c_str());
+    }
+    return true;
 }
 
 LOTRO_DAT::BinaryData LOTRO_DAT::Subfile::GetFileData(long long offset) const {
@@ -186,8 +285,9 @@ LOTRO_DAT::BinaryData LOTRO_DAT::Subfile::GetFileData(long long offset) const {
 
     BinaryData fragments_count(4);
     dat_->ReadData(fragments_count, 4, file_offset());
+
     long long fragments_number = fragments_count.ToNumber<4>(0);
-    
+
     long long current_block_size = block_size() - offset - 8 * fragments_number;
 
     dat_->ReadData(data, current_block_size , file_offset() + offset);
@@ -323,7 +423,7 @@ const LOTRO_DAT::BinaryData LOTRO_DAT::Subfile::PrepareAsOGG() const {
 }
 
 const LOTRO_DAT::BinaryData LOTRO_DAT::Subfile::PrepareAsFont() const {
-    throw DatException("Bad Subfile::ExportAsFont() - this function hasn't been implemented yet", EXPORT_EXCEPTION);
+    return GetFileData();
 }
 
 const LOTRO_DAT::BinaryData LOTRO_DAT::Subfile::PrepareAsWAV() const {
@@ -331,5 +431,5 @@ const LOTRO_DAT::BinaryData LOTRO_DAT::Subfile::PrepareAsWAV() const {
 }
 
 const LOTRO_DAT::BinaryData LOTRO_DAT::Subfile::PrepareAsUnk() const {
-    throw DatException("Bad Subfile::ExportAsUnk() - this function hasn't been implemented yet", EXPORT_EXCEPTION);
+    return GetFileData();
 }

Subfile.h

@@ -30,7 +30,7 @@ namespace LOTRO_DAT
         bool ExportFile(const char* filename, Database *db = nullptr) const;
 		
 		bool ExportFileAsTXT(const char *filename, Database *db = nullptr) const; // Need to have special function,
-																						 // as TXT file consisits of more than 1 file
+                                                                                  // as TXT file consisits of more than 1 file
 		bool PatchFile();
 
         long long fragments_count() const;
@@ -81,5 +81,4 @@ namespace LOTRO_DAT
 }
 };
 
-
 #endif //LOTRO_DAT_PATCHER_SUBFILE_H

test.cpp

@@ -5,31 +5,35 @@
 #include <iostream>
 #include <windows.h>
 #include <ctime>
-
+#include <clocale>
+using namespace std;
 #include "LotroDatPatcher.h"
 using namespace LOTRO_DAT;
 
+#include <string>
+#include <locale>
+#include <codecvt>
 
 int main() {
     const clock_t begin_time = clock();
 
 	// Change these 2 variables to your path and name of .dat file
-    std::string path = "E:\\SteamLibrary\\steamapps\\common\\Lord of the Rings Online\\";
-    std::string filename = "client_highres.dat";
+    std::string path = "E:\\SteamLibrary\\steamapps\\common\\";//Lord Of The Rings Online\\";
+    std::string filename = "client_local_English.dat";
 
-    bool exportTextsToFiles = false; // Not implemented yet - do not turn to true;
-    bool exportImagesToFiles = true;
-    bool exportFontsToFiles = false; // Not implemented yet - do not turn to true;
+    bool exportTextsToFiles = false;
+    bool exportImagesToFiles = false;
+    bool exportFontsToFiles = true;
     bool exportSoundsToFiles = false;
-    bool exportTexturesToFiles = true;
-    bool exportUnknownToFiles = false; // Not implemented yet - do not turn to true;
+    bool exportTexturesToFiles = false;
+    bool exportUnknownToFiles = true;
 
-    bool exportTextsToDb = false; // Not implemented yet - do not turn to true;
+    bool exportTextsToDb = false;
     bool exportImagesToDb = false;
-    bool exportFontsToDb = false; // Not implemented yet - do not turn to true;
+    bool exportFontsToDb = true;
     bool exportSoundsToDb = false;
     bool exportTexturesToDb = false;
-    bool exportUnknownToDb = false; // Not implemented yet - do not turn to true;
+    bool exportUnknownToDb = false;
 
 	// There is no need to touch something else