Added zlib library and support of extracting .dds compressed textures

BinaryData.cpp

@@ -4,6 +4,8 @@
 
 #include "BinaryData.h"
 #include "DatException.h"
+#include <cassert>
+#include "zlib/zlib.h"
 
 extern  "C++"
 {
@@ -15,7 +17,7 @@ namespace LOTRO_DAT {
     }
 
     BinaryData::BinaryData(const BinaryData &d) {
-        size_ = d.size_;
+		size_ = d.size_;
         data_ = new unsigned char[size_];
         memcpy(data_, d.data_, size_);
     }
@@ -36,17 +38,17 @@ namespace LOTRO_DAT {
         return data_[pos];
     }
 
-    // Translates data into specified number type due to UTF-16LE encoding of the .dat file
+    // Translates T bytes from data into number using UTF-16LE encoding of the .dat file
     template<int T>
     long long BinaryData::ToNumber(const long long &pos) const {
         try {
             long long ans = 0;
 
-            if (pos + 3 >= size_)
+            if (pos + T >= size_)
                 throw DatException("Bad BinaryData::ToNumber(). Reached end of BinaryData!", DATA_EXCEPTION);
 
             for (int i = T - 1; i >= 0; i--)
-                ans = ((ans << 8ll) | (unsigned char)(data_[pos + i]));
+                ans = ((ans << 8ll) | data_[pos + i]);
 
             return ans;
         } catch (...) {
@@ -54,7 +56,25 @@ namespace LOTRO_DAT {
         }
     }
 
-    // Makes data from specified number type
+    // Translates T bytes from data into number in raw format
+    template<int T>
+    long long BinaryData::ToNumberRAW(const long long &pos) const {
+        try {
+            long long ans = 0;
+
+            if (pos + T >= size_)
+                throw DatException("Bad BinaryData::ToNumber(). Reached end of BinaryData!", DATA_EXCEPTION);
+
+            for (int i = 0; i < T; i++)
+                ans = ((ans << 8ll) | data_[pos + i]);
+
+            return ans;
+        } catch (...) {
+            throw DatException("Bad BinaryData::ToNumber(). Error while using template function", DATA_EXCEPTION);
+        }
+    }
+
+    // Makes data from specified T bytes of number in Little Endian encoding
     template<int T>
     void BinaryData::FromNumber(const long long &number) {
         if (T < 0)
@@ -76,6 +96,28 @@ namespace LOTRO_DAT {
         }
     }
 
+    // Makes data from specified T bytes of number in raw
+    template<int T>
+    void BinaryData::FromNumberRAW(const long long &number) {
+        if (T < 0)
+            throw DatException("Bad BinaryData::FromNumber() - trying to make data from amount of bytes < 0");
+        try {
+            if (size_ != 0 && data_ != nullptr)
+                delete[] data_;
+
+            size_ = size_t(T);
+            data_ = new unsigned char[size_];
+
+            for (size_t i = size_ - 1; i >= 0; i--)
+            {
+                data_[i] = (unsigned char)((number >> (8 * i)) & 0xFF);
+            }
+
+        } catch (...) {
+            throw DatException("Bad BinaryData::ToNumber(). Error in using template function", DATA_EXCEPTION);
+        }
+    }
+
     BinaryData &BinaryData::operator=(const BinaryData &data) {
         if (&data == this)
             return *this;
@@ -90,6 +132,7 @@ namespace LOTRO_DAT {
             data_ = new unsigned char[size_];
             memcpy(data_, data.data_, size_);
         }
+
         return *this;
     }
 
@@ -127,6 +170,30 @@ namespace LOTRO_DAT {
 		return true;
 	}
 
+    bool BinaryData::CheckCompression() const {
+        if (size() < 10)
+            return false;
+        auto header = ToNumberRAW<2>(4);
+        return (header == 0x7801 || header == 0x789C || header == 0x78DA);
+    }
+
+    void BinaryData::DecompressData(unsigned int offset) {
+        const unsigned max_unpacked_size = 1024 * 1024 * 40; // Setting 40 MB as a maximum unpacked data;
+
+        BinaryData decompressed(max_unpacked_size);
+
+        uLongf new_size = max_unpacked_size;
+        int res = uncompress(decompressed.data_, &new_size, data_ + offset, size_ - offset);
+		
+		if (res != 0) {
+			throw DatException("Bad BinaryData::DecompressData() - uncompress() failed!", DATA_EXCEPTION);
+		}
+
+		decompressed.size_ = (unsigned int)new_size;
+		*this = decompressed;
+		return;
+	}
+
     template long long BinaryData::ToNumber<1>(long long const&) const;
     template long long BinaryData::ToNumber<2>(long long const&) const;
     template long long BinaryData::ToNumber<4>(long long const&) const;
@@ -136,5 +203,16 @@ namespace LOTRO_DAT {
     template void BinaryData::FromNumber<2>(const long long &);
     template void BinaryData::FromNumber<4>(const long long &);
     template void BinaryData::FromNumber<8>(const long long &);
+
+    template long long BinaryData::ToNumberRAW<1>(long long const&) const;
+    template long long BinaryData::ToNumberRAW<2>(long long const&) const;
+    template long long BinaryData::ToNumberRAW<4>(long long const&) const;
+    template long long BinaryData::ToNumberRAW<8>(long long const&) const;
+
+    template void BinaryData::FromNumberRAW<1>(const long long &);
+    template void BinaryData::FromNumberRAW<2>(const long long &);
+    template void BinaryData::FromNumberRAW<4>(const long long &);
+    template void BinaryData::FromNumberRAW<8>(const long long &);
+
 }
 }

BinaryData.h

@@ -25,9 +25,15 @@ namespace LOTRO_DAT
         template <int T>
         long long ToNumber(const long long &pos) const;
 
+        template <int T>
+        long long ToNumberRAW(const long long &pos) const;
+
         template <int T>
         void FromNumber(const long long &number);
 
+        template <int T>
+        void FromNumberRAW(const long long &number);
+
         size_t size() const;
         unsigned char* data() const;
 
@@ -36,6 +42,9 @@ namespace LOTRO_DAT
 		bool WriteToFile(const char *filename) const;
 		bool WriteToFile(const std::string &filename) const;
 
+        bool CheckCompression() const;
+		void DecompressData(unsigned int offset = 0);
+
     private:
         unsigned char *data_;
         size_t size_;

CMakeLists.txt

@@ -7,4 +7,8 @@ set(SOURCE_FILES DatFile.cpp DatFile.h BinaryData.cpp BinaryData.h DatException.
 #set(SOURCE_FILES DatFile.cpp DatFile.h BinaryData.cpp BinaryData.h DatException.h LotroDatPatcher.h)
 
 #add_library(LotRO_dat_Patcher SHARED ${SOURCE_FILES})
+#target_link_libraries(LotRO_dat_Patcher ${CMAKE_SOURCE_DIR}/zlib/libzlibstatic.a)
+
 add_executable(LotRO_dat_tester ${SOURCE_FILES} test.cpp)
+target_link_libraries(LotRO_dat_tester ${CMAKE_SOURCE_DIR}/zlib/libzlibstatic.a)
+

DatFile.cpp

@@ -139,6 +139,7 @@ namespace LOTRO_DAT
 
         _fseeki64(file_handler_, offset, SEEK_SET);
         fread(data.data() + data_offset, size, 1, file_handler_);
+        data.CheckCompression();
     }
 
     void DatFile::WriteData(const BinaryData &data, long long size, long long offset, long long data_offset)

SubDirectory.cpp

@@ -55,9 +55,8 @@ void LOTRO_DAT::SubDirectory::MakeSubDirectories() {
                             dat_
                     )
             );
-        } catch (DatException &e) {
+        } catch (...) {
             fprintf(stderr, "Making SubDirectory at offset %lld failed, continuing\n",  data.ToNumber<4>(i + 4));
-            // TODO - do something here....
         }
     }
 }

Subfile.cpp

@@ -12,6 +12,7 @@ const long long MAXSIZE = 10000000ll;
 
 LOTRO_DAT::Subfile::Subfile() {
 	ext_ = UNKNOWN;
+    compressed_ = false;
 }
 
 LOTRO_DAT::Subfile::Subfile(DatFile *dat, long long unknown0, long long unknown1, long long file_id, long long file_offset, long long size1,
@@ -21,6 +22,14 @@ LOTRO_DAT::Subfile::Subfile(DatFile *dat, long long unknown0, long long unknown1
 	if (size1_ > MAXSIZE)
 		throw DatException("Bad Subfile::Subfile() - File size is too much... Maybe it's incorrect..?", SUBFILE_EXCEPTION);
 	ext_ = GetExtension();
+
+    BinaryData header(20);
+    dat_->ReadData(header, 10, file_offset_);
+    compressed_ = header.CheckCompression();
+
+    //if (compressed_) {
+     //   fprintf(stderr, "Found compressed file with id = %lld and offset = %lld!\n", file_id_, file_offset_);
+    //}
 }
 
 long long LOTRO_DAT::Subfile::unknown0() const {
@@ -119,9 +128,10 @@ bool LOTRO_DAT::Subfile::ExportFile(const char *filename) const {
 		if (ext() == UNKNOWN)
 			return ExportAsUnk(filename);
 	}
-	catch (...) {
+	catch (DatException &e) {
 		fprintf(stderr, "Caught exception while unpacking the file with id %lld and offset %lld. Continuing...\n", file_id(), file_offset());
-		return false;
+		fprintf(stderr, "%s\n", e.what());
+        return false;
 	}
     throw DatException("Bad Subfile::ExportFile() - got incorrect extension", EXPORT_EXCEPTION);
 }
@@ -138,13 +148,18 @@ bool LOTRO_DAT::Subfile::ExportAsJPG(const char *filename) const {
 
 bool LOTRO_DAT::Subfile::ExportAsDDS(const char *filename) const {
     BinaryData data((unsigned) size1_);
-    dat_->ReadData(data, (unsigned) size1_, (unsigned) file_offset_);
+    dat_->ReadData(data, (unsigned) size1_, file_offset_);
+    if (compressed_)
+        data.DecompressData(4);
+    if (data.size() == 0) {
+        return false;
+    }
 
-    BinaryData ddsData((unsigned) size1_ - 24 + 128);
+    BinaryData ddsData(data.size() - 24 + 128);
     for (int i = 0; i < 128; i++)
         ddsData[i] = 0;
 
-    memcpy(ddsData.data() + 128, data.data() + 24, (size_t) size1_ - 24);
+    memcpy(ddsData.data() + 128, data.data() + 24, data.size() - 24);
 
     ddsData[0] = 0x44; // D
     ddsData[1] = 0x44; // D

Subfile.h

@@ -56,6 +56,8 @@ namespace LOTRO_DAT
         long long version_;
         long long size2_;
 
+        bool compressed_;
+
         DatFile *dat_;
 		EXTENSION ext_;
 

test.cpp

@@ -4,17 +4,21 @@
 #define _CRT_SECURE_NO_WARNINGS
 #include <iostream>
 #include <windows.h>
+#include <ctime>
 
 #include "LotroDatPatcher.h"
 using namespace LOTRO_DAT;
 
 
 int main() {
-    std::ios_base::sync_with_stdio(0);
-    std::cin.tie(0);
+    const clock_t begin_time = clock();
+
+	// Change these 2 variables to your path and name of .dat file
+    std::string path = "E:\\SteamLibrary\\steamapps\\common\\Lord of the Rings Online\\";
+    std::string filename = "client_local_English.dat";
+
+	// There is no need to touch something else
 
-	std::string path = "E:\\SteamLibrary\\steamapps\\common\\Lord of the Rings Online\\";
-	std::string filename = "client_local_English.dat";
 	std::string output_dir = std::string("Data (") + filename + std::string(")\\");
 
 	std::string txt_dir = "txt\\";
@@ -133,7 +137,9 @@ int main() {
 		<< "Fonts " << font_suc << std::endl
 		<< "Unknown: " << unk_suc << std::endl;
 
-	std::cerr << "Success. Press enter to close";
+	fprintf(stderr, "Spent %f seconds on running unpacker! Thank you for your attention!\n", float(clock() - begin_time) / CLOCKS_PER_SEC);
+	std::cerr << "Success. Press enter to close\n";
+
 	system("pause");
 	return 0;
 }