LotroDat/Subfile.cpp

//
// Created by Иван_Архипов on 01.11.2017.
//

#include "Subfile.h"
#include "BinaryData.h"
#include "DatFile.h"
#include "DatException.h"
#include "Database.h"
#include "CommonFunctions.h"

#include <algorithm>

const long long MAXSIZE = 50ll * 1024ll * 1024ll; // Setting maximal file size 50 MB; Files with size more than 50 mb
                                                  // will be recognized as incorrect and passed.
                                                  // This should be done because

LOTRO_DAT::Subfile::Subfile() {
	ext_ = UNKNOWN;
    compressed_ = false;
}

LOTRO_DAT::Subfile::Subfile(DatFile *dat, long long fragments_count, long long unknown1, long long file_id, long long file_offset,
                            long long file_size, long long timestamp, long long version, long long block_size) :
        dat_(dat), fragments_count_(fragments_count), unknown1_(unknown1), file_id_(file_id), file_offset_(file_offset),
        file_size_(file_size), timestamp_(timestamp), version_(version), block_size_(block_size) {

    if (file_size_ > MAXSIZE)
		throw DatException("Bad Subfile::Subfile() - File size is too much... Maybe it's incorrect..?", SUBFILE_EXCEPTION);
	ext_ = GetExtension();

    BinaryData header(20);
    dat_->ReadData(header, 10, file_offset_ + 8);
    compressed_ = header.CheckCompression();
}

long long LOTRO_DAT::Subfile::fragments_count() const {
    return fragments_count_;
}

long long LOTRO_DAT::Subfile::unknown1() const {
    return unknown1_;
}

long long LOTRO_DAT::Subfile::file_id() const {
    return file_id_;
}

long long LOTRO_DAT::Subfile::file_offset() const {
    return file_offset_;
}

long long LOTRO_DAT::Subfile::file_size() const {
    return file_size_;
}

long long LOTRO_DAT::Subfile::timestamp() const {
    return timestamp_;
}

long long LOTRO_DAT::Subfile::version() const {
    return version_;
}

long long LOTRO_DAT::Subfile::block_size() const {
    return block_size_;
}

LOTRO_DAT::EXTENSION LOTRO_DAT::Subfile::GetExtension() const {
    // Text check based on file_id
    if ((file_id_ >> 24ll) == 0x25ll)
        return TXT;

    // Font check based on file_id
    if ((file_id_ >> 24ll) == 0x42ll)
        return FONT;

    BinaryData header(64);
    try {
        dat_->ReadData(header, 64, (unsigned) file_offset_ + 8);
    } catch (DatException &e) {
        if (e.type() == READ_EXCEPTION) {
            std::string err =
                    "Bad Subfile::getExtension() - unable to read header of file with id = " + std::to_string(file_id()) +
                    " and offset = " + std::to_string(file_offset());
            throw DatException(err.c_str(), SUBFILE_EXCEPTION);
        } else
            throw e;
    }

    // jpeg / dds check
    if ((file_id_ >> 24ll) == 0x41ll) {
        long long soi = header.ToNumber<2>(24);
        long long marker = header.ToNumber<2>(26);

        //auto markerSize = header.ToNumber<short>(28);
        //auto four = header.ToNumber<int>(30);

		if (soi == 0xD8FFll && marker == 0xE0FFll || marker == 0xE1FFll)
            return JPG;
        return DDS;
    }

    // Ogg and Wav check
    if (header[8] == 0x4F && header[9] == 0x67 && header[10] == 0x67 && header[11] == 0x53)
        return OGG;
    if (header[8] == 0x52 && header[9] == 0x49 && header[10] == 0x46 && header[11] == 0x46)
        return WAV;

    return UNKNOWN;
}

LOTRO_DAT::EXTENSION LOTRO_DAT::Subfile::ext() const {
	return ext_;
}

std::string LOTRO_DAT::Subfile::ExtensionToString(LOTRO_DAT::EXTENSION ext) const {
	switch (ext)
	{
	case LOTRO_DAT::TXT:
		return ".txt";
	case LOTRO_DAT::JPG:
		return ".jpg";
	case LOTRO_DAT::DDS:
		return ".dds";
	case LOTRO_DAT::WAV:
		return ".wav";
	case LOTRO_DAT::OGG:
		return ".ogg";
	case LOTRO_DAT::FONT:
		return ".fontbin";
	case LOTRO_DAT::UNKNOWN:
		return ".unk";
	default:
		return "";
	}
}

bool LOTRO_DAT::Subfile::PatchBinaryFile(const LOTRO_DAT::BinaryData &file) {
    LOTRO_DAT::BinaryData data;

    // Making data for placing in .dat, depending on file extension
    switch (ext_)
    {
        case LOTRO_DAT::TXT:
            throw LOTRO_DAT::DatException("Bad Subfile::PatchBinaryFile() - trying to patch text file");
        case LOTRO_DAT::JPG:
            data = MakeFromJPG(file); break;
        case LOTRO_DAT::DDS:
            data = MakeFromDDS(file); break;
        case LOTRO_DAT::WAV:
            data = MakeFromWAV(file); break;
        case LOTRO_DAT::OGG:
            data = MakeFromOGG(file); break;
        case LOTRO_DAT::FONT:
            data = MakeFromFont(file); break;
        case LOTRO_DAT::UNKNOWN:
            data = MakeFromUnk(file); break;
        default:
            break;
    }

    if (block_size() >= data.size() + 8) {
        dat_->WriteData(data, data.size(), file_offset() + 8);
        return true;
    }
    throw DatException("Bad Subfile::PatchBinaryFile() - new data size is bigger, than block_size. This is not implemented yet"
    , IMPORT_EXCEPTION);
    return true;
}

bool LOTRO_DAT::Subfile::ExportFile(const char *filename, Database *db) const {
	try {
        BinaryData data;

		switch (ext_)
		{
		case LOTRO_DAT::TXT:
			return ExportFileAsTXT(filename, db);
		case LOTRO_DAT::JPG:
			data = PrepareAsJPG(); break;
		case LOTRO_DAT::DDS:
			data = PrepareAsDDS(); break;
		case LOTRO_DAT::WAV:
			data = PrepareAsWAV(); break;
		case LOTRO_DAT::OGG:
			data = PrepareAsOGG(); break;
		case LOTRO_DAT::FONT:
			data = PrepareAsFont(); break;
		case LOTRO_DAT::UNKNOWN:
			data = PrepareAsUnk(); break;
		default:
			break;
		}

		if (db == nullptr) {
            std::string s = std::string(filename) + std::string(ExtensionToString(ext_));
			return data.WriteToFile(s);
		} else {
            db->PushBinaryFile(file_id_, data);
            return true;
		}
	}
	catch (DatException &e) {
		fprintf(stderr, "Caught exception while unpacking the file with id %lld and offset %lld. Continuing without this file...\n", file_id(), file_offset());
		fprintf(stderr, "%s\n", e.what());
        return false;
	}
}

bool LOTRO_DAT::Subfile::ExportFileAsTXT(const char *filename, Database *db) const {
    if (db == nullptr) {
        return false;
    }

    if (file_size() == 10) // File is empty, nothing to do;
        return false;

    BinaryData data = GetFileData();

    long long offset = 9; // first 4 bytes - file_id, then 4 bytes - unknown, then 1 byte - unknown

	long long text_fragment_num = data.ToNumber<1>(offset);
	if ((text_fragment_num & 0x80) != 0) {
		text_fragment_num = (((text_fragment_num ^ 0x80) << 8) | data.ToNumber<1>(offset + 1));
		offset += 1;
	}
	offset += 1;

    for (long long i = 0; i < text_fragment_num; i++) {
        long long fragment_id = data.ToNumber<8>(offset);
        offset += 8;

        long long num_pieces = data.ToNumber<4>(offset);
        offset += 4;

        std::vector<std::u16string> text_pieces;
        std::vector<long long> arg_references;
        std::vector<std::vector<BinaryData> > arg_strings;

        for (long long j = 0; j < num_pieces; j++) {
            long long piece_size = data.ToNumber<1>(offset);
            if ((piece_size & 128) != 0) {
                piece_size = (((piece_size ^ 128) << 8) | data.ToNumber<1>(offset + 1));
                offset += 1;
            }
            offset += 1;

            BinaryData piece_data = data.CutData(offset, offset + piece_size * 2);
            std::u16string piece;

            for (long long k = 0; k < piece_size; k++) {
                char16_t c = char16_t(((short(piece_data[2 * k + 1])) << 8) | (short(piece_data[2 * k])));
                piece += c;
            }

            text_pieces.push_back(piece);
            offset += piece_size * 2;
        }

        long long num_references = data.ToNumber<4>(offset);
        offset += 4;

        for (long long j = 0; j < num_references; j++) {
            arg_references.emplace_back(data.ToNumber<4>(offset));
            offset += 4;
        }

        long long num_arg_strings = data.ToNumber<1>(offset);
        offset += 1;

        for (long long j = 0; j < num_arg_strings; j++) {
            long long num_args = data.ToNumber<4>(offset);
            offset += 4;

            arg_strings.emplace_back();
            for (long long k = 0; k < num_args; k++) {
                long long string_size = data.ToNumber<1>(offset);
                if ((string_size & 0x80) != 0) {
                    string_size = (((string_size ^ 0x80) << 8) | data.ToNumber<1>(offset + 1));
                    offset += 1;
                }
                offset += 1;

                arg_strings[j].emplace_back(data.CutData(offset, offset + string_size * 2));
                offset += string_size * 2;
            }
        }

        std::u16string text = u"[";
        for (int i = 0; i + 1 < text_pieces.size(); i++)
            text += text_pieces[i] + u"<--DO_NOT_TOUCH!-->";
        text += text_pieces[text_pieces.size() - 1] + u"]";

        std::string arguments;
        for (int i = 0; i + 1 < arg_references.size(); i++)
            arguments += std::to_string(arg_references[i]) + "-";
        if (arg_references.size() >= 1)
            arguments += std::to_string(arg_references[arg_references.size() - 1]);

        db->PushTextFile(file_id(), fragment_id, text.c_str(), arguments.c_str());
    }
    return true;
}

LOTRO_DAT::BinaryData LOTRO_DAT::Subfile::GetFileData(long long offset) const {
    BinaryData mfile_id(4);
    dat_->ReadData(mfile_id, 4, file_offset() + 8);
    if (file_id() != mfile_id.ToNumber<4>(0))
        throw DatException("Bad Subfile::GetFileData() - file_id doesn't match to dictionary", READ_EXCEPTION);

    BinaryData data((unsigned)(file_size()));
    if (block_size() >= file_size() + 8) {
        dat_->ReadData(data, file_size(), file_offset() + offset);
        return data;
    }

    BinaryData fragments_count(4);
    dat_->ReadData(fragments_count, 4, file_offset());

    long long fragments_number = fragments_count.ToNumber<4>(0);

    long long current_block_size = block_size() - offset - 8 * fragments_number;

    dat_->ReadData(data, current_block_size , file_offset() + offset);

    BinaryData FragmentsDictionary(8 * unsigned(fragments_number));
    dat_->ReadData(FragmentsDictionary, 8 * unsigned(fragments_number), file_offset() + block_size() - 8 * fragments_number);


    for (long long i = 0; i < fragments_number; i++) {
        long long fragment_size = FragmentsDictionary.ToNumber<4>(8 * i);
        long long fragment_offset = FragmentsDictionary.ToNumber<4>(8 * i + 4);
        dat_->ReadData(data, std::min(fragment_size, file_size() - current_block_size), fragment_offset, current_block_size );
        current_block_size += fragment_size;
    }

    return data;
}

const LOTRO_DAT::BinaryData LOTRO_DAT::Subfile::PrepareAsJPG() const {
    return GetFileData().CutData(24);
}

const LOTRO_DAT::BinaryData LOTRO_DAT::Subfile::PrepareAsDDS() const {
    BinaryData data = GetFileData();
    if (compressed_)
        data.DecompressData(4);

    BinaryData ddsData(data.size() - 24 + 128);
    for (int i = 0; i < 128; i++)
        ddsData[i] = 0;

    memcpy(ddsData.data() + 128, data.data() + 24, data.size() - 24);

    ddsData[0] = 0x44; // D
    ddsData[1] = 0x44; // D
    ddsData[2] = 0x53; // S
    ddsData[3] = 0x20;
    ddsData[4] = 0x7C;

    ddsData[8] = 7;
    ddsData[9] = 0x10;

    // width, height
    ddsData[12] = data[12];
    ddsData[13] = data[13];
    ddsData[14] = data[14];
    ddsData[15] = data[15];

    ddsData[16] = data[8];
    ddsData[17] = data[9];
    ddsData[18] = data[10];
    ddsData[19] = data[11];

    long long compression = data.ToNumber<4>(0x10);

    switch (compression) {
        case 20:        // 14 00 00 00 - 888 (R8G8B8)
            ddsData[0x4C] = 0x20;  // ?
            ddsData[0x50] = 0x40;  // compressed or not

            ddsData[0x58] = 0x18;  // bytes per pixel
            ddsData[0x5E] = 0xFF;
            ddsData[0x61] = 0xFF;
            ddsData[0x64] = 0xFF;
            break;
        case 21:        // 15 00 00 00 - 8888 (R8G8B8A8)
            ddsData[0x4C] = 0x20;  // ?
            ddsData[0x50] = 0x40;  // compressed or not

            ddsData[0x58] = 0x20;  // bytes per pixel
            ddsData[0x5E] = 0xFF;
            ddsData[0x61] = 0xFF;
            ddsData[0x64] = 0xFF;
            ddsData[0x6B] = 0xFF;
            break;
        case 28:        // 1C 00 00 00 - 332 (?)
            ddsData[0x4C] = 0x20;  // ?
            ddsData[0x50] = 0x40;  // compressed or not

            ddsData[0x58] = 0x08;  // bytes per pixel
            ddsData[0x5E] = 0xFF;
            ddsData[0x61] = 0xFF;
            ddsData[0x64] = 0xFF;
            break;
        case 827611204: // 44 58 54 31 - DXT1
            ddsData[76] = 32;
            ddsData[80] = 4;

            ddsData[84] = 68;
            ddsData[85] = 88;
            ddsData[86] = 84;
            ddsData[87] = 49;
            break;
        case 861165636: // 44 58 54 33 - DXT3
            ddsData[22] = 1;
            ddsData[76] = 32;
            ddsData[80] = 4;

            ddsData[84] = 68;
            ddsData[85] = 88;
            ddsData[86] = 84;
            ddsData[87] = 51;

            ddsData[108] = 8;
            ddsData[109] = 16;
            ddsData[110] = 64;
            break;
        case 894720068: // 44 58 54 35 - DXT5
            ddsData[10] = 8;
            ddsData[22] = 1;
            ddsData[28] = 1;
            ddsData[76] = 32;
            ddsData[80] = 4;

            ddsData[84] = 68;
            ddsData[85] = 88;
            ddsData[86] = 84;
            ddsData[87] = 53;

            ddsData[88] = 32;
            ddsData[94] = 255;
            ddsData[97] = 255;
            ddsData[100] = 255;
            ddsData[107] = 255;
            ddsData[109] = 16;
            break;
    }
	return ddsData;
}

const LOTRO_DAT::BinaryData LOTRO_DAT::Subfile::PrepareAsOGG() const {
    return GetFileData().CutData(8);
}

const LOTRO_DAT::BinaryData LOTRO_DAT::Subfile::PrepareAsFont() const {
    return GetFileData();
}

const LOTRO_DAT::BinaryData LOTRO_DAT::Subfile::PrepareAsWAV() const {
    return GetFileData().CutData(8);
}

const LOTRO_DAT::BinaryData LOTRO_DAT::Subfile::PrepareAsUnk() const {
    return GetFileData();
}

const LOTRO_DAT::BinaryData LOTRO_DAT::Subfile::MakeFromJPG(const LOTRO_DAT::BinaryData &file) const {
    return GetFileData().CutData(0, 24) + file;
}

const LOTRO_DAT::BinaryData LOTRO_DAT::Subfile::MakeFromDDS(const LOTRO_DAT::BinaryData &file) const {
    throw LOTRO_DAT::DatException("Bad Subfile::MakeFromDDS() - not implemented yet", IMPORT_EXCEPTION);
}

const LOTRO_DAT::BinaryData LOTRO_DAT::Subfile::MakeFromOGG(const LOTRO_DAT::BinaryData &file) const {
    throw LOTRO_DAT::DatException("Bad Subfile::MakeFromOGG() - not implemented yet", IMPORT_EXCEPTION);
}

const LOTRO_DAT::BinaryData LOTRO_DAT::Subfile::MakeFromFont(const LOTRO_DAT::BinaryData &file) const {
    throw LOTRO_DAT::DatException("Bad Subfile::MakeFromFont() - not implemented yet", IMPORT_EXCEPTION);
}

const LOTRO_DAT::BinaryData LOTRO_DAT::Subfile::MakeFromWAV(const LOTRO_DAT::BinaryData &file) const {
    throw LOTRO_DAT::DatException("Bad Subfile::MakeFromWAV() - not implemented yet", IMPORT_EXCEPTION);
}

const LOTRO_DAT::BinaryData LOTRO_DAT::Subfile::MakeFromUnk(const LOTRO_DAT::BinaryData &file) const {
    throw LOTRO_DAT::DatException("Bad Subfile::MakeFromUnk() - not implemented yet", IMPORT_EXCEPTION);
}