LotroDat/src/Subfiles/TextSubfile.cpp

//
// Created by Иван_Архипов on 24.11.2017.
//

#include "Subfiles/TextSubfile.h"

#include "BinaryData.h"
#include "DatFile.h"
#include "DatException.h"
#include "SubfileData.h"
#include "EasyLogging++/easylogging++.h"

#include <codecvt>
#include <locale>

std::u16string to_utf16(long long x) {
    std::u16string res;
    while (x > 0) {
        res += char16_t(u'0' + x % 10);
        x /= 10ll;
    }
    std::reverse(res.begin(), res.end());
    return res;
}

long long from_utf16(const std::u16string &num) {
    long long res = 0;
    for (auto c : num) {
        res = res * 10ll + (c - u'0');
    }
    return res;
}

std::string argumentsFromUtf16(const std::u16string &args) {
    std::string res;

    size_t pointer = 0;
    while (pointer < args.length()) {
        size_t pointer1 = args.find(u'-', pointer);
        if (pointer1 == std::u16string::npos)
            pointer1 = args.length();
        if (!res.empty())
            res += "-";
        res += std::to_string(from_utf16(args.substr(pointer, pointer1 - pointer)));
        pointer = pointer1 + 1;
    }

    return res;
}

namespace LOTRO_DAT {
    TextSubfile::TextSubfile() = default;

    TextSubfile::TextSubfile(DatFile *dat, long long dictionary_offset, long long fragments_count, long long unknown1,
                           long long file_id, long long file_offset, long long file_size,
                           long long timestamp,
                           long long version, long long block_size)
            : Subfile(dat, dictionary_offset, fragments_count, unknown1, file_id, file_offset, file_size, timestamp, version, block_size) {
    }

    FILE_TYPE TextSubfile::FileType() const {
        return TEXT;
    }

    std::string TextSubfile::Extension() const {
        return std::string(".txt");
    }

    SubfileData TextSubfile::PrepareForExport(const BinaryData &file_data) {
        SubfileData result;
        long long offset = 9; // first 4 bytes - file_id, then 4 bytes - unknown, then 1 byte - unknown

        long long text_fragment_num = file_data.ToNumber<1>(offset);
        if ((text_fragment_num & 0x80) != 0) {
            text_fragment_num = (((text_fragment_num ^ 0x80) << 8) | file_data.ToNumber<1>(offset + 1));
            offset += 1;
        }
        offset += 1;

        for (long long i = 0; i < text_fragment_num; i++) {
            long long fragment_id = file_data.ToNumber<8>(offset);
            offset += 8;

            std::vector<std::u16string> text_pieces = MakePieces(file_data, offset);
            std::vector<long long> arg_references = MakeArgumentReferences(file_data, offset);
            std::vector<std::vector<BinaryData>> arg_strings = MakeArgumentStrings(file_data, offset);

            std::u16string text = u"[";
            for (size_t j = 0; j + 1 < text_pieces.size(); j++)
                text += text_pieces[j] + u"<--DO_NOT_TOUCH!-->";
            text += text_pieces[text_pieces.size() - 1] + u"]";

            std::u16string arguments;
            for (size_t j = 0; j + 1 < arg_references.size(); j++)
                arguments += to_utf16(arg_references[j]) + u"-";
            if (!arg_references.empty())
                arguments += to_utf16(arg_references[arg_references.size() - 1]);

            if (result.text_data.length() > 0)
                result.text_data += u"|||";

            result.text_data += to_utf16(fragment_id) + u":::";
            result.text_data += arguments + u":::";
            result.text_data += text;
        }
        result.options["fid"] = file_id();
        result.options["ext"] = Extension();
        return result;
    }

    BinaryData TextSubfile::MakeForImport(const BinaryData &old_data, const SubfileData &data) {
        LOG(DEBUG) << "Preparing text file " << file_id() << " for import.";
        std::unordered_map<long long, SubfileData> patch_fragments = ParsePatchFragments(data);

        BinaryData new_data;

        if (file_size() <= 10 + 8) // File is empty, nothing to do;
            return old_data;

        long long offset = 9 + 8; // first 8 bytes - file_info. After them:
        // first 4 bytes - file_id, then 4 bytes - unknown, then 1 byte - unknown

        long long text_fragment_num = old_data.ToNumber<1>(offset);
        if ((text_fragment_num & 0x80) != 0) {
            text_fragment_num = (((text_fragment_num ^ 0x80) << 8) | old_data.ToNumber<1>(offset + 1));
            offset += 1;
        }
        offset += 1;

        new_data = new_data + old_data.CutData(0, offset); // Adding file info

        for (long long i = 0; i < text_fragment_num; i++) {
            long long fragment_id = old_data.ToNumber<8>(offset);
            offset += 8;

            new_data = new_data + old_data.CutData(offset - 8, offset);

            if (patch_fragments.count(fragment_id) == 0) {
                try {
                    // Retrieving old pieces
                    new_data = new_data + GetPieceData(old_data, offset);
                } catch (std::exception &e) {
                    LOG(ERROR) << "Caught " << e.what() << " exception.";
                    LOG(DEBUG) << "Unable to get piece data for file_id " << file_id() << " and fragment_id " << fragment_id;
                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
                }

                try {
                    // Retrieving old references
                    new_data = new_data + GetArgumentReferenceData(old_data, offset);
                } catch (std::exception &e) {
                    LOG(ERROR) << "Caught " << e.what() << " exception.";
                    LOG(DEBUG) << "Unable to get argument reference data for file_id " << file_id() << " and fragment_id " << fragment_id;
                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
                }

                try {
                    // Retrieving old ref_strings
                    new_data = new_data + GetArgumentStringsData(old_data, offset);
                } catch (std::exception &e) {
                    LOG(ERROR) << "Caught " << e.what() << " exception.";
                    LOG(DEBUG) << "Unable to get argument string for file_id " << file_id() << " and fragment_id " << fragment_id;
                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
                }

            } else {
                try {
                    // Making and adding new pieces
                    new_data = new_data + BuildPieces(old_data, patch_fragments[fragment_id], offset);
                } catch (std::exception &e) {
                    LOG(ERROR) << "Caught " << e.what() << " exception.";
                    LOG(DEBUG) << "Unable to build piece data for file_id " << file_id() << " and fragment_id " << fragment_id;
                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
                }

                try {
                    // Making and adding new references
                    new_data = new_data + BuildArgumentReferences(old_data, patch_fragments[fragment_id], offset);
                } catch (std::exception &e) {
                    LOG(ERROR) << "Caught " << e.what() << " exception.";
                    LOG(DEBUG) << "Unable to build argument data for file_id " << file_id() << " and fragment_id " << fragment_id;
                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
                }

                try {
                    // Making and adding new strings
                    new_data = new_data + BuildArgumentStrings(old_data, patch_fragments[fragment_id], offset);
                } catch (std::exception &e) {
                    LOG(ERROR) << "Caught " << e.what() << " exception.";
                    LOG(DEBUG) << "Unable to build argument string data for file_id " << file_id() << " and fragment_id " << fragment_id;
                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
                }
            }
        }
        new_data = new_data + old_data.CutData(offset); // Adding elapsed file data
        return new_data;
    }

    std::unordered_map<long long, SubfileData> TextSubfile::ParsePatchFragments(const SubfileData &data) {
        LOG(DEBUG) << "Started parsing patch fragments";
        std::unordered_map<long long, SubfileData> res;
        std::u16string text = data.text_data;

        size_t pointer = 0;
        while (pointer < text.length()) {
            // Parsing fragment_id
            size_t pointer1 = text.find(u":::", pointer);
            if (pointer1 == std::u16string::npos)
                throw DatException("Bad TextSubfile::ParsePatchFragments() - Unable to parse fragment id! Cannot find '...' divider");
            long long fragment_id = from_utf16(text.substr(pointer, pointer1 - pointer));
            pointer = pointer1 + 3;
            res[fragment_id] = SubfileData();
            res[fragment_id].options["gid"] = fragment_id;

            // Parsing arguments
            pointer1 = text.find(u":::", pointer);
            if (pointer1 == std::u16string::npos)
                throw DatException("Bad TextSubfile::ParsePatchFragments() - Unable to parse arguments! Cannot find '...' divider");
            std::u16string arguments = text.substr(pointer, pointer1 - pointer);
            pointer = pointer1 + 3;
            if (arguments.length() > 0) {
                res[fragment_id].options["args"] = argumentsFromUtf16(arguments);
            }

            // Parsing text
            pointer1 = text.find(u"|||", pointer);
            if (pointer1 == std::u16string::npos)
                pointer1 = text.length();
            std::u16string text_data = text.substr(pointer, pointer1 - pointer);
            pointer = pointer1  + 3;
            res[fragment_id].text_data = text_data;
        }
        LOG(DEBUG) << "Finished parsing text patch fragments";
        return res;
    }

    // Make pieces/arguments/argument strings functions

    std::vector<std::u16string> TextSubfile::MakePieces(const BinaryData &data, long long &offset) {
        LOG(DEBUG) << "Started making pieces";
        long long num_pieces = data.ToNumber<4>(offset);
        offset += 4;

        std::vector<std::u16string> text_pieces;

        for (long long j = 0; j < num_pieces; j++) {
            long long piece_size = data.ToNumber<1>(offset);
            if ((piece_size & 128) != 0) {
                piece_size = (((piece_size ^ 128) << 8) | data.ToNumber<1>(offset + 1));
                offset += 1;
            }
            offset += 1;

            BinaryData piece_data = data.CutData(offset, offset + piece_size * 2);
            std::u16string piece;

            for (long long k = 0; k < piece_size; k++) {
                char16_t c = char16_t(
                        ((short(piece_data[2 * unsigned(k) + 1])) << 8) | (short(piece_data[2 * unsigned(k)])));
                piece += c;
            }

            text_pieces.push_back(piece);
            offset += piece_size * 2;
        }
        LOG(DEBUG) << "Finished making pieces";
        return text_pieces;
    }

    std::vector<long long> TextSubfile::MakeArgumentReferences(const BinaryData &data, long long &offset) {
        LOG(DEBUG) << "Started making argument references";
        std::vector<long long> arg_references;

        long long num_references = data.ToNumber<4>(offset);
        offset += 4;

        for (long long j = 0; j < num_references; j++) {
            arg_references.emplace_back(data.ToNumber<4>(offset));
            offset += 4;
        }
        LOG(DEBUG) << "Finished making argument references";
        return arg_references;
    }

    std::vector<std::vector<BinaryData>> TextSubfile::MakeArgumentStrings(const BinaryData &data, long long &offset) {
        LOG(DEBUG) << "Started making argument strings";
        std::vector<std::vector<BinaryData> > arg_strings;
        long long num_arg_strings = data.ToNumber<1>(offset);
        offset += 1;

        for (long long j = 0; j < num_arg_strings; j++) {
            long long num_args = data.ToNumber<4>(offset);
            offset += 4;

            arg_strings.emplace_back();
            for (long long k = 0; k < num_args; k++) {
                long long string_size = data.ToNumber<1>(offset);
                if ((string_size & 0x80) != 0) {
                    string_size = (((string_size ^ 0x80) << 8) | data.ToNumber<1>(offset + 1));
                    offset += 1;
                }
                offset += 1;

                arg_strings[unsigned(j)].emplace_back(data.CutData(offset, offset + string_size * 2));
                offset += string_size * 2;
            }
        }
        LOG(DEBUG) << "Finished making argument strings";
        return arg_strings;
    }

    // Build pieces/arguments/argument strings functions from fragment SubfileData

    BinaryData TextSubfile::BuildPieces(const BinaryData &data, const SubfileData &new_data, long long &offset) {
        LOG(DEBUG) << "Started building pieces";
        try {
            // Moving &offset pointer in &data
            GetPieceData(data, offset);
        } catch (std::exception &e) {
            LOG(ERROR) << "Caught " << e.what() << " exception.";
            LOG(DEBUG) << "Unable to get piece data for file_id " << file_id();
            throw DatException("Bad TextSubfile::BuildPieces()", IMPORT_EXCEPTION);
        }

        // Deleting '[' and ']' brackets
        std::u16string text_data = new_data.text_data.substr(1, new_data.text_data.size() - 2);
        std::vector<std::u16string> pieces;

        const std::u16string DNT = u"<--DO_NOT_TOUCH!-->";
        size_t prev = 0;
        size_t next = text_data.find(DNT, prev);

        while (next != std::string::npos) {
            std::u16string piece = text_data.substr(prev, next - prev);
            pieces.push_back(piece);
            prev = next + DNT.length();
            next = text_data.find(DNT, prev);
        }
        std::u16string piece = text_data.substr(prev);
        pieces.push_back(piece);

        // Building BinaryData from pieces
        BinaryData result;
        BinaryData temp_data = BinaryData::FromNumber<4>(pieces.size());
        result = result + temp_data;

        for (auto piece : pieces) {
            long long piece_size = piece.length();
            if (piece_size < 128) {
                temp_data = BinaryData::FromNumber<1>(piece_size);
            } else {
                temp_data = BinaryData::FromNumberRAW<2>((piece_size | 32768));
            }
            result = result + temp_data;

            for (long long j = 0; j < piece_size; j++) {
                temp_data = BinaryData::FromNumber<2>(short(piece[j]));
                result = result + temp_data;
            }
        }
        LOG(DEBUG) << "Pieces built successfully";
        return result;
    }

    BinaryData TextSubfile::BuildArgumentReferences(const BinaryData &data, const SubfileData &new_data,
                                                    long long &offset) {
        LOG(DEBUG) << "Started building argument refs";
        // Moving &offset pointer in &data
        GetArgumentReferenceData(data, offset);

        // If there are no args - making 4 null-bytes and return;
        if (!new_data.options["args"]) {
            BinaryData result = BinaryData::FromNumber<4>(0);
            return result;
        }

        // Parsing arguments from list in options["args"]
        std::string args_list = new_data.options["args"].as<std::string>();
        std::vector<long long> arguments;

        size_t prev = 0;
        size_t next = args_list.find('-', prev);
        while (next != std::string::npos) {
            std::string argument = args_list.substr(prev, next - prev);
            arguments.push_back(std::stoll(argument));
            prev = next + 1;
            next = args_list.find('-', prev);
        }
        std::string argument = args_list.substr(prev);
        arguments.push_back(std::stoll(argument));

        BinaryData result;
        BinaryData temp_data = BinaryData::FromNumber<4>(arguments.size());
        result = result + temp_data;
        for (auto arg_reference : arguments) {
            temp_data = BinaryData::FromNumber<4>(arg_reference);
            result = result + temp_data;
        }
        LOG(DEBUG) << "Argument refs built successfully";
        return result;
    }

    BinaryData TextSubfile::BuildArgumentStrings(const BinaryData &data, const SubfileData &new_data,
                                                 long long &offset) {
        LOG(DEBUG) << "Started building argument strings";
        LOG(DEBUG) << "Built arg strings successfully";
        return GetArgumentStringsData(data, offset);
    }

    // Get BinaryData contents of pieces/arguments/argument strings

    BinaryData TextSubfile::GetPieceData(const BinaryData &data, long long &offset) const {
        LOG(DEBUG) << "Started getting piece data";
        long long old_offset = offset;

        long long num_pieces = data.ToNumber<4>(offset);
        offset += 4;

        for (long long j = 0; j < num_pieces; j++) {
            long long piece_size = data.ToNumber<1>(offset);
            if ((piece_size & 128) != 0) {
                piece_size = (((piece_size ^ 128) << 8) | data.ToNumber<1>(offset + 1));
                offset += 1;
            }
            offset += 1;
            offset += piece_size * 2;
        }
        LOG(DEBUG) << "Got piece data";
        return data.CutData(old_offset, offset);
    }

    BinaryData TextSubfile::GetArgumentReferenceData(const BinaryData &data, long long &offset) const {
        LOG(DEBUG) << "Started getting arg refs data";
        long long old_offset = offset;
        long long num_references = data.ToNumber<4>(offset);
        offset += 4;
        offset += 4 * num_references;
        LOG(DEBUG) << "Finished getting arg refs data";
        return data.CutData(old_offset, offset);
    }

    BinaryData TextSubfile::GetArgumentStringsData(const BinaryData &data, long long &offset) const {
        LOG(DEBUG) << "Started getting arg strings data";
        long long old_offset = offset;

        long long num_arg_strings = data.ToNumber<1>(offset);
        offset += 1;

        for (long long j = 0; j < num_arg_strings; j++) {
            long long num_args = data.ToNumber<4>(offset);
            offset += 4;

            for (long long k = 0; k < num_args; k++) {
                long long string_size = data.ToNumber<1>(offset);
                if ((string_size & 0x80) != 0) {
                    string_size = (((string_size ^ 0x80) << 8) | data.ToNumber<1>(offset + 1));
                    offset += 1;
                }
                offset += 1;
                offset += string_size * 2;
            }
        }
        LOG(DEBUG) << "Finished getting arg strings data";
        return data.CutData(old_offset, offset);
    }

};