LotroDat/Source/Subfiles/TextSubfile.cpp

//
// Created by Иван_Архипов on 24.11.2017.
//

#include "TextSubfile.h"
#include "../BinaryData.h"
#include "../DatFile.h"
#include "../Common/DatException.h"
#include "../SubfileData.h"
#include <algorithm>
#include <codecvt>
#include <locale>

std::u16string to_utf16(long long x) {
    std::u16string res;
    while (x > 0) {
        res += u'0' + (x % 10);
        x /= 10ll;
    }
    std::reverse(res.begin(), res.end());
    return res;
}

long long from_utf16(const std::u16string &num) {
    long long res = 0;
    for (auto c : num) {
        res = res * 10ll + (c - u'0');
    }
    std::cout << "fragment_id: " << res << std::endl;
    return res;
}

std::string argumentsFromUtf16(const std::u16string &args) {
    std::string res;

    size_t pointer = 0;
    while (pointer < args.length()) {
        size_t pointer1 = args.find(u'-', pointer);
        if (pointer1 == std::u16string::npos)
            pointer1 = args.length();
        if (!res.empty())
            res += "-";
        res += std::to_string(from_utf16(args.substr(pointer, pointer1 - pointer)));
        pointer = pointer1 + 1;
    }

    std::wstring_convert<std::codecvt_utf8_utf16<char16_t>,char16_t> codecvt;
    std::cout << "Arguments old: "<< codecvt.to_bytes(args) << '\n' ;
    std::cout << "Arguments new: "<< res << '\n' ;
    return res;
}

namespace LOTRO_DAT {
    TextSubfile::TextSubfile() = default;

    TextSubfile::TextSubfile(DatFile *dat, long long dictionary_offset, long long fragments_count, long long unknown1,
                           long long file_id, long long file_offset, long long file_size,
                           long long timestamp,
                           long long version, long long block_size)
            : Subfile(dat, dictionary_offset, fragments_count, unknown1, file_id, file_offset, file_size, timestamp, version, block_size) {
    }

    FILE_TYPE TextSubfile::FileType() const {
        return TEXT;
    }

    std::string TextSubfile::Extension() const {
        return std::string(".txt");
    }

    SubfileData TextSubfile::PrepareForExport(const BinaryData &file_data) {
        SubfileData result;
        if (file_size() <= 10) // File is empty, nothing to do;
            return result;

        long long offset = 9; // first 4 bytes - file_id, then 4 bytes - unknown, then 1 byte - unknown

        long long text_fragment_num = file_data.ToNumber<1>(offset);
        if ((text_fragment_num & 0x80) != 0) {
            text_fragment_num = (((text_fragment_num ^ 0x80) << 8) | file_data.ToNumber<1>(offset + 1));
            offset += 1;
        }
        offset += 1;

        for (long long i = 0; i < text_fragment_num; i++) {
            long long fragment_id = file_data.ToNumber<8>(offset);
            offset += 8;

            std::vector<std::u16string> text_pieces = MakePieces(file_data, offset);
            std::vector<long long> arg_references = MakeArgumentReferences(file_data, offset);
            std::vector<std::vector<BinaryData>> arg_strings = MakeArgumentStrings(file_data, offset);

            std::u16string text = u"[";
            for (size_t j = 0; j + 1 < text_pieces.size(); j++)
                text += text_pieces[j] + u"<--DO_NOT_TOUCH!-->";
            text += text_pieces[text_pieces.size() - 1] + u"]";

            std::u16string arguments;
            for (size_t j = 0; j + 1 < arg_references.size(); j++)
                arguments += to_utf16(arg_references[j]) + u"-";
            if (!arg_references.empty())
                arguments += to_utf16(arg_references[arg_references.size() - 1]);

            if (result.text_data.length() > 0)
                result.text_data += u"|||";

            result.text_data += to_utf16(fragment_id) + u":::";
            result.text_data += arguments + u":::";
            result.text_data += text;
        }
        result.options["fid"] = file_id();
        result.options["ext"] = Extension();
        return result;
    }

    BinaryData TextSubfile::MakeForImport(const BinaryData &old_data, const SubfileData &data) {
        std::unordered_map<long long, SubfileData> patch_fragments = ParsePatchFragments(data);
        std::cout << "Made fragments. Fragments are: " << std::flush;
        for (auto &i : patch_fragments)
            std::cout << i.first << " " << std::flush;
        std::cout << std::endl;

        BinaryData new_data;

        if (file_size() <= 10 + 8) // File is empty, nothing to do;
            return old_data;

        long long offset = 9 + 8; // first 8 bytes - file_info. After them:
        // first 4 bytes - file_id, then 4 bytes - unknown, then 1 byte - unknown

        long long text_fragment_num = old_data.ToNumber<1>(offset);
        if ((text_fragment_num & 0x80) != 0) {
            text_fragment_num = (((text_fragment_num ^ 0x80) << 8) | old_data.ToNumber<1>(offset + 1));
            offset += 1;
        }
        offset += 1;

        new_data = new_data + old_data.CutData(0, offset); // Adding file info

        for (long long i = 0; i < text_fragment_num; i++) {
            long long fragment_id = old_data.ToNumber<8>(offset);
            offset += 8;

            new_data = new_data + old_data.CutData(offset - 8, offset);

            if (patch_fragments.count(fragment_id) == 0) {
                std::cerr << "Omg, what..? " << file_id() << " " << fragment_id << " is not in patch? " << std::endl;
                try {
                    // Retrieving old pieces
                    new_data = new_data + GetPieceData(old_data, offset);
                } catch (std::exception &e) {
                    fprintf(stderr, "Caught %s exception.\n", e.what());
                    fprintf(stderr, "ERROR TextSubfile::MakeForImport() - unable to get piece data for file_id %lld and fragment_id %lld", file_id(), fragment_id);
                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
                }

                try {
                    // Retrieving old references
                    new_data = new_data + GetArgumentReferenceData(old_data, offset);
                } catch (std::exception &e) {
                    fprintf(stderr, "Caught %s exception.\n", e.what());
                    fprintf(stderr, "ERROR TextSubfile::MakeForImport() - unable to get argument reference data for file_id %lld and fragment_id %lld", file_id(), fragment_id);
                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
                }

                try {
                    // Retrieving old ref_strings
                    new_data = new_data + GetArgumentStringsData(old_data, offset);
                } catch (std::exception &e) {
                    fprintf(stderr, "Caught %s exception.\n", e.what());
                    fprintf(stderr, "ERROR TextSubfile::MakeForImport() - unable to get argument string for file_id %lld and fragment_id %lld", file_id(), fragment_id);
                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
                }

            } else {
                try {
                    // Making and adding new pieces
                    new_data = new_data + BuildPieces(old_data, patch_fragments[fragment_id], offset);
                } catch (std::exception &e) {
                    fprintf(stderr, "Caught %s exception.\n", e.what());
                    fprintf(stderr, "ERROR TextSubfile::MakeForImport() - unable to build piece data for file_id %lld and fragment_id %lld", file_id(), fragment_id);
                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
                }

                try {
                    // Making and adding new references
                    new_data = new_data + BuildArgumentReferences(old_data, patch_fragments[fragment_id], offset);
                } catch (std::exception &e) {
                    fprintf(stderr, "Caught %s exception.\n", e.what());
                    fprintf(stderr, "ERROR TextSubfile::MakeForImport() - unable to build argument references data for file_id %lld and fragment_id %lld", file_id(), fragment_id);
                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
                }

                try {
                    // Making and adding new strings
                    new_data = new_data + BuildArgumentStrings(old_data, patch_fragments[fragment_id], offset);
                } catch (std::exception &e) {
                    fprintf(stderr, "Caught %s exception.\n", e.what());
                    fprintf(stderr, "ERROR TextSubfile::MakeForImport() - unable to build argument strings data for file_id %lld and fragment_id %lld", file_id(), fragment_id);
                    throw DatException("Bad TextSubfile::MakeForImport()", IMPORT_EXCEPTION);
                }
            }
        }
        new_data = new_data + old_data.CutData(offset); // Adding elapsed file data
        return new_data;
    }

    std::unordered_map<long long, SubfileData> TextSubfile::ParsePatchFragments(const SubfileData &data) {
        std::unordered_map<long long, SubfileData> res;
        std::u16string text = data.text_data;

        size_t pointer = 0;
        while (pointer < text.length()) {
            // Parsing fragment_id
            size_t pointer1 = text.find(u":::", pointer);
            if (pointer1 == std::u16string::npos)
                throw DatException("Bad TextSubfile::ParsePatchFragments() - Unable to parse fragment id! Cannot find '...' divider");
            long long fragment_id = from_utf16(text.substr(pointer, pointer1 - pointer));
            pointer = pointer1 + 3;
            res[fragment_id] = SubfileData();
            res[fragment_id].options["gid"] = fragment_id;

            // Parsing arguments
            pointer1 = text.find(u":::", pointer);
            if (pointer1 == std::u16string::npos)
                throw DatException("Bad TextSubfile::ParsePatchFragments() - Unable to parse arguments! Cannot find '...' divider");
            std::u16string arguments = text.substr(pointer, pointer1 - pointer);
            pointer = pointer1 + 3;
            if (arguments.length() > 0) {
                res[fragment_id].options["args"] = argumentsFromUtf16(arguments);
            }

            // Parsing text
            pointer1 = text.find(u"|||", pointer);
            if (pointer1 == std::u16string::npos)
                pointer1 = text.length();
            std::u16string text_data = text.substr(pointer, pointer1 - pointer);
            pointer = pointer1  + 3;
            res[fragment_id].text_data = text_data;


            std::wstring_convert<std::codecvt_utf8_utf16<char16_t>,char16_t> codecvt;
            std::cout << "Fragment id: " << res[fragment_id].options["gid"].as<long long>() << "\n";
            if (res[fragment_id].options["args"])
                std::cout << "Args: " << res[fragment_id].options["args"].as<std::string>() << "\n";
            else
                std::cout << "Args: none \n";
            std::cout << "Text: " << codecvt.to_bytes(res[fragment_id].text_data) << '\n' ;
        }
        return res;
    }

    // Make pieces/arguments/argument strings functions

    std::vector<std::u16string> TextSubfile::MakePieces(const BinaryData &data, long long &offset) {
        long long num_pieces = data.ToNumber<4>(offset);
        offset += 4;

        std::vector<std::u16string> text_pieces;

        for (long long j = 0; j < num_pieces; j++) {
            long long piece_size = data.ToNumber<1>(offset);
            if ((piece_size & 128) != 0) {
                piece_size = (((piece_size ^ 128) << 8) | data.ToNumber<1>(offset + 1));
                offset += 1;
            }
            offset += 1;

            BinaryData piece_data = data.CutData(offset, offset + piece_size * 2);
            std::u16string piece;

            for (long long k = 0; k < piece_size; k++) {
                char16_t c = char16_t(
                        ((short(piece_data[2 * unsigned(k) + 1])) << 8) | (short(piece_data[2 * unsigned(k)])));
                piece += c;
            }

            text_pieces.push_back(piece);
            offset += piece_size * 2;
        }
        return text_pieces;
    }

    std::vector<long long> TextSubfile::MakeArgumentReferences(const BinaryData &data, long long &offset) {
        std::vector<long long> arg_references;

        long long num_references = data.ToNumber<4>(offset);
        offset += 4;

        for (long long j = 0; j < num_references; j++) {
            arg_references.emplace_back(data.ToNumber<4>(offset));
            offset += 4;
        }

        return arg_references;
    }

    std::vector<std::vector<BinaryData>> TextSubfile::MakeArgumentStrings(const BinaryData &data, long long &offset) {
        std::vector<std::vector<BinaryData> > arg_strings;
        long long num_arg_strings = data.ToNumber<1>(offset);
        offset += 1;

        for (long long j = 0; j < num_arg_strings; j++) {
            long long num_args = data.ToNumber<4>(offset);
            offset += 4;

            arg_strings.emplace_back();
            for (long long k = 0; k < num_args; k++) {
                long long string_size = data.ToNumber<1>(offset);
                if ((string_size & 0x80) != 0) {
                    string_size = (((string_size ^ 0x80) << 8) | data.ToNumber<1>(offset + 1));
                    offset += 1;
                }
                offset += 1;

                arg_strings[unsigned(j)].emplace_back(data.CutData(offset, offset + string_size * 2));
                offset += string_size * 2;
            }
        }
        return arg_strings;
    }

    // Build pieces/arguments/argument strings functions from fragment SubfileData

    BinaryData TextSubfile::BuildPieces(const BinaryData &data, const SubfileData &new_data, long long &offset) {
        try {
            // Moving &offset pointer in &data
            GetPieceData(data, offset);
        } catch (std::exception &e) {
            fprintf(stderr, "Caught %s exception.\n", e.what());
            fprintf(stderr, "ERROR TextSubfile::BuildPieces() - unable to get piece data for file_id %lld", file_id());
            throw DatException("Bad TextSubfile::BuildPieces()", IMPORT_EXCEPTION);
        }

        // Deleting '[' and ']' brackets
        std::u16string text_data = new_data.text_data.substr(1, new_data.text_data.size() - 2);
        std::vector<std::u16string> pieces;

        std::wstring_convert<std::codecvt_utf8_utf16<char16_t>,char16_t> codecvt;
        std::cout << "AAAAA0: " << codecvt.to_bytes(text_data) << '\n' ;

        const std::u16string DNT = u"<--DO_NOT_TOUCH!-->";
        size_t prev = 0;
        size_t next = text_data.find(DNT, prev);

        while (next != std::string::npos) {
            std::u16string piece = text_data.substr(prev, next - prev);
            pieces.push_back(piece);
            prev = next + DNT.length();
            next = text_data.find(DNT, prev);
        }
        std::u16string piece = text_data.substr(prev);
        pieces.push_back(piece);

        std::cout << "AAAAA: " << codecvt.to_bytes(piece) << '\n' ;

        // Building BinaryData from pieces
        BinaryData result;
        BinaryData temp_data(4);

        temp_data.FromNumber<4>(pieces.size());
        result = result + temp_data;

        for (long long i = 0; i < pieces.size(); i++) {
            long long piece_size = pieces[i].length();
            if (piece_size < 128) {
                temp_data.FromNumber<1>(piece_size);
            } else {
                temp_data.FromNumberRAW<2>((piece_size | 32768));
            }
            result = result + temp_data;

            for (long long j = 0; j < piece_size; j++) {
                temp_data.FromNumber<2>(short(pieces[i][j]));
                result = result + temp_data;
            }
        }
        return result;
    }

    BinaryData TextSubfile::BuildArgumentReferences(const BinaryData &data, const SubfileData &new_data,
                                                    long long &offset) {
        // Moving &offset pointer in &data
        GetArgumentReferenceData(data, offset);

        // If there are no args - making 4 null-bytes and return;
        if (!new_data.options["args"]) {
            BinaryData result;
            result.FromNumber<4>(0);
            return result;
        }

        // Parsing arguments from list in options["args"]
        std::string args_list = new_data.options["args"].as<std::string>();
        std::vector<long long> arguments;

        size_t prev = 0;
        size_t next = args_list.find('-', prev);
        while (next != std::string::npos) {
            std::string argument = args_list.substr(prev, next - prev);
            arguments.push_back(std::stoll(argument));
            prev = next + 1;
            next = args_list.find('-', prev);
        }
        std::string argument = args_list.substr(prev);
        std::cout << "DDD: Argument is " << argument << std::endl;
        std::cout << "DDD: Arguments are" << new_data.options["args"].as<std::string>() << std::endl;
        arguments.push_back(std::stoll(argument));

        BinaryData result;
        BinaryData temp_data(4);
        temp_data.FromNumber<4>(arguments.size());
        result = result + temp_data;
        for (auto arg_reference : arguments) {
            temp_data.FromNumber<4>(arg_reference);
            result = result + temp_data;
        }
        return result;
    }

    BinaryData TextSubfile::BuildArgumentStrings(const BinaryData &data, const SubfileData &new_data,
                                                 long long &offset) {
        return GetArgumentStringsData(data, offset);
    }

    // Get BinaryData contents of pieces/arguments/argument strings

    BinaryData TextSubfile::GetPieceData(const BinaryData &data, long long &offset) const {
        long long old_offset = offset;

        long long num_pieces = data.ToNumber<4>(offset);
        offset += 4;

        for (long long j = 0; j < num_pieces; j++) {
            long long piece_size = data.ToNumber<1>(offset);
            if ((piece_size & 128) != 0) {
                piece_size = (((piece_size ^ 128) << 8) | data.ToNumber<1>(offset + 1));
                offset += 1;
            }
            offset += 1;
            offset += piece_size * 2;
        }
        std::cout << "DDDDDDDDDDDD: " << old_offset << " " << offset << std::endl;
        return data.CutData(old_offset, offset);
    }

    BinaryData TextSubfile::GetArgumentReferenceData(const BinaryData &data, long long &offset) const {
        long long old_offset = offset;
        long long num_references = data.ToNumber<4>(offset);
        offset += 4;
        offset += 4 * num_references;
        return data.CutData(old_offset, offset);
    }

    BinaryData TextSubfile::GetArgumentStringsData(const BinaryData &data, long long &offset) const {
        long long old_offset = offset;

        long long num_arg_strings = data.ToNumber<1>(offset);
        offset += 1;

        for (long long j = 0; j < num_arg_strings; j++) {
            long long num_args = data.ToNumber<4>(offset);
            offset += 4;

            for (long long k = 0; k < num_args; k++) {
                long long string_size = data.ToNumber<1>(offset);
                if ((string_size & 0x80) != 0) {
                    string_size = (((string_size ^ 0x80) << 8) | data.ToNumber<1>(offset + 1));
                    offset += 1;
                }
                offset += 1;
                offset += string_size * 2;
            }
        }
        return data.CutData(old_offset, offset);
    }

};