// // Created by Иван_Архипов on 24.11.2017. // #include "Subfiles/TextSubfile.h" #include "BinaryData.h" #include "DatFile.h" #include "SubfileData.h" #include "EasyLogging++/easylogging++.h" #include #include std::u16string to_utf16(long long x) { std::u16string res; while (x > 0) { res += char16_t(u'0' + x % 10); x /= 10ll; } std::reverse(res.begin(), res.end()); return res; } long long from_utf16(const std::u16string &num) { long long res = 0; for (auto c : num) { res = res * 10ll + (c - u'0'); } return res; } std::string argumentsFromUtf16(const std::u16string &args) { std::string res; size_t pointer = 0; while (pointer < args.length()) { size_t pointer1 = args.find(u'-', pointer); if (pointer1 == std::u16string::npos) pointer1 = args.length(); if (!res.empty()) res += "-"; res += std::to_string(from_utf16(args.substr(pointer, pointer1 - pointer))); pointer = pointer1 + 1; } return res; } namespace LOTRO_DAT { TextSubfile::TextSubfile() = default; TextSubfile::TextSubfile(DatFile *dat, long long dictionary_offset, long long unknown1, long long file_id, long long file_offset, long long file_size, long long timestamp, long long version, long long block_size, long long unknown2) : Subfile(dat, dictionary_offset, unknown1, file_id, file_offset, file_size, timestamp, version, block_size, unknown2) { } FILE_TYPE TextSubfile::FileType() const { return TEXT; } std::string TextSubfile::Extension() const { return std::string(".txt"); } SubfileData TextSubfile::PrepareForExport(const BinaryData &file_data) { if (file_data.Empty()) { return SubfileData(); } SubfileData result; long long offset = 9; // first 4 bytes - file_id, then 4 bytes - unknown, then 1 byte - unknown long long text_fragment_num = file_data.ToNumber<1>(offset); if ((text_fragment_num & 0x80) != 0) { text_fragment_num = (((text_fragment_num ^ 0x80) << 8) | file_data.ToNumber<1>(offset + 1)); offset += 1; } offset += 1; for (long long i = 0; i < text_fragment_num; i++) { long long fragment_id = file_data.ToNumber<8>(offset); offset += 8; std::vector text_pieces = MakePieces(file_data, offset); std::vector arg_references = MakeArgumentReferences(file_data, offset); std::vector> arg_strings = MakeArgumentStrings(file_data, offset); std::u16string text = u"["; for (size_t j = 0; j + 1 < text_pieces.size(); j++) text += text_pieces[j] + u"<--DO_NOT_TOUCH!-->"; text += text_pieces[text_pieces.size() - 1] + u"]"; std::u16string arguments; for (size_t j = 0; j + 1 < arg_references.size(); j++) arguments += to_utf16(arg_references[j]) + u"-"; if (!arg_references.empty()) arguments += to_utf16(arg_references[arg_references.size() - 1]); if (result.text_data.length() > 0) result.text_data += u"|||"; result.text_data += to_utf16(fragment_id) + u":::"; result.text_data += arguments + u":::"; result.text_data += text; } result.options["fid"] = file_id(); result.options["ext"] = Extension(); return result; } BinaryData TextSubfile::MakeForImport(const BinaryData &old_data, const SubfileData &data) { LOG(DEBUG) << "Preparing text file " << file_id() << " for import."; std::unordered_map patch_fragments = ParsePatchFragments(data); BinaryData new_data; if (file_size() <= 10 + 8) // File is empty, nothing to do; return old_data; long long offset = 9 + 8; // first 8 bytes - file_info. After them: // first 4 bytes - file_id, then 4 bytes - unknown, then 1 byte - unknown long long text_fragment_num = old_data.ToNumber<1>(offset); if ((text_fragment_num & 0x80) != 0) { text_fragment_num = (((text_fragment_num ^ 0x80) << 8) | old_data.ToNumber<1>(offset + 1)); offset += 1; } offset += 1; new_data = new_data + old_data.CutData(0, offset); // Adding file info for (long long i = 0; i < text_fragment_num; i++) { long long fragment_id = old_data.ToNumber<8>(offset); offset += 8; new_data = new_data + old_data.CutData(offset - 8, offset); if (patch_fragments.count(fragment_id) == 0) { // Retrieving old pieces new_data = new_data + GetPieceData(old_data, offset); // Retrieving old references new_data = new_data + GetArgumentReferenceData(old_data, offset); // Retrieving old ref_strings new_data = new_data + GetArgumentStringsData(old_data, offset); } else { // Making and adding new pieces new_data = new_data + BuildPieces(old_data, patch_fragments[fragment_id], offset); // Making and adding new references new_data = new_data + BuildArgumentReferences(old_data, patch_fragments[fragment_id], offset); // Making and adding new strings new_data = new_data + BuildArgumentStrings(old_data, patch_fragments[fragment_id], offset); } } new_data = new_data + old_data.CutData(offset); // Adding elapsed file data return new_data; } std::unordered_map TextSubfile::ParsePatchFragments(const SubfileData &data) { LOG(DEBUG) << "Started parsing patch fragments"; std::unordered_map res; std::u16string text = data.text_data; size_t pointer = 0; while (pointer < text.length()) { // Parsing fragment_id size_t pointer1 = text.find(u":::", pointer); if (pointer1 == std::u16string::npos) { LOG(ERROR) << "Unable to parse fragment id! Cannot find '...' divider. File_id = " << file_id_; return res; } long long fragment_id = from_utf16(text.substr(pointer, pointer1 - pointer)); pointer = pointer1 + 3; res[fragment_id] = SubfileData(); res[fragment_id].options["gid"] = fragment_id; // Parsing arguments pointer1 = text.find(u":::", pointer); if (pointer1 == std::u16string::npos) { LOG(ERROR) << "Unable to parse arguments! Cannot find '...' divider. File_id = " << file_id_; return res; } std::u16string arguments = text.substr(pointer, pointer1 - pointer); pointer = pointer1 + 3; if (arguments.length() > 0) { res[fragment_id].options["args"] = argumentsFromUtf16(arguments); } // Parsing text pointer1 = text.find(u"|||", pointer); if (pointer1 == std::u16string::npos) pointer1 = text.length(); std::u16string text_data = text.substr(pointer, pointer1 - pointer); pointer = pointer1 + 3; res[fragment_id].text_data = text_data; } LOG(DEBUG) << "Finished parsing text patch fragments"; return res; } // Make pieces/arguments/argument strings functions std::vector TextSubfile::MakePieces(const BinaryData &data, long long &offset) { LOG(DEBUG) << "Started making pieces"; long long num_pieces = data.ToNumber<4>(offset); offset += 4; std::vector text_pieces; for (long long j = 0; j < num_pieces; j++) { long long piece_size = data.ToNumber<1>(offset); if ((piece_size & 128) != 0) { piece_size = (((piece_size ^ 128) << 8) | data.ToNumber<1>(offset + 1)); offset += 1; } offset += 1; BinaryData piece_data = data.CutData(offset, offset + piece_size * 2); std::u16string piece; for (long long k = 0; k < piece_size; k++) { char16_t c = char16_t( ((short(piece_data[2 * unsigned(k) + 1])) << 8) | (short(piece_data[2 * unsigned(k)]))); piece += c; } text_pieces.push_back(piece); offset += piece_size * 2; } LOG(DEBUG) << "Finished making pieces"; return text_pieces; } std::vector TextSubfile::MakeArgumentReferences(const BinaryData &data, long long &offset) { LOG(DEBUG) << "Started making argument references"; std::vector arg_references; long long num_references = data.ToNumber<4>(offset); offset += 4; for (long long j = 0; j < num_references; j++) { arg_references.emplace_back(data.ToNumber<4>(offset)); offset += 4; } LOG(DEBUG) << "Finished making argument references"; return arg_references; } std::vector> TextSubfile::MakeArgumentStrings(const BinaryData &data, long long &offset) { LOG(DEBUG) << "Started making argument strings"; std::vector > arg_strings; long long num_arg_strings = data.ToNumber<1>(offset); offset += 1; for (long long j = 0; j < num_arg_strings; j++) { long long num_args = data.ToNumber<4>(offset); offset += 4; arg_strings.emplace_back(); for (long long k = 0; k < num_args; k++) { long long string_size = data.ToNumber<1>(offset); if ((string_size & 0x80) != 0) { string_size = (((string_size ^ 0x80) << 8) | data.ToNumber<1>(offset + 1)); offset += 1; } offset += 1; arg_strings[unsigned(j)].emplace_back(data.CutData(offset, offset + string_size * 2)); offset += string_size * 2; } } LOG(DEBUG) << "Finished making argument strings"; return arg_strings; } // Build pieces/arguments/argument strings functions from fragment SubfileData BinaryData TextSubfile::BuildPieces(const BinaryData &data, const SubfileData &new_data, long long &offset) { LOG(DEBUG) << "Started building pieces"; // Moving &offset pointer in &data GetPieceData(data, offset); // Deleting '[' and ']' brackets std::u16string text_data = new_data.text_data.substr(1, new_data.text_data.size() - 2); std::vector pieces; const std::u16string DNT = u"<--DO_NOT_TOUCH!-->"; size_t prev = 0; size_t next = text_data.find(DNT, prev); while (next != std::string::npos) { std::u16string piece = text_data.substr(prev, next - prev); pieces.emplace_back(piece); prev = next + DNT.length(); next = text_data.find(DNT, prev); } pieces.emplace_back(text_data.substr(prev)); // Building BinaryData from pieces BinaryData result; BinaryData temp_data = BinaryData::FromNumber<4>(pieces.size()); result = result + temp_data; for (auto piece : pieces) { long long piece_size = piece.length(); if (piece_size < 128) { temp_data = BinaryData::FromNumber<1>(piece_size); } else { temp_data = BinaryData::FromNumberRAW<2>((piece_size | 32768)); } result = result + temp_data; for (long long j = 0; j < piece_size; j++) { temp_data = BinaryData::FromNumber<2>(short(piece[j])); result = result + temp_data; } } LOG(DEBUG) << "Pieces built successfully"; return result; } BinaryData TextSubfile::BuildArgumentReferences(const BinaryData &data, const SubfileData &new_data, long long &offset) { LOG(DEBUG) << "Started building argument refs"; // Moving &offset pointer in &data GetArgumentReferenceData(data, offset); // If there are no args - making 4 null-bytes and return; if (!new_data.options["args"]) { BinaryData result = BinaryData::FromNumber<4>(0); return result; } // Parsing arguments from list in options["args"] std::string args_list = new_data.options["args"].as(); std::vector arguments; size_t prev = 0; size_t next = args_list.find('-', prev); while (next != std::string::npos) { std::string argument = args_list.substr(prev, next - prev); arguments.push_back(std::stoll(argument)); prev = next + 1; next = args_list.find('-', prev); } std::string argument = args_list.substr(prev); arguments.push_back(std::stoll(argument)); BinaryData result; BinaryData temp_data = BinaryData::FromNumber<4>(arguments.size()); result = result + temp_data; for (auto arg_reference : arguments) { temp_data = BinaryData::FromNumber<4>(arg_reference); result = result + temp_data; } LOG(DEBUG) << "Argument refs built successfully"; return result; } BinaryData TextSubfile::BuildArgumentStrings(const BinaryData &data, const SubfileData &, long long &offset) { LOG(DEBUG) << "Started building argument strings"; LOG(DEBUG) << "Built arg strings successfully"; return GetArgumentStringsData(data, offset); } // Get BinaryData contents of pieces/arguments/argument strings BinaryData TextSubfile::GetPieceData(const BinaryData &data, long long &offset) const { LOG(DEBUG) << "Started getting piece data"; long long old_offset = offset; long long num_pieces = data.ToNumber<4>(offset); offset += 4; for (long long j = 0; j < num_pieces; j++) { long long piece_size = data.ToNumber<1>(offset); if ((piece_size & 128) != 0) { piece_size = (((piece_size ^ 128) << 8) | data.ToNumber<1>(offset + 1)); offset += 1; } offset += 1; offset += piece_size * 2; } LOG(DEBUG) << "Got piece data"; return data.CutData(old_offset, offset); } BinaryData TextSubfile::GetArgumentReferenceData(const BinaryData &data, long long &offset) const { LOG(DEBUG) << "Started getting arg refs data"; long long old_offset = offset; long long num_references = data.ToNumber<4>(offset); offset += 4; offset += 4 * num_references; LOG(DEBUG) << "Finished getting arg refs data"; return data.CutData(old_offset, offset); } BinaryData TextSubfile::GetArgumentStringsData(const BinaryData &data, long long &offset) const { LOG(DEBUG) << "Started getting arg strings data"; long long old_offset = offset; long long num_arg_strings = data.ToNumber<1>(offset); offset += 1; for (long long j = 0; j < num_arg_strings; j++) { long long num_args = data.ToNumber<4>(offset); offset += 4; for (long long k = 0; k < num_args; k++) { long long string_size = data.ToNumber<1>(offset); if ((string_size & 0x80) != 0) { string_size = (((string_size ^ 0x80) << 8) | data.ToNumber<1>(offset + 1)); offset += 1; } offset += 1; offset += string_size * 2; } } LOG(DEBUG) << "Finished getting arg strings data"; return data.CutData(old_offset, offset); } };