// // Created by Иван_Архипов on 24.11.2017. // #include "Subfiles/TextSubFile.h" #include "BinaryData.h" #include "DatFile.h" #include "SubfileData.h" #include "EasyLogging++/easylogging++.h" #include #include std::u16string to_utf16(long long x) { std::u16string res; while (x > 0) { res += char16_t(u'0' + x % 10); x /= 10ll; } std::reverse(res.begin(), res.end()); return res; } long long from_utf16(const std::u16string &num) { long long res = 0; for (auto c : num) { res = res * 10ll + (c - u'0'); } return res; } std::string argumentsFromUtf16(const std::u16string &args) { std::string res; size_t pointer = 0; while (pointer < args.length()) { size_t pointer1 = args.find(u'-', pointer); if (pointer1 == std::u16string::npos) pointer1 = args.length(); if (!res.empty()) res += "-"; res += std::to_string(from_utf16(args.substr(pointer, pointer1 - pointer))); pointer = pointer1 + 1; } return res; } namespace LOTRO_DAT { BinaryData TextSubFile::buffer_ = BinaryData(10 * 1024 * 1024); TextSubFile::TextSubFile(DatFile &dat, long long dictionary_offset, long long unknown1, long long file_id, long long file_offset, long long file_size, long long timestamp, long long version, long long block_size, long long unknown2) : SubFile(dat, dictionary_offset, unknown1, file_id, file_offset, file_size, timestamp, version, block_size, unknown2) { } FILE_TYPE TextSubFile::FileType() const { return TEXT; } std::string TextSubFile::Extension() const { return std::string(".txt"); } SubfileData TextSubFile::PrepareForExport(const BinaryData &file_data) { if (file_data.Empty()) { patch_fragments_.clear(); text_pieces_.clear(); argument_references_.clear(); argument_strings_.clear(); return SubfileData(); } SubfileData result; long long offset = 9; // first 4 bytes - file_id, then 4 bytes - unknown, then 1 byte - unknown long long text_fragment_num = file_data.ToNumber<1>(offset); if ((text_fragment_num & 0x80) != 0) { text_fragment_num = (((text_fragment_num ^ 0x80) << 8) | file_data.ToNumber<1>(offset + 1)); offset += 1; } offset += 1; for (long long i = 0; i < text_fragment_num; i++) { long long fragment_id = file_data.ToNumber<8>(offset); offset += 8; MakePieces(file_data, offset); MakeArgumentReferences(file_data, offset); MakeArgumentStrings(file_data, offset); std::u16string text = u"["; for (size_t j = 0; j + 1 < text_pieces_.size(); j++) text += text_pieces_[j] + u"<--DO_NOT_TOUCH!-->"; text += text_pieces_[text_pieces_.size() - 1] + u"]"; std::u16string arguments; for (size_t j = 0; j + 1 < argument_references_.size(); j++) arguments += to_utf16(argument_references_[j]) + u"-"; if (!argument_references_.empty()) arguments += to_utf16(argument_references_[argument_references_.size() - 1]); if (result.text_data.length() > 0) result.text_data += u"|||"; result.text_data += to_utf16(fragment_id) + u":::"; result.text_data += arguments + u":::"; result.text_data += text; } result.options["fid"] = file_id(); result.options["ext"] = Extension(); patch_fragments_.clear(); text_pieces_.clear(); argument_references_.clear(); argument_strings_.clear(); return result; } BinaryData TextSubFile::MakeForImport(const BinaryData &old_data, const SubfileData &data) { LOG(DEBUG) << "Preparing text file " << file_id() << " for import."; ParsePatchFragments(data); if (file_size() <= 10 + 8) {// File is empty, nothing to do; patch_fragments_.clear(); text_pieces_.clear(); argument_references_.clear(); argument_strings_.clear(); return old_data; } BinaryData new_data; long long offset = 9 + 8; // first 8 bytes - file_info. After them: // first 4 bytes - file_id, then 4 bytes - unknown, then 1 byte - unknown long long text_fragment_num = old_data.ToNumber<1>(offset); if ((text_fragment_num & 0x80) != 0) { text_fragment_num = (((text_fragment_num ^ 0x80) << 8) | old_data.ToNumber<1>(offset + 1)); offset += 1; } offset += 1; new_data = new_data + old_data.CutData(0, offset); // Adding file info for (long long i = 0; i < text_fragment_num; i++) { long long fragment_id = old_data.ToNumber<8>(offset); offset += 8; new_data = new_data + old_data.CutData(offset - 8, offset); TextFragment id_comp; id_comp.fragment_id = fragment_id; auto fragment_iterator = std::lower_bound(patch_fragments_.begin(), patch_fragments_.end(), id_comp); if (fragment_iterator == patch_fragments_.end()) { // Retrieving old pieces new_data = new_data + GetPieceData(old_data, offset); // Retrieving old references new_data = new_data + GetArgumentReferenceData(old_data, offset); // Retrieving old ref_strings new_data = new_data + GetArgumentStringsData(old_data, offset); } else { // Making and adding new pieces new_data = new_data + BuildPieces(old_data, *fragment_iterator, offset); // Making and adding new references new_data = new_data + BuildArgumentReferences(old_data, *fragment_iterator, offset); // Making and adding new strings new_data = new_data + BuildArgumentStrings(old_data, *fragment_iterator, offset); } } new_data = new_data + old_data.CutData(offset); // Adding elapsed file data patch_fragments_.clear(); text_pieces_.clear(); argument_references_.clear(); argument_strings_.clear(); return new_data; } void TextSubFile::ParsePatchFragments(const SubfileData &data) { LOG(DEBUG) << "Started parsing patch fragments"; size_t pointer = 0; while (pointer < data.text_data.length()) { // Parsing fragment_id size_t pointer1 = data.text_data.find(u":::", pointer); if (pointer1 == std::u16string::npos) { LOG(ERROR) << "Unable to parse fragment id! Cannot find '...' divider. File_id = " << file_id_; return; } long long fragment_id = from_utf16(data.text_data.substr(pointer, pointer1 - pointer)); pointer = pointer1 + 3; TextFragment fragment; fragment.fragment_id = fragment_id; // Parsing arguments pointer1 = data.text_data.find(u":::", pointer); if (pointer1 == std::u16string::npos) { LOG(ERROR) << "Unable to parse arguments! Cannot find '...' divider. File_id = " << file_id_; return; } std::u16string arguments = data.text_data.substr(pointer, pointer1 - pointer); pointer = pointer1 + 3; if (arguments.length() > 0) { fragment.args = argumentsFromUtf16(arguments); } // Parsing text pointer1 = data.text_data.find(u"|||", pointer); if (pointer1 == std::u16string::npos) pointer1 = data.text_data.length(); fragment.text = data.text_data.substr(pointer, pointer1 - pointer); pointer = pointer1 + 3; patch_fragments_.push_back(fragment); } std::sort(patch_fragments_.begin(), patch_fragments_.end()); LOG(DEBUG) << "Finished parsing text patch fragments"; } // Make pieces/arguments/argument strings functions void TextSubFile::MakePieces(const BinaryData &data, long long &offset) { LOG(DEBUG) << "Started making pieces"; long long num_pieces = data.ToNumber<4>(offset); offset += 4; text_pieces_.resize(num_pieces); for (long long j = 0; j < num_pieces; j++) { long long piece_size = data.ToNumber<1>(offset); if ((piece_size & 128) != 0) { piece_size = (((piece_size ^ 128) << 8) | data.ToNumber<1>(offset + 1)); offset += 1; } offset += 1; BinaryData piece_data = data.CutData(offset, offset + piece_size * 2); std::u16string piece; for (long long k = 0; k < piece_size; k++) { char16_t c = char16_t( ((short(piece_data[2 * unsigned(k) + 1])) << 8) | (short(piece_data[2 * unsigned(k)]))); piece += c; } text_pieces_[j] = piece; offset += piece_size * 2; } LOG(DEBUG) << "Finished making pieces"; } void TextSubFile::MakeArgumentReferences(const BinaryData &data, long long &offset) { LOG(DEBUG) << "Started making argument references"; long long num_references = data.ToNumber<4>(offset); offset += 4; argument_references_.resize(num_references); for (long long j = 0; j < num_references; j++) { argument_references_[j] = data.ToNumber<4>(offset); offset += 4; } LOG(DEBUG) << "Finished making argument references"; } void TextSubFile::MakeArgumentStrings(const BinaryData &data, long long &offset) { LOG(DEBUG) << "Started making argument strings"; long long num_arg_strings = data.ToNumber<1>(offset); offset += 1; argument_strings_.resize(num_arg_strings); for (long long j = 0; j < num_arg_strings; j++) { long long num_args = data.ToNumber<4>(offset); offset += 4; argument_strings_[j].resize(num_args); for (long long k = 0; k < num_args; k++) { long long string_size = data.ToNumber<1>(offset); if ((string_size & 0x80) != 0) { string_size = (((string_size ^ 0x80) << 8) | data.ToNumber<1>(offset + 1)); offset += 1; } offset += 1; argument_strings_[j][k] = data.CutData(offset, offset + string_size * 2); offset += string_size * 2; } } LOG(DEBUG) << "Finished making argument strings"; } // Build pieces/arguments/argument strings functions from fragment SubfileData BinaryData TextSubFile::BuildPieces(const BinaryData &data, const TextFragment &new_data, long long &offset) { LOG(DEBUG) << "Started building pieces"; // Moving &offset pointer in &data GetPieceData(data, offset); std::u16string file_data = u" fid:" + to_utf16(file_id_) + u" gid:" + to_utf16(new_data.fragment_id); // Deleting '[' and ']' brackets std::u16string text_data = new_data.text.substr(1, new_data.text.size() - 2) + file_data; text_pieces_.clear(); const std::u16string DNT = u"<--DO_NOT_TOUCH!-->"; size_t prev = 0; size_t next = text_data.find(DNT, prev); while (next != std::string::npos) { std::u16string piece = text_data.substr(prev, next - prev); text_pieces_.push_back(piece); prev = next + DNT.length(); next = text_data.find(DNT, prev); } text_pieces_.push_back(text_data.substr(prev)); // Building BinaryData from pieces unsigned buffer_offset = 0; buffer_.Append(BinaryData::FromNumber<4>(text_pieces_.size()), buffer_offset); buffer_offset += 4; for (const std::u16string &piece : text_pieces_) { long long piece_size = piece.length(); if (piece_size < 128) { buffer_.Append(BinaryData::FromNumber<1>(piece_size), buffer_offset); buffer_offset += 1; } else { buffer_.Append(BinaryData::FromNumberRAW<2>((piece_size | 32768)), buffer_offset); buffer_offset += 2; } for (long long j = 0; j < piece_size; j++) { buffer_.Append(BinaryData::FromNumber<2>(short(piece[j])), buffer_offset); buffer_offset += 2; } } LOG(DEBUG) << "Pieces built successfully"; return buffer_.CutData(0, buffer_offset); } BinaryData TextSubFile::BuildArgumentReferences(const BinaryData &data, const TextFragment &new_data, long long &offset) { LOG(DEBUG) << "Started building argument refs"; // Moving &offset pointer in &data GetArgumentReferenceData(data, offset); // If there are no args - making 4 null-bytes and return; if (new_data.args.empty()) { BinaryData result = BinaryData::FromNumber<4>(0); return result; } // Parsing arguments from list in options["args"] std::string args_list = new_data.args; argument_references_.clear(); size_t prev = 0; size_t next = args_list.find('-', prev); while (next != std::string::npos) { std::string argument = args_list.substr(prev, next - prev); argument_references_.push_back(std::stoll(argument)); prev = next + 1; next = args_list.find('-', prev); } std::string argument = args_list.substr(prev); argument_references_.push_back(std::stoll(argument)); BinaryData result; BinaryData temp_data = BinaryData::FromNumber<4>(argument_references_.size()); result = result + temp_data; for (const long long &arg_reference : argument_references_) { temp_data = BinaryData::FromNumber<4>(arg_reference); result = result + temp_data; } LOG(DEBUG) << "Argument refs built successfully"; return result; } BinaryData TextSubFile::BuildArgumentStrings(const BinaryData &data, const TextFragment &, long long &offset) { LOG(DEBUG) << "Started building argument strings"; LOG(DEBUG) << "Built arg strings successfully"; return GetArgumentStringsData(data, offset); } // Get BinaryData contents of pieces/arguments/argument strings BinaryData TextSubFile::GetPieceData(const BinaryData &data, long long &offset) const { LOG(DEBUG) << "Started getting piece data"; long long old_offset = offset; long long num_pieces = data.ToNumber<4>(offset); offset += 4; for (long long j = 0; j < num_pieces; j++) { long long piece_size = data.ToNumber<1>(offset); if ((piece_size & 128) != 0) { piece_size = (((piece_size ^ 128) << 8) | data.ToNumber<1>(offset + 1)); offset += 1; } offset += 1; offset += piece_size * 2; } LOG(DEBUG) << "Got piece data"; return data.CutData(old_offset, offset); } BinaryData TextSubFile::GetArgumentReferenceData(const BinaryData &data, long long &offset) const { LOG(DEBUG) << "Started getting arg refs data"; long long old_offset = offset; long long num_references = data.ToNumber<4>(offset); offset += 4; offset += 4 * num_references; LOG(DEBUG) << "Finished getting arg refs data"; return data.CutData(old_offset, offset); } BinaryData TextSubFile::GetArgumentStringsData(const BinaryData &data, long long &offset) const { LOG(DEBUG) << "Started getting arg strings data"; long long old_offset = offset; long long num_arg_strings = data.ToNumber<1>(offset); offset += 1; for (long long j = 0; j < num_arg_strings; j++) { long long num_args = data.ToNumber<4>(offset); offset += 4; for (long long k = 0; k < num_args; k++) { long long string_size = data.ToNumber<1>(offset); if ((string_size & 0x80) != 0) { string_size = (((string_size ^ 0x80) << 8) | data.ToNumber<1>(offset + 1)); offset += 1; } offset += 1; offset += string_size * 2; } } LOG(DEBUG) << "Finished getting arg strings data"; return data.CutData(old_offset, offset); } };