pip/utils/value_tree_translator/main.cpp

/*
    PIP - Platform Independent Primitives
	ValueTree translator
    Ivan Pelipenko peri4ko@yandex.ru

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Lesser General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include "picli.h"
#include "pidir.h"
#include "pifile.h"
#include "piiostream.h"
#include "pijson.h"
#include "pivaluetree.h"
#include "pivaluetree_conversions.h"

#include <iostream>

using namespace PICoutManipulators;
using Attribute          = PIValueTree::Attribute;

const char help_string[] = "Read input files or entire directories as PIValueTree\n"
						   "according to extension, and create QtLinguist translation\n"
						   "file (*.ts). Extensions:\n"
						   " * conf - PIValueTreeConversions::fromText\n"
						   " * json - PIValueTreeConversions::fromJSON\n"
						   " * bin  - binary deserialization\n"
						   "Output file can be translated by QtLinguist and used\n"
						   "by QAD PIValueTreeEdit widget to translate strings.\n"
						   "";

void header() {
	piCout << Bold << "PIP ValueTree translator";
	piCout << Cyan << "Version" << Bold << PIPVersion() << NewLine;
	piCout << Green << Bold << "Usage:" << Default
		   << "\"pip_vtt [-hHqn] -l <lang> -o <output_file> <file1/dir1> [<file2/dir2>] [<file3/dir3>] [...]\"" << NewLine;
}

void usage() {
	header();
	piCout << Green << Bold << "Details:";
	piCout << Bold << "Debug control";
	piCout << "-h                " << Green << "- display this message and exit";
	piCout << "-H                " << Green << "- display details help";
	piCout << "-q                " << Green << "- quiet, no debug output to console";
	piCout << "";
	piCout << Bold << "Output control";
	piCout << "-l <lang>         " << Green << "- translation language (e.g. en_US, ru_RU)";
	piCout << "-n, --no-obsolete " << Green << "- drop unused translations in output file";
	piCout << "-o <output_file>  " << Green << "- output file for translation (QtLinguist *.ts file)";
	piCout << "";
	piCout << Bold << "Input control";
	piCout << "<file/dir>        " << Green << "- add file or dir translation";
}

void help() {
	header();
	piCout << help_string;
}

void printError(const PIString & msg) {
	std::cerr << msg.data() << std::endl;
}


PISet<PIString> strings;
PIMap<uint, PIString> locations;

PIString mask(const PIString & in) {
	static PIVector<PIPair<PIString, PIString>> map = {
		{"&",  "&amp;" },
		{"<",  "&lt;"  },
		{">",  "&gt;"  },
		{"'",  "&apos;"},
		{"\"", "&quot;"},
	};
	PIString ret = in;
	for (const auto & i: map)
		ret.replaceAll(i.first, i.second);
	return ret;
}

PIString unmask(const PIString & in) {
	static PIVector<PIPair<PIString, PIString>> map = {
		{"<",  "&lt;"  },
		{">",  "&gt;"  },
		{"'",  "&apos;"},
		{"\"", "&quot;"},
		{"&",  "&amp;" },
	};
	PIString ret = in;
	for (const auto & i: map)
		ret.replaceAll(i.second, i.first);
	return ret;
}

void addString(const PIString & s, const PIString & loc) {
	if (s.isEmpty()) return;
	strings << s;
	locations[s.hash()] = loc;
}

void gatherStrings(const PIValueTree & vt, const PIString & loc) {
	const static PIStringList attrs({Attribute::prefix, Attribute::suffix});
	for (const auto & c: vt.children()) {
		addString(c.name(), loc);
		addString(c.comment(), loc);
		for (const auto & a: attrs) {
			if (c.attributes().contains(a)) {
				addString(c.attributes().value(a).toString(), loc);
			}
		}
		if (!c.isArray()) gatherStrings(c, loc);
	}
}

const PIString context = "QAD::PIValueTreeEdit";

struct TSMessage {
	PIString source;
	PIString translation;
	PIString type;
	PIString filename;
	PIString line;
};

PIMap<PIString, TSMessage> readTS(const PIString & path) {
	PIMap<PIString, TSMessage> ret;
	PIFile f(path, PIIODevice::ReadOnly);
	if (!f.isOpened()) return ret;
	PIIOTextStream ts(&f);
	TSMessage msg;
	int phase = 0;
	while (!ts.isEnd()) {
		auto line = ts.readLine().trim();
		switch (phase) {
		case 0:
			if (line == "<context>") phase = 1;
			break;
		case 1:
			if (line.startsWith("<name>")) {
				line.cutLeft(6).cutRight(7);
				if (line == context) phase = 2;
			}
			break;
		case 2:
			if (line == "<message>") {
				msg   = {};
				phase = 3;
			}
			break;
		case 3:
			if (line == "</message>") {
				ret[msg.source] = msg;
				phase           = 2;
			} else if (line.startsWith("<source>")) {
				line.cutLeft(8).cutRight(9);
				msg.source = unmask(line);
			} else if (line.startsWith("<location")) {
				PIString trs = line.takeRange('<', '>').cutLeft(8);
				while (trs.isNotEmpty()) {
					PIString t = trs.takeCWord();
					trs.cutLeft(1);
					PIString v = trs.takeRange('\"', '\"');
					if (t == "filename") msg.filename = v;
					if (t == "line") msg.line = v;
					if (trs.size_s() <= 2) break;
				}
			} else if (line.startsWith("<translation")) {
				PIString trs = line.takeRange('<', '>').cutLeft(11);
				while (trs.isNotEmpty()) {
					PIString t = trs.takeCWord();
					trs.cutLeft(1);
					PIString v = trs.takeRange('\"', '\"');
					if (t == "type") msg.type = v;
				}
				line.cutRight(14);
				msg.translation = unmask(line);
			}
			break;
		}
		if (line == "</context>") phase = 0;
	}
	return ret;
}


int main(int argc, char * argv[]) {
	PICLI cli(argc, argv);
	// piCout << cli.rawArguments();
	cli.setOptionalArgumentsCount(-1);
	cli.addArgument("output", true);
	cli.addArgument("language", true);
	cli.addArgument("help");
	cli.addArgument("Help");
	cli.addArgument("quiet");
	cli.addArgument("no-obsolete");
	if (cli.hasArgument("Help")) {
		help();
		return 0;
	}
	if (cli.hasArgument("help") || cli.argumentValue("output").isEmpty() || cli.argumentValue("language").isEmpty() ||
		cli.optionalArguments().isEmpty()) {
		usage();
		return 0;
	}
	piDebug           = !cli.hasArgument("quiet");
	PIString out_path = cli.argumentValue("output");

	PIStringList files;
	const static PIStringList ext({"conf", "json", "bin"});
	for (const PIString & a: cli.optionalArguments()) {
		if (PIDir::isExists(a)) {
			auto dl = PIDir(a).allEntries();
			for (const auto & i: dl) {
				if (!i.isFile()) continue;
				if (ext.contains(i.extension().toLowerCase().trim())) files << i.path;
			}
		} else {
			if (PIFile::isExists(a)) files << a;
		}
	}


	piCout << Cyan << Bold << "Read" << files.size_s() << "files ...";
	PIDir out_dir(PIFile::FileInfo(out_path).dir());
	for (const auto & p: files) {
		PIString ext = PIFile::FileInfo(p).extension().toLowerCase().trim();
		PIFile f(p, PIIODevice::ReadOnly);
		if (ext == "conf") gatherStrings(PIValueTreeConversions::fromText(PIString::fromUTF8(f.readAll())), out_dir.relative(p));
		if (ext == "json")
			gatherStrings(PIValueTreeConversions::fromJSON(PIJSON::fromJSON(PIString::fromUTF8(f.readAll()))), out_dir.relative(p));
		if (ext == "bin") gatherStrings(piDeserialize<PIValueTree>(f.readAll()), PIString());
	}
	piCout << Cyan << Bold << "Reading done";


	piCout << Cyan << Bold << "Reading ts file ...";

	auto old = readTS(out_path);

	piCout << Cyan << Bold << "Reading done";


	piCout << Cyan << Bold << "Writing ts file ...";

	PIFile outf(out_path, PIIODevice::ReadWrite);
	if (!outf.isOpened()) {
		printError("Can`t open \"" + outf.path() + "\"!");
		return 1;
	}
	outf.clear();
	PIIOTextStream ts(&outf);
	auto writeTSMessage = [&ts](const PIString & s, const TSMessage & m) {
		ts << "  <message>\n";
		ts << "    <source>" << mask(s) << "</source>\n";
		if (m.filename.isNotEmpty()) {
			ts << "    <location filename=\"" << m.filename << "\"";
			if (m.line.isNotEmpty()) ts << " line=\"" << m.line << "\"";
			ts << "/>\n";
		}
		ts << "    <translation";
		if (m.source.isEmpty()) {
			ts << " type=\"unfinished\"";
		} else {
			if (m.type.isNotEmpty()) ts << " type=\"" << m.type << "\"";
		}
		ts << ">" << mask(m.translation) << "</translation>\n";
		ts << "  </message>\n";
	};
	ts << "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n";
	ts << "<!DOCTYPE TS>\n";
	ts << "<TS version=\"2.1\" language=\"" << cli.argumentValue("language") << "\">\n";
	ts << "<context>\n";
	ts << "  <name>" << context << "</name>\n";
	for (const auto & s: strings) {
		TSMessage m = old.value(s);
		m.filename  = locations.value(s.hash());
		writeTSMessage(s, m);
		old.remove(s);
	}
	if (!cli.hasArgument("no-obsolete")) {
		for (const auto & i: old) {
			writeTSMessage(i.first, i.second);
		}
	}
	ts << "</context>\n";
	ts << "</TS>\n";

	piCout << Cyan << Bold << "Writing done";


	return 0;
}