count.cpp - master - Boxapp/repa

count.cpp
101 строка3 KB
ArchAsus
implement counting UTF-8 characters
28 сен 2024, 18:14
28 сен 2024, 18:141acf384
#include <algorithm>
#include <clocale>
#include <codecvt>
#include <cstdint>
#include <fstream>
#include <iostream>
#include <locale>
#include <map>
#include <string>
#include <vector>

using std::cin;
using std::codecvt_utf8_utf16;
using std::cout;
using std::endl;
using std::ifstream;
using std::istream;
using std::map;
using std::pair;
using std::sort;
using std::string;
using std::vector;
using std::wstring;
using std::wstring_convert;

vector<pair<wchar_t, size_t>> mostFrequent(const wstring &text) {
	map<wchar_t, size_t> count;
	for (wchar_t c : text) {
		if (count.find(c) == count.end()) count[c] = 1;
		else ++count[c];
	}
	vector<pair<wchar_t, size_t>> res;
	for (const auto &unicode : count) res.push_back(unicode);
	sort(res.begin(), res.end(), [](const pair<wchar_t, size_t> &x, const pair<wchar_t, size_t> &y){
		if (x.second == y.second) return x.first < y.first;
		return x.second > y.second;
	});
	return res;
}

string readFile(istream &in) {
	string res;
	char c = '\0';
	while (in.get(c)) res += c;
	return res;
}
// NOTE https://mobiarch.wordpress.com/2022/12/03/playing-with-utf-8-in-c/
string utf8_encode(unsigned long code_point) {
	string str;
	if (code_point <= 0x007F) {
		char ch = static_cast<char>(code_point);
		str.push_back(ch);
	} else if (code_point <= 0x07FF) {
		uint8_t b2 = 0x80 | (code_point & 0x3F);
		uint8_t b1 = 0xC0 | (code_point >> 6);
		str.push_back(b1);
		str.push_back(b2);
	} else if (code_point <= 0xFFFF) {
		uint8_t b3 = 0x80 | (code_point & 0x3F);
		uint8_t b2 = 0x80 | ((code_point >> 6) & 0x3F);
		uint8_t b1 = 0xE0 | (code_point >> 12);
		str.push_back(b1);
		str.push_back(b2);
		str.push_back(b3);
	} else if (code_point <= 0x10FFFF) {
		uint8_t b4 = 0x80 | (code_point & 0x3F);
		uint8_t b3 = 0x80 | ((code_point >> 6) & 0x3F);
		uint8_t b2 = 0x80 | ((code_point >> 12) & 0x3F);
		uint8_t b1 = 0xF0 | (code_point >> 18);
		str.push_back(b1);
		str.push_back(b2);
		str.push_back(b3);
		str.push_back(b4);
	}
	return str;
}

int main(int argc, char **argv) {
	setlocale(LC_ALL, "");
	string input_file;
	if (argc > 1) {
		string cmd = argv[1];
		if (cmd == "--help") {
			cout << "Usage: " << argv[0] << " [--help | FILENAME]" << endl;
			return 0;
		} else input_file = cmd;
	}
	string text;
	if (input_file.empty()) text = readFile(cin);
	else {
		ifstream in(input_file);
		text = readFile(in);
	}
	cout << text.size() << " bytes" << endl;
	wstring unicode = wstring_convert<codecvt_utf8_utf16<wchar_t>, wchar_t>{}.from_bytes(text); // TODO what if charset is not UTF-8
	cout << unicode.size() << " characters" << endl;
	vector<pair<wchar_t, size_t>> freq = mostFrequent(unicode);
	cout << freq.size() << " distinct characters" << endl;
	for (const pair<wchar_t, size_t> &unicode : freq) cout << "'" << utf8_encode(unicode.first) << "' " << unicode.second << endl;
	return 0;
}