geostat/geostat.cpp

#include "okapi.h"
#include "gpx.h"
#include "cache.h"
#include "debug.h"
#include "heat.h"
#include "ocdb.h"

#include <iostream>
#include <iomanip>
#include <algorithm>
#include <unistd.h>
#include <map>

const int HIST_MAX = 20;

void show_usage() {
	std::cout << "Usage: geostat [-o] [-g file] [-qn] [-MDHLTOYh] [-s size] [-m map]\n";
	std::cout << "Generate stats from Opencaching data or GPX files.\n\n";
	std::cout << " * Data sources:\n";
	std::cout << "\t-o[user]\tuse Opencaching; if specified user will be a user name used for all OC instances\n";
	std::cout << "\t-p user\t\tuser for opencaching.pl\n";
	std::cout << "\t-d user\t\tuser for opencaching.de\n";
	std::cout << "\t-u user\t\tuser for opencaching.us\n";
	std::cout << "\t-n user\t\tuser for opencaching.nl\n";
	std::cout << "\t-r user\t\tuser for opencaching.ro\n";
	std::cout << "\t-k user\t\tuser for opencaching.uk\n";
	std::cout << "\t-g file\t\tuse specified gpx file\n";
	std::cout << "\t-q\t\tuse local SQLite file with dump of OC database\n";
	std::cout << " * Output:\n";
	std::cout << "\t-W\t\toutput all data in HTML format\n";
	std::cout << "\t-N\t\tcompute stats only for unfound caches (works only with SQLite)\n";
	std::cout << "\t-M\t\tprint geographically extreme caches\n";
	std::cout << "\t-D\t\tprint furthest and closest caches\n";
	std::cout << "\t-H file\t\trender a heat map to a file\n";
	std::cout << "\t-s n\t\tstamp size for a heat map (default = 15)\n";
	std::cout << "\t-e\t\tuse exponential to flatten the heat map\n";
	std::cout << "\t-m map\t\tchosen map: Poland, Poland_relief, Poland_big, Europe or a name of voivodeship (default = Poland)\n";
	std::cout << "\t-L\t\tprint all caches\n";
	std::cout << "\t-T\t\tprint D/T matrix\n";
	std::cout << "\t-O\t\tprint owner count for found caches\n";
	std::cout << "\t-Y\t\tprint types count for found caches\n";
	std::cout << "\t-h\t\tdisplay this help screen\n";
	std::exit(EXIT_FAILURE);
}

void show_histogram(Caches* cc, std::string Cache::*ptr, std::string caption, bool html = 0, bool sort_by_val = 1) {
	std::map<std::string, int> histogram;
	std::vector<std::pair<std::string, int>> pairs;

	for (auto el : *cc)
		histogram[el.*ptr]++;
	for (auto el : histogram)
		pairs.push_back(el);
	if (sort_by_val)
		sort(pairs.begin(), pairs.end(), [&](std::pair<std::string, int>& a, std::pair<std::string, int>& b) { return a.second > b.second; });
	else
		sort(pairs.begin(), pairs.end(), [&](std::pair<std::string, int>& a, std::pair<std::string, int>& b) { return a.first < b.first; });

	if (html) {
		int max;
		if (sort_by_val)
			max = pairs[0].second;
		else
			max = std::max_element(pairs.begin(), pairs.end(), [&](std::pair<std::string, int>& a, std::pair<std::string, int>& b) { return a.second < b.second; })->second;
		int i = 0;
		std::string::size_type n;
		std::cout << "<h2>" << caption << "</h2>\n";
		std::cout << "<dl class=\"histogram\">\n";
		for (auto own : pairs) {
			if (own.first.empty()) own.first = "[unknown]";
			while ((n = own.first.find('&', n+1)) != std::string::npos)
				own.first.replace(n, 1, "&amp;");

			if (i < HIST_MAX)
				std::cout << "<dd class=\"bar\" style=\"--percent: " << 100 * own.second / max << "%;\"><span class=\"text\">" << own.first << ": " << own.second << "</span></dd>\n";
			else if (i == HIST_MAX) {
				std::cout << "</dl>\n";
				std::cout << "<div class=\"histogram_others\">Other: " << own.first;
			}
			if (i > HIST_MAX)
				std::cout << ", " << own.first;
			i++;
		}
		if (i < HIST_MAX)
			std::cout << "</dl>\n";
		else
			std::cout << "</div>\n";
	} else {
		std::cout << caption << '\n';
		for (auto own : pairs)
			std::cout << own.first << ": " << own.second << '\n';
	}
}

bool same_day(const std::tm& a, const std::tm& b) {
	return (a.tm_year == b.tm_year && a.tm_mon == b.tm_mon && a.tm_mday == b.tm_mday);
}

void next_day(std::tm& a) {
	a.tm_mday++;
	mktime(&a);
}

int find_streak(const std::multimap<std::time_t, const Cache*>& cc, std::tm& start) {
	int max_str = 0;
	int cur_str = 0;

	std::tm cur_tm;
	cur_tm.tm_year = 0;
	std::tm cur_start_tm = cur_tm;
	start = cur_tm;

	for (auto i : cc) {
		if (same_day(cur_tm, *std::localtime(&i.first)))
			continue;
		next_day(cur_tm);
		if (same_day(cur_tm, *std::localtime(&i.first))) {
			cur_str++;
			continue;
		}
		if (max_str < cur_str) {
			max_str = cur_str;
			start = cur_start_tm;
		}
		cur_str = 1;
		cur_tm = cur_start_tm = *std::localtime(&i.first);
	}
	return max_str;
}

int main(int argc, char** argv) {
	bool show_minmax = 0;
	bool show_dist = 0;
	bool show_list = 0;
	bool show_dt = 0;
	bool show_owners = 0;
	bool show_types = 0;
	bool show_calendar = 0;
	bool show_html = 0;
	std::string heat_file;
	int heat_stamp_size = 15;
	std::string heat_map = "Poland";
	bool heat_exp = 0;
	bool use_oc = 0;
	bool use_ocpl_db = 0;
	bool get_not_found = 0;
	std::string ocpl_user;
	std::string ocde_user;
	std::string ocus_user;
	std::string ocnl_user;
	std::string ocro_user;
	std::string ocuk_user;
	std::string ocpl_user_profile;
	std::string ocde_user_profile;
	std::string ocus_user_profile;
	std::string ocnl_user_profile;
	std::string ocro_user_profile;
	std::string ocuk_user_profile;
	std::string ocpl_url = "https://opencaching.pl/okapi/";
	std::string ocde_url = "https://www.opencaching.de/okapi/";
	std::string ocus_url = "http://www.opencaching.us/okapi/";
	std::string ocnl_url = "http://www.opencaching.nl/okapi/";
	std::string ocro_url = "http://www.opencaching.ro/okapi/";
	std::string ocuk_url = "https://opencache.uk/okapi/";
	const std::string Database = "ocpl.sqlite";
	std::string gpx_file;

#include "config_user.h"

	if (argc == 1) show_usage();

	int o;
	while ((o = getopt(argc, argv, "qNg:o::p:d:u:n:r:k:MDOH:s:m:eLTYWh?")) != -1)
		switch (o) {
// 			case 'd':
// 				try {
// 					if (std::stoi(optarg) > 0) {
// 						// Debug(1) << "Setting debug level to " << optarg;
// 						Debug::set_debug_level(std::stoi(optarg));
// 					}
// 				}
// 				catch (...) {
// 					std::cout << "Option \"-d\" requires a valid number as an argument\n";
// 					std::exit(EXIT_FAILURE);
// 				}
// 				break;
			case 'g':
				gpx_file = optarg;
				break;
			case 'o':
				use_oc = 1;
				if (optarg) {
					ocpl_user = optarg;
					ocde_user = optarg;
					ocus_user = optarg;
					ocnl_user = optarg;
					ocro_user = optarg;
					ocuk_user = optarg;
				}
				break;
			case 'p':
				ocpl_user = optarg;
				break;
			case 'd':
				ocde_user = optarg;
				break;
			case 'u':
				ocus_user = optarg;
				break;
			case 'n':
				ocnl_user = optarg;
				break;
			case 'r':
				ocro_user = optarg;
				break;
			case 'k':
				ocuk_user = optarg;
				break;
			case 'q':
				use_ocpl_db = 1;
				break;
			case 'N':
				get_not_found = 1;
				break;
			case 'M':
				show_minmax = 1;
				break;
			case 'D':
				show_dist = 1;
				break;
			case 'O':
				show_owners = 1;
				break;
			case 'Y':
				show_types = 1;
				break;
			case 'W':
				show_html = 1;
				break;
			case 'H':
				heat_file = optarg;
				break;
			case 's':
				try {
					if (std::stoi(optarg) > 0) {
						heat_stamp_size = std::stoi(optarg);
					}
				}
				catch (...) {
					std::cout << "Option \"-s\" requires a valid number as an argument\n";
					std::exit(EXIT_FAILURE);
				}
				break;
			case 'm':
				heat_map = optarg;
				break;
			case 'e':
				heat_exp = 1;
				break;
			case 'L':
				show_list = 1;
				break;
			case 'T':
				show_dt = 1;
				break;
			case 'h':
			case '?':
			default:
				show_usage();
		}

	if (show_html) {
		show_dist = 0;
		show_list = 0;
		show_dt = 0;
		show_owners = 0;
		show_types = 0;
	}

	Caches cc;

	if (use_oc) {
		if (!ocpl_user_uuid.empty() || !ocpl_user.empty()) {
			Okapi OCpl(ocpl_url, ocpl_key);
			if (!ocpl_user.empty()) ocpl_user_uuid = OCpl.get_uuid(ocpl_user);
			cc.merge(OCpl.get_user_caches(ocpl_user_uuid, 0));
			ocpl_user_profile = OCpl.get_profile_url(ocpl_user_uuid);
		}
		if (!ocde_user_uuid.empty() || !ocde_user.empty()) {
			Okapi OCde(ocde_url, ocde_key);
			if (!ocde_user.empty()) ocde_user_uuid = OCde.get_uuid(ocde_user);
			cc.merge(OCde.get_user_caches(ocde_user_uuid, 0));
		}
		if (!ocus_user_uuid.empty() || !ocus_user.empty()) {
			Okapi OCus(ocus_url, ocus_key);
			if (!ocus_user.empty()) ocus_user_uuid = OCus.get_uuid(ocus_user);
			cc.merge(OCus.get_user_caches(ocus_user_uuid, 0));
		}
		if (!ocnl_user_uuid.empty() || !ocnl_user.empty()) {
			Okapi OCnl(ocnl_url, ocnl_key);
			if (!ocnl_user.empty()) ocnl_user_uuid = OCnl.get_uuid(ocnl_user);
			cc.merge(OCnl.get_user_caches(ocnl_user_uuid, 0));
		}
		if (!ocro_user_uuid.empty() || !ocro_user.empty()) {
			Okapi OCro(ocro_url, ocro_key);
			if (!ocro_user.empty()) ocro_user_uuid = OCro.get_uuid(ocro_user);
			cc.merge(OCro.get_user_caches(ocro_user_uuid, 0));
		}
		if (!ocuk_user_uuid.empty() || !ocuk_user.empty()) {
			Okapi OCuk(ocuk_url, ocuk_key);
			if (!ocuk_user.empty()) ocuk_user_uuid = OCuk.get_uuid(ocuk_user);
			cc.merge(OCuk.get_user_caches(ocuk_user_uuid, 0));
		}
	}

	if (!gpx_file.empty()) {
		GPX gpxfile(gpx_file);
		cc.merge(gpxfile.get_user_caches());
	}

	if (use_ocpl_db) {
		OCdb db(Database);
		Okapi OCpl(ocpl_url, ocpl_key);
		if (!ocpl_user.empty()) {
			ocpl_user_uuid = OCpl.get_uuid(ocpl_user);
		}
		if (get_not_found) {
			cc.merge(db.get_user_caches_not_found(ocpl_user_uuid));
		} else {
			cc.merge(db.get_user_caches(ocpl_user_uuid, 0));
		}
	}

	// TODO: some cache deduplication is needed

	Debug(2) << "Caches read: " << cc.size() << '\n';
	if (cc.size() == 0) {
		std::cout << "No caches found, aborting.\n";
		std::exit(EXIT_FAILURE);
	}

	if (show_html) {
		std::cout << "<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML 4.01//EN\" \"http://www.w3.org/TR/html4/strict.dtd\">\n";
		std::cout << "<html>\n";
		std::cout << " <head>\n";
		std::cout << "  <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n";
		std::cout << "  <link rel=\"stylesheet\" type=\"text/css\" href=\"geo.css?ver=3\">\n";
		std::cout << "  <title>Geocaching stats</title>\n";
		std::cout << "</head>\n";
		std::cout << "<body>\n";
		std::cout << "<h1>Geocaching stats for user <a href=\"" << ocpl_user_profile << "\">" << ocpl_user << "</a></h1>\n";
	}

	if (!heat_file.empty()) {
		const Map* chosen_map;
		if (maps.count(heat_map) > 0)
			chosen_map = &maps.at(heat_map);
		else {
			std::cout << "Map " << heat_map << " not found.\n";
			std::exit(EXIT_FAILURE);
		}
		Heat hmap(&cc, chosen_map);
		hmap.generate(heat_file, heat_stamp_size, (heat_exp == 1 ? "exp" : "soft"));
		if (show_html)
			std::cout << "<img class=\"heatmap\" src=\"" << heat_file << "\" alt=\"heat map\">\n";
	}

	if (show_html && !get_not_found) {
		std::cout << "<div class=\"basic_stats\">\n";
		std::cout << "Number of caches found: <span class=\"value\">" << cc.size() << "</span><br>\n";
		std::map<std::string, int> dates;
		std::multimap<std::time_t, const Cache*> sorted_caches;

		std::tm tmp;
		for (auto& i : cc) {
			dates[i.date]++;
			if (i.type != "Moving" && i.type != "Own") {
				tmp = i.date_tm;
				sorted_caches.insert({std::mktime(&tmp), &i});
			}
		}
		auto best_day = std::max_element(dates.begin(), dates.end(), [&](auto& a, auto& b) { return a.second < b.second; });
		std::cout << "Number of caching days: <span class=\"value\">" << dates.size() << "</span><br>\n";
		std::cout << "Average caches per caching day: <span class=\"value\">" << std::setprecision(3) << (1.0 * cc.size()) / dates.size() << "</span><br>\n";
		std::cout << "Best caching day: <span class=\"value\">" << best_day->first << "</span>, found <span class=\"value\">" << best_day->second << "</span> caches\n";
		std::cout << "<span class=\"wrap-collabsible\">\n  <input id=\"collapsible\" class=\"toggle\" type=\"checkbox\">\n  <label for=\"collapsible\" class=\"lbl-toggle\">Details</label>\n  <div class=\"collapsible-content\">\n    <div class=\"content-inner\">\n";
		for (auto& i : dates)
			std::cout << i.first << ", found " << i.second << " caches<br>\n";
		std::cout << "   </div>\n  </div>\n</span>\n";

		std::tm streak;
		int longest_str = find_streak(sorted_caches, streak);
		char str_tmp[20];
		std::strftime(str_tmp, 20, "%F", &streak);
		std::cout << "Longest caching streak: <span class=\"value\">" << longest_str << "</span> starting on <span class=\"value\">" << str_tmp << "</span><br>\n";

		int tot_dist = 0;
		for (auto i = sorted_caches.begin(); i != std::prev(sorted_caches.end()); i++) {
// 			std::cout << "Distance between " << i->second->name << " and " << std::next(i)->second->name << " is " << cache_distance(*i->second, *std::next(i)->second) << "<br>";
			tot_dist += cache_distance(*i->second, *std::next(i)->second);
		}
		std::cout << "Total distance between caches: <span class=\"value\">" << tot_dist << "</span> km (equivalent to <span class=\"value\">" << tot_dist/(2 * Pi * Earth_radius) << "x</span> trips around Earth)<br>\n";
		std::cout << "Average distance between caches: <span class=\"value\">" << tot_dist / sorted_caches.size() << "</span> km<br>\n";
		std::cout << "</div>\n";
	}

	if (show_html && get_not_found) {
		std::cout << "<div class=\"basic_stats\">\n";
		std::cout << "Still <span class=\"value\">" << cc.size() << "</span> caches to be found...<br>\n";
		std::cout << "</div>\n";
	}

	if ((show_minmax || show_html) && !get_not_found) {
		auto N = std::max_element(cc.begin(), cc.end(), [&](const Cache& a, const Cache& b) { return a.pos.lat < b.pos.lat; });
		auto S = std::min_element(cc.begin(), cc.end(), [&](const Cache& a, const Cache& b) { return a.pos.lat < b.pos.lat; });
		auto E = std::max_element(cc.begin(), cc.end(), [&](const Cache& a, const Cache& b) { return a.pos.lon < b.pos.lon; });
		auto W = std::min_element(cc.begin(), cc.end(), [&](const Cache& a, const Cache& b) { return a.pos.lon < b.pos.lon; });

		if (show_minmax) {
			std::cout << "Most N:\n";
			N->show();
			std::cout << "Most S:\n";
			S->show();
			std::cout << "Most E:\n";
			E->show();
			std::cout << "Most W:\n";
			W->show();
		} else {
			std::cout << "<h2>Geographically extreme caches</h2>\n";
			std::cout << "<table class=\"minmax\">\n";
			std::cout << "<tr>\n";
			std::cout << "  <td>North:</td><td><a href=\"" << N->link() << "\">" << N->name << " (" << N->code << ")</a></td><td>" << N->pos.lat << "</td>\n";
			std::cout << "</tr><tr>\n";
			std::cout << "  <td>South:</td><td><a href=\"" << S->link() << "\">" << S->name << " (" << S->code << ")</a></td><td>" << S->pos.lat << "</td>\n";
			std::cout << "</tr><tr>\n";
			std::cout << "  <td>East:</td> <td><a href=\"" << E->link() << "\">" << E->name << " (" << E->code << ")</a></td><td>" << E->pos.lon << "</td>\n";
			std::cout << "</tr><tr>\n";
			std::cout << "  <td>West:</td> <td><a href=\"" << W->link() << "\">" << W->name << " (" << W->code << ")</a></td><td>" << W->pos.lon << "</td>\n";
			std::cout << "</tr>\n";
			std::cout << "</table>\n";
		}
	}

	if (show_dist && !get_not_found) {
		auto far = std::max_element(cc.begin(), cc.end(), [&](const Cache& a, const Cache& b) { return a.distance() < b.distance(); });
		auto near = std::min_element(cc.begin(), cc.end(), [&](const Cache& a, const Cache& b) { return a.distance() < b.distance(); });

		std::cout << "Nearest cache: " << near->distance() << " km\n";
		near->show();
		std::cout << "Furthest cache: " << far->distance() << " km\n";
		far->show();
	}

	if (show_list) {
		for (auto el : cc)
			el.show();
	}

	if ((show_owners || show_html) && !get_not_found) {
		show_histogram(&cc, &Cache::owner, "Cache owners", show_html);
	}

	if ((show_types || show_html) && !get_not_found) {
		show_histogram(&cc, &Cache::type, "Cache types", show_html);
		show_histogram(&cc, &Cache::size, "Cache sizes", show_html);
		if (!show_html) show_histogram(&cc, &Cache::origin, "Services", 0);
		show_histogram(&cc, &Cache::region, "Regions", show_html);
	}

	if ((show_calendar || show_html) && !get_not_found) {
		show_histogram(&cc, &Cache::day_of_week, "Days of the week", show_html, 0);
		show_histogram(&cc, &Cache::mon, "Months", show_html, 0);
		show_histogram(&cc, &Cache::year, "Years", show_html, 0);
	}

	if (show_dt && !get_not_found) {
		short dt_table[11][11];

		std::cout << std::setw(5) << "D\\T";
		for (int j = 2; j <= 10; j++) { // print table terr headers
			std::cout << std::setw(5) << j / 2.0;
		}
		std::cout << '\n';

		for (int i = 2; i <= 10; i++) { // i -> diff in rows
			std::cout << std::setw(5) << i / 2.0;
			for (int j = 2; j <= 10; j++) { // j -> terr in cols
				dt_table[i][j] = std::count_if(cc.begin(), cc.end(), [i, j](Cache c) { return (c.diff == i / 2.0 && c.terr == j / 2.0); });
				std::cout << std::setw(5) << dt_table[i][j];
			}
			std::cout << '\n';
		}
	}
	if (show_html && !get_not_found) {
		short dt_table[11][11];
		int n = 0;

		std::cout << "<h2>Difficulty / terrain matrix</h2>\n";
		std::cout << "<table class=\"dt\">\n";
		std::cout << "<tr class=\"dt_head\"><td></td>";
		for (int j = 2; j <= 10; j++) { // print table terr headers
			std::cout << "<td>" << j / 2.0 << "</td>";
		}
		std::cout << "</tr>\n";

		for (int i = 2; i <= 10; i++) { // i -> diff in rows
			std::cout << "<tr><td class=\"dt_head\">" << i / 2.0 << "</td> ";
			for (int j = 2; j <= 10; j++) { // j -> terr in cols
				dt_table[i][j] = std::count_if(cc.begin(), cc.end(), [i, j](Cache c) { return (c.diff == i / 2.0 && c.terr == j / 2.0); });
				if (dt_table[i][j] == 0)
					std::cout << "<td class=\"dt_zero\">" << dt_table[i][j] << "</td>";
				else {
					std::cout << "<td>" << dt_table[i][j] << "</td>";
					n++;
				}
			}
			std::cout << "</tr>\n";
		}
		std::cout << "</table>\n";
		std::cout << "<div class=\"basic_stats\">\n";
		std::cout << "Total <span class=\"value\">" << n << "</span> combinations found out of 81 (" << std::setprecision(3) << n/0.81 << "%).\n";
		std::cout << "</div>\n";

	}

	if (show_html) {
		std::cout << "<div id=\"footer\">\n";
		std::cout << "Stats generated by <a href=\"https://gitlab.com/tomaszg/geostat\">GeoStat</a>.<br>\n";
		std::cout << "Data obtained from Opencaching sites via <a href=\"https://github.com/opencaching/okapi/\">OKAPI</a>:<br>\n";
		std::cout << "<a href=\"https://www.opencaching.pl/\">Opencaching.pl</a>, ";
		std::cout << "<a href=\"https://www.opencaching.de/\">Opencaching.de</a>, ";
		std::cout << "<a href=\"http://www.opencaching.us/\">Opencaching.us</a>, ";
		std::cout << "<a href=\"http://www.opencaching.nl/\">Opencaching.nl</a>, ";
		std::cout << "<a href=\"http://www.opencaching.ro/\">Opencaching.ro</a>, ";
		std::cout << "<a href=\"https://www.opencache.uk/\">Opencaching.uk</a>.";
		std::cout << "</div>\n";

		std::cout << "</body>\n</html>\n";
	}
}