kopia lustrzana https://gitlab.com/tomaszg/geostat
Initial support for handling OC database snapshots storing it in a local sqlite file.
Tomasz Golinski
rodzic
9ab6c3271a
commit
26d2f9ffb4
|
|
@ -0,0 +1,112 @@
|
|||
#include "debug.h"
|
||||
#include "ocdb.h"
|
||||
#include "okapi.h"
|
||||
|
||||
#include <string>
|
||||
#include <fstream>
|
||||
// #include <filesystem>
|
||||
#include <sqlite3.h>
|
||||
#include <unistd.h>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
const std::string Database = "ocpl.sqlite";
|
||||
const std::string Dump_path = "/home/tomaszg/src/geo/dump_mini/";
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
|
||||
int o;
|
||||
while ((o = getopt(argc, argv, "d:h?")) != -1)
|
||||
switch (o) {
|
||||
case 'd':
|
||||
try {
|
||||
if (std::stoi(optarg) > 0) {
|
||||
// Debug(1) << "Setting debug level to " << optarg;
|
||||
Debug::set_debug_level(std::stoi(optarg));
|
||||
}
|
||||
}
|
||||
catch (...) {
|
||||
std::cout << "Option \"-d\" requires a valid number as an argument\n";
|
||||
std::exit(EXIT_FAILURE);
|
||||
}
|
||||
break;
|
||||
// case 'g':
|
||||
// gpx_file = optarg;
|
||||
// break;
|
||||
// case 'p':
|
||||
// use_ocpl = 1;
|
||||
// break;
|
||||
// case 'u':
|
||||
// ocpl_user = optarg;
|
||||
// break;
|
||||
// case 'M':
|
||||
// show_minmax = 1;
|
||||
// break;
|
||||
// case 'D':
|
||||
// show_dist = 1;
|
||||
// break;
|
||||
// case 'O':
|
||||
// show_owners = 1;
|
||||
// break;
|
||||
// case 'Y':
|
||||
// show_types = 1;
|
||||
// break;
|
||||
// case 'H':
|
||||
// heat_file = optarg;
|
||||
// break;
|
||||
// case 's':
|
||||
// try {
|
||||
// if (std::stoi(optarg) > 0) {
|
||||
// heat_stamp_size = std::stoi(optarg);
|
||||
// }
|
||||
// }
|
||||
// catch (...) {
|
||||
// std::cout << "Option \"-s\" requires a valid number as an argument\n";
|
||||
// std::exit(EXIT_FAILURE);
|
||||
// }
|
||||
// break;
|
||||
// case 'm':
|
||||
// heat_map = optarg;
|
||||
// break;
|
||||
// case 'e':
|
||||
// heat_exp = 1;
|
||||
// break;
|
||||
// case 'L':
|
||||
// show_list = 1;
|
||||
// break;
|
||||
// case 'T':
|
||||
// show_dt = 1;
|
||||
// break;
|
||||
case 'h':
|
||||
case '?':
|
||||
default:
|
||||
std::cout << "Usage: ???\n";
|
||||
// std::cout << "Generate stats from Opencaching data or GPX files.\n\n";
|
||||
// std::cout << " * Data source:\n";
|
||||
// std::cout << "\t-p\tuse Opencaching.pl\n";
|
||||
// std::cout << "\t-u user\tuser for Opencaching\n";
|
||||
// std::cout << "\t-g file\tuse specified gpx file\n";
|
||||
// std::cout << " * Output:\n";
|
||||
// std::cout << "\t-M\tprint geographically extreme caches\n";
|
||||
// std::cout << "\t-D\tprint furthest and closest caches\n";
|
||||
// std::cout << "\t-H file\trender a heat map to a file\n";
|
||||
// std::cout << "\t-s n\tstamp size for a heat map (default = 15)\n";
|
||||
// std::cout << "\t-e\tuse exponential to flatten the heat map\n";
|
||||
// std::cout << "\t-m map\tchosen map: Poland, Poland_relief, Poland_big, Europe or a name of voivodeship (default = Poland)\n";
|
||||
// std::cout << "\t-L\tprint all caches\n";
|
||||
// std::cout << "\t-T\tprint D/T matrix\n";
|
||||
// std::cout << "\t-O\tprint owner count for found caches\n";
|
||||
// std::cout << "\t-Y\tprint types count for found caches\n";
|
||||
// std::cout << "\t-h\tdisplay this help screen\n";
|
||||
// std::exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
OCdb db(Database);
|
||||
db.init(Dump_path);
|
||||
|
||||
// std::string ocpl_url = "https://opencaching.pl/okapi/";
|
||||
// #include "config_user.h"
|
||||
// Okapi OCpl(ocpl_url, ocpl_key);
|
||||
// db.update(OCpl);
|
||||
}
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
project('geostat', 'cpp', default_options : ['cpp_std=c++2a'])
|
||||
curl_dep = dependency('libcurl')
|
||||
json_dep = dependency('nlohmann_json')
|
||||
sqlite_dep = dependency('sqlite3')
|
||||
#tinyxml_dep = dependency('tinyxml2')
|
||||
|
||||
magick_dep = dependency('Magick++', required : false)
|
||||
|
|
@ -15,8 +16,10 @@ endif
|
|||
|
||||
link = ['-lgpx', '-lheatmap']
|
||||
src = ['geostat.cpp', 'okapi.cpp', 'gpx.cpp', 'cache.cpp', 'debug.cpp', 'heat.cpp']
|
||||
src_db = ['geodb.cpp', 'debug.cpp', 'ocdb.cpp', 'okapi.cpp']
|
||||
|
||||
executable('geostat', src, dependencies : [curl_dep, json_dep, magick_dep], link_args: link)
|
||||
executable('geodb', src_db, dependencies : [sqlite_dep, json_dep, curl_dep])
|
||||
|
||||
configure_file(input: 'Poland.png', output: 'Poland.png', copy: true)
|
||||
configure_file(input: 'Poland_big.png', output: 'Poland_big.png', copy: true)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,243 @@
|
|||
#include "ocdb.h"
|
||||
#include "debug.h"
|
||||
#include "okapi.h"
|
||||
|
||||
#include <string>
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <sqlite3.h>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
using json = nlohmann::json;
|
||||
// using namespace std::literals::string_literals;
|
||||
|
||||
OCdb::OCdb(std::string db_file) {
|
||||
int res;
|
||||
|
||||
res = sqlite3_open(db_file.c_str(), &db);
|
||||
if (res != SQLITE_OK) {
|
||||
Debug(1) << sqlite3_errmsg(db);
|
||||
throw 1;
|
||||
}
|
||||
if (!request("CREATE TABLE IF NOT EXISTS revision (revision INTEGER PRIMARY KEY);") ||
|
||||
!request("CREATE TABLE IF NOT EXISTS caches (code TEXT PRIMARY KEY, name TEXT, location TEXT, type TEXT, status TEXT, size TEXT, difficulty REAL, terrain REAL, country TEXT, region TEXT, owner TEXT);") ||
|
||||
!request("CREATE TABLE IF NOT EXISTS logs (uuid TEXT PRIMARY KEY, cache_code TEXT, date TEXT, user TEXT, type TEXT);"))
|
||||
throw 1;
|
||||
// TODO: set revision
|
||||
}
|
||||
|
||||
OCdb::~OCdb() {
|
||||
request("UPDATE revision SET revision = " + std::to_string(revision) + ';');
|
||||
sqlite3_close(db);
|
||||
}
|
||||
|
||||
bool OCdb::request(std::string req) {
|
||||
int res;
|
||||
|
||||
res = sqlite3_prepare_v2(db, req.c_str(), req.length() + 1, &stmt, NULL);
|
||||
if (res != SQLITE_OK) {
|
||||
Debug(1) << "Request \"" << req << "\" failed:\n" << sqlite3_errmsg(db);
|
||||
return 0;
|
||||
}
|
||||
res = sqlite3_step(stmt);
|
||||
if (res != SQLITE_DONE) {
|
||||
Debug(1) << "Request \"" << req << "\" failed:\n" << sqlite3_errmsg(db);
|
||||
return 0;
|
||||
}
|
||||
sqlite3_finalize(stmt);
|
||||
return 1;
|
||||
}
|
||||
|
||||
bool OCdb::init(std::string dump_path) {
|
||||
request("DELETE FROM caches;");
|
||||
request("DELETE FROM logs;");
|
||||
request("DELETE FROM revision;");
|
||||
|
||||
std::ifstream file(dump_path + "index.json");
|
||||
json j;
|
||||
file >> j;
|
||||
revision = j["revision"];
|
||||
|
||||
request("INSERT INTO revision VALUES (" + std::to_string(revision) + ");");
|
||||
|
||||
for (auto& el : j["data_files"].items()) {
|
||||
init_part(dump_path + el.value().get<std::string>());
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
bool OCdb::init_part(std::string json_file) {
|
||||
Debug(1) << "Processing file: " << json_file << '\n';
|
||||
|
||||
std::ifstream file(json_file);
|
||||
json j;
|
||||
file >> j;
|
||||
|
||||
Debug(3) << "File: " << json_file << " read.\n";
|
||||
for (auto& el : j.items()) {
|
||||
parse_item(el.value());
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
bool OCdb::update(Okapi& oc) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
bool OCdb::parse_item(json j) {
|
||||
if (j.count("object_type") > 0 && j["object_type"] == "geocache") {
|
||||
if (j.count("change_type") > 0 && j["change_type"] == "replace") {
|
||||
Debug(2) << "Inserting/updating cache " << j["object_key"]["code"].get<std::string>() << ".\n";
|
||||
// if (j["object_key"]["code"] != j["data"]["code"]) {
|
||||
// Debug(1) << "Code change " << j["object_key"]["code"] << " -> " << j["data"]["code"] <<'\n';
|
||||
// }
|
||||
update_cache(j);
|
||||
}
|
||||
else if (j.count("change_type") > 0 && j["change_type"] == "delete") {
|
||||
Debug(1) << "Deleting cache " << j["object_key"]["code"].get<std::string>() << ".\n";
|
||||
std::string sql = "DELETE FROM caches WHERE code='" + j["object_key"]["code"].get<std::string>() + "';";
|
||||
request(sql);
|
||||
}
|
||||
else {
|
||||
Debug(1) << "Incorrect change type: " << j["change_type"] << ".\n";
|
||||
}
|
||||
}
|
||||
else if (j.count("object_type") > 0 && j["object_type"] == "log") {
|
||||
if (j["change_type"] == "replace") {
|
||||
Debug(2) << "Updating log " << j["object_key"]["uuid"] << ".\n";
|
||||
update_log(j);
|
||||
}
|
||||
else if (j.count("change_type") > 0 && j["change_type"] == "delete") {
|
||||
Debug(2) << "Deleting log " << j["object_key"]["uuid"] << ".\n";
|
||||
std::string sql = "DELETE FROM logs WHERE uuid='" + j["object_key"]["uuid"].get<std::string>() + "';";
|
||||
request(sql);
|
||||
}
|
||||
else {
|
||||
Debug(1) << "Incorrect change type: " << j["change_type"] << ".\n";
|
||||
}
|
||||
}
|
||||
else {
|
||||
Debug(1) << "Incorrect object type: " << j["object_type"] << ".\n";
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
bool OCdb::update_cache(json j) {
|
||||
// (code TEXT PRIMARY KEY, name TEXT, location TEXT, type TEXT, status TEXT, size TEXT, difficulty REAL, terrain REAL, country TEXT, region TEXT, owner TEXT)
|
||||
std::map<std::string, std::string> fields;
|
||||
std::map<std::string, float> fields2;
|
||||
int res;
|
||||
|
||||
std::string code = j["object_key"]["code"].get<std::string>();
|
||||
if (j["data"]["names"].count("pl") > 0 && !j["data"]["names"]["pl"].is_null())
|
||||
fields["name"] = j["data"]["names"]["pl"].get<std::string>();
|
||||
else if (j["data"]["names"].count("en") > 0 && !j["data"]["names"]["en"].is_null())
|
||||
fields["name"] = j["data"]["names"]["en"].get<std::string>();
|
||||
if (j["data"].count("location") > 0 && !j["data"]["location"].is_null())
|
||||
fields["location"] = j["data"]["location"].get<std::string>();
|
||||
if (j["data"].count("type") > 0 && !j["data"]["type"].is_null())
|
||||
fields["type"] = j["data"]["type"].get<std::string>();
|
||||
if (j["data"].count("status") > 0 && !j["data"]["status"].is_null())
|
||||
fields["status"] = j["data"]["status"].get<std::string>();
|
||||
if (j["data"].count("size2") > 0 && !j["data"]["size2"].is_null())
|
||||
fields["size"] = j["data"]["size2"].get<std::string>();
|
||||
if (j["data"].count("difficulty") > 0 && !j["data"]["difficulty"].is_null())
|
||||
fields2["difficulty"] = j["data"]["difficulty"].get<float>();
|
||||
if (j["data"].count("terrain") > 0 && !j["data"]["terrain"].is_null())
|
||||
fields2["terrain"] = j["data"]["terrain"].get<float>();
|
||||
if (j["data"].count("country") > 0 && !j["data"]["country"].is_null())
|
||||
fields["country"] = j["data"]["country"].get<std::string>();
|
||||
if (j["data"].count("region") > 0 && !j["data"]["region"].is_null())
|
||||
fields["region"] = j["data"]["region"].get<std::string>();
|
||||
else if (j["data"].count("state") > 0 && !j["data"]["state"].is_null())
|
||||
fields["region"] = j["data"]["state"].get<std::string>();
|
||||
if (j["data"]["owner"].count("uuid") > 0 && !j["data"]["owner"]["uuid"].is_null())
|
||||
fields["owner"] = j["data"]["owner"]["uuid"].get<std::string>();
|
||||
|
||||
std::string sql = "INSERT INTO caches (code,";
|
||||
for (auto& i : fields)
|
||||
sql += i.first + ',';
|
||||
for (auto& i : fields2)
|
||||
sql += i.first + ',';
|
||||
sql.pop_back();
|
||||
sql += ") VALUES ('" + code + "',";
|
||||
for (auto& i : fields)
|
||||
sql += "?,";
|
||||
for (auto& i : fields2)
|
||||
sql += "?,";
|
||||
sql.pop_back();
|
||||
sql += ") ON CONFLICT(code) DO UPDATE SET ";
|
||||
for (auto& i : fields)
|
||||
sql += i.first + "=excluded." + i.first + ',';
|
||||
for (auto& i : fields2)
|
||||
sql += i.first + "=excluded." + i.first + ',';
|
||||
sql.pop_back();
|
||||
sql += ';';
|
||||
|
||||
res = sqlite3_prepare_v2(db, sql.c_str(), sql.length() + 1, &stmt, NULL);
|
||||
if (res != SQLITE_OK) {
|
||||
Debug(1) << "Request \"" << sql << "\" failed:\n" << sqlite3_errmsg(db);
|
||||
return 0;
|
||||
}
|
||||
int n = 1;
|
||||
for (auto& i : fields) {
|
||||
sqlite3_bind_text(stmt, n++, i.second.c_str(), -1, nullptr);
|
||||
}
|
||||
for (auto& i : fields2) {
|
||||
sqlite3_bind_double(stmt, n++, i.second);
|
||||
}
|
||||
res = sqlite3_step(stmt);
|
||||
if (res != SQLITE_DONE) {
|
||||
Debug(1) << "Request \"" << sql << "\" failed:\n" << sqlite3_errmsg(db);
|
||||
return 0;
|
||||
}
|
||||
sqlite3_finalize(stmt);
|
||||
return 1;
|
||||
}
|
||||
|
||||
bool OCdb::update_log(json j) {
|
||||
// logs (uuid TEXT PRIMARY KEY, cache_code TEXT, date TEXT, user TEXT, type TEXT);"))
|
||||
std::map<std::string, std::string> fields;
|
||||
int res;
|
||||
|
||||
std::string uuid = j["object_key"]["uuid"].get<std::string>();
|
||||
if (j["data"].count("cache_code") > 0 && !j["data"]["cache_code"].is_null())
|
||||
fields["cache_code"] = j["data"]["cache_code"].get<std::string>();
|
||||
if (j["data"].count("date") > 0 && !j["data"]["date"].is_null())
|
||||
fields["date"] = j["data"]["date"].get<std::string>();
|
||||
if (j["data"]["user"].count("uuid") > 0 && !j["data"]["user"]["uuid"].is_null())
|
||||
fields["user"] = j["data"]["user"]["uuid"].get<std::string>();
|
||||
if (j["data"].count("text") > 0 && !j["data"]["text"].is_null())
|
||||
fields["text"] = j["data"]["text"].get<std::string>();
|
||||
|
||||
std::string sql = "INSERT INTO logs (uuid,";
|
||||
for (auto& i : fields)
|
||||
sql += i.first + ',';
|
||||
sql.pop_back();
|
||||
sql += ") VALUES ('" + uuid + "',";
|
||||
for (auto& i : fields)
|
||||
sql += "?,";
|
||||
sql.pop_back();
|
||||
sql += ") ON CONFLICT(uuid) DO UPDATE SET ";
|
||||
for (auto& i : fields)
|
||||
sql += i.first + "=excluded." + i.first + ',';
|
||||
sql.pop_back();
|
||||
sql += ';';
|
||||
|
||||
res = sqlite3_prepare_v2(db, sql.c_str(), sql.length() + 1, &stmt, NULL);
|
||||
if (res != SQLITE_OK) {
|
||||
Debug(1) << "Request \"" << sql << "\" failed:\n" << sqlite3_errmsg(db);
|
||||
return 0;
|
||||
}
|
||||
int n = 1;
|
||||
for (auto& i : fields) {
|
||||
sqlite3_bind_text(stmt, n++, i.second.c_str(), -1, nullptr);
|
||||
}
|
||||
res = sqlite3_step(stmt);
|
||||
if (res != SQLITE_DONE) {
|
||||
Debug(1) << "Request \"" << sql << "\" failed:\n" << sqlite3_errmsg(db);
|
||||
return 0;
|
||||
}
|
||||
sqlite3_finalize(stmt);
|
||||
return 1;
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <nlohmann/json_fwd.hpp>
|
||||
|
||||
typedef struct sqlite3 sqlite3;
|
||||
typedef struct sqlite3_stmt sqlite3_stmt;
|
||||
class Okapi;
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
class OCdb {
|
||||
private:
|
||||
sqlite3* db;
|
||||
sqlite3_stmt* stmt;
|
||||
|
||||
bool request(std::string req);
|
||||
bool init_part(std::string json_file); // read db dump
|
||||
bool parse_item(json j);
|
||||
bool update_cache(json j);
|
||||
bool update_log(json j);
|
||||
public:
|
||||
int revision;
|
||||
|
||||
OCdb(std::string db_file);
|
||||
~OCdb();
|
||||
|
||||
bool init(std::string json_file); // read db dump
|
||||
bool update(Okapi& oc); // apply changelog
|
||||
};
|
||||
Ładowanie…
Reference in New Issue