Preliminary support for native Habitat upload.

pull/6/head
Michal Fratczak 2018-11-08 00:13:48 +01:00
rodzic cedf33b690
commit 6be2d4d589
7 zmienionych plików z 692 dodań i 14 usunięć

Wyświetl plik

@ -0,0 +1,102 @@
#include "http_request.h"
#include <iostream>
#include <string>
#include <vector>
#include <map>
#include <sstream>
#include <chrono>
#include <thread>
#include <boost/beast/core.hpp>
#include <boost/beast/http.hpp>
#include <boost/beast/version.hpp>
#include <boost/asio/connect.hpp>
#include <boost/asio/ip/tcp.hpp>
namespace habdec
{
int HttpRequest(
const std::string host,
const std::string target,
const int port,
habdec::HTTP_VERB i_verb,
const std::string i_content_type,
const std::string i_body,
std::string& o_body
)
{
using namespace std;
using tcp = boost::asio::ip::tcp;
namespace http = boost::beast::http;
o_body = "";
try
{
boost::asio::io_context ioc;
tcp::resolver resolver{ioc};
tcp::socket socket{ioc};
auto const results = resolver.resolve(host, std::to_string(port));
boost::asio::connect(socket, results.begin(), results.end());
http::request<http::string_body> req; //{_verb, target, 10/*version*/};
req.set(http::field::host, host);
req.set(http::field::user_agent, "habdec");
req.version(10); // 11 ?
req.target(target);
if(i_verb == HTTP_VERB::kGet)
req.method( http::verb::get );
else if(i_verb == HTTP_VERB::kPost)
req.method( http::verb::post );
if( i_content_type != "" )
req.set(http::field::content_type, i_content_type);
if( i_body != "" )
req.body() = i_body;
req.prepare_payload();
http::write(socket, req);
boost::beast::flat_buffer buffer;
http::response<http::dynamic_body> res;
try {
http::read(socket, buffer, res);
}
catch(std::exception const& e) {
o_body += e.what() + string("\n");
o_body += string(res.reason());
std::cerr << "Error: " << o_body << std::endl;
return 0;
}
// std::cout << res <<std::endl;
// for(auto const& field : res) std::cout << field.name() << " = " << field.value() << "\n";
// Gracefully close the socket
boost::system::error_code ec;
socket.shutdown(tcp::socket::shutdown_both, ec);
// not_connected happens sometimes
// so don't bother reporting it.
//
if(ec && ec != boost::system::errc::not_connected)
throw boost::system::system_error{ec};
// If we get here then the connection is closed gracefully
o_body = buffers_to_string( res.body().data()) ;
return res.result_int();
}
catch(std::exception const& e)
{
o_body = e.what();
std::cerr << "Error: " << o_body << std::endl;
return 0;
}
return 0;
}
} // namespace habdec

Wyświetl plik

@ -0,0 +1,21 @@
#pragma once
#include <string>
namespace habdec
{
enum class HTTP_VERB { kUnknown = 0, kGet, kPost };
int HttpRequest(
const std::string host,
const std::string target,
const int port,
HTTP_VERB i_verb,
const std::string i_content_type,
const std::string i_body,
std::string& o_body
);
} // namespace habdec

Wyświetl plik

@ -0,0 +1,380 @@
/*
The MIT License (MIT)
Copyright (C) 2017 okdshin
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
//https://github.com/okdshin/PicoSHA2
#ifndef PICOSHA2_H
#define PICOSHA2_H
// picosha2:20140213
#ifndef PICOSHA2_BUFFER_SIZE_FOR_INPUT_ITERATOR
#define PICOSHA2_BUFFER_SIZE_FOR_INPUT_ITERATOR \
1048576 //=1024*1024: default is 1MB memory
#endif
#include <algorithm>
#include <cassert>
#include <iterator>
#include <sstream>
#include <vector>
#include <fstream>
namespace picosha2 {
typedef unsigned long word_t;
typedef unsigned char byte_t;
static const size_t k_digest_size = 32;
namespace detail {
inline byte_t mask_8bit(byte_t x) { return x & 0xff; }
inline word_t mask_32bit(word_t x) { return x & 0xffffffff; }
const word_t add_constant[64] = {
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1,
0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786,
0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147,
0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b,
0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a,
0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2};
const word_t initial_message_digest[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372,
0xa54ff53a, 0x510e527f, 0x9b05688c,
0x1f83d9ab, 0x5be0cd19};
inline word_t ch(word_t x, word_t y, word_t z) { return (x & y) ^ ((~x) & z); }
inline word_t maj(word_t x, word_t y, word_t z) {
return (x & y) ^ (x & z) ^ (y & z);
}
inline word_t rotr(word_t x, std::size_t n) {
assert(n < 32);
return mask_32bit((x >> n) | (x << (32 - n)));
}
inline word_t bsig0(word_t x) { return rotr(x, 2) ^ rotr(x, 13) ^ rotr(x, 22); }
inline word_t bsig1(word_t x) { return rotr(x, 6) ^ rotr(x, 11) ^ rotr(x, 25); }
inline word_t shr(word_t x, std::size_t n) {
assert(n < 32);
return x >> n;
}
inline word_t ssig0(word_t x) { return rotr(x, 7) ^ rotr(x, 18) ^ shr(x, 3); }
inline word_t ssig1(word_t x) { return rotr(x, 17) ^ rotr(x, 19) ^ shr(x, 10); }
template <typename RaIter1, typename RaIter2>
void hash256_block(RaIter1 message_digest, RaIter2 first, RaIter2 last) {
assert(first + 64 == last);
static_cast<void>(last); // for avoiding unused-variable warning
word_t w[64];
std::fill(w, w + 64, 0);
for (std::size_t i = 0; i < 16; ++i) {
w[i] = (static_cast<word_t>(mask_8bit(*(first + i * 4))) << 24) |
(static_cast<word_t>(mask_8bit(*(first + i * 4 + 1))) << 16) |
(static_cast<word_t>(mask_8bit(*(first + i * 4 + 2))) << 8) |
(static_cast<word_t>(mask_8bit(*(first + i * 4 + 3))));
}
for (std::size_t i = 16; i < 64; ++i) {
w[i] = mask_32bit(ssig1(w[i - 2]) + w[i - 7] + ssig0(w[i - 15]) +
w[i - 16]);
}
word_t a = *message_digest;
word_t b = *(message_digest + 1);
word_t c = *(message_digest + 2);
word_t d = *(message_digest + 3);
word_t e = *(message_digest + 4);
word_t f = *(message_digest + 5);
word_t g = *(message_digest + 6);
word_t h = *(message_digest + 7);
for (std::size_t i = 0; i < 64; ++i) {
word_t temp1 = h + bsig1(e) + ch(e, f, g) + add_constant[i] + w[i];
word_t temp2 = bsig0(a) + maj(a, b, c);
h = g;
g = f;
f = e;
e = mask_32bit(d + temp1);
d = c;
c = b;
b = a;
a = mask_32bit(temp1 + temp2);
}
*message_digest += a;
*(message_digest + 1) += b;
*(message_digest + 2) += c;
*(message_digest + 3) += d;
*(message_digest + 4) += e;
*(message_digest + 5) += f;
*(message_digest + 6) += g;
*(message_digest + 7) += h;
for (std::size_t i = 0; i < 8; ++i) {
*(message_digest + i) = mask_32bit(*(message_digest + i));
}
}
} // namespace detail
template <typename InIter>
void output_hex(InIter first, InIter last, std::ostream& os) {
os.setf(std::ios::hex, std::ios::basefield);
while (first != last) {
os.width(2);
os.fill('0');
os << static_cast<unsigned int>(*first);
++first;
}
os.setf(std::ios::dec, std::ios::basefield);
}
template <typename InIter>
void bytes_to_hex_string(InIter first, InIter last, std::string& hex_str) {
std::ostringstream oss;
output_hex(first, last, oss);
hex_str.assign(oss.str());
}
template <typename InContainer>
void bytes_to_hex_string(const InContainer& bytes, std::string& hex_str) {
bytes_to_hex_string(bytes.begin(), bytes.end(), hex_str);
}
template <typename InIter>
std::string bytes_to_hex_string(InIter first, InIter last) {
std::string hex_str;
bytes_to_hex_string(first, last, hex_str);
return hex_str;
}
template <typename InContainer>
std::string bytes_to_hex_string(const InContainer& bytes) {
std::string hex_str;
bytes_to_hex_string(bytes, hex_str);
return hex_str;
}
class hash256_one_by_one {
public:
hash256_one_by_one() { init(); }
void init() {
buffer_.clear();
std::fill(data_length_digits_, data_length_digits_ + 4, 0);
std::copy(detail::initial_message_digest,
detail::initial_message_digest + 8, h_);
}
template <typename RaIter>
void process(RaIter first, RaIter last) {
add_to_data_length(static_cast<word_t>(std::distance(first, last)));
std::copy(first, last, std::back_inserter(buffer_));
std::size_t i = 0;
for (; i + 64 <= buffer_.size(); i += 64) {
detail::hash256_block(h_, buffer_.begin() + i,
buffer_.begin() + i + 64);
}
buffer_.erase(buffer_.begin(), buffer_.begin() + i);
}
void finish() {
byte_t temp[64];
std::fill(temp, temp + 64, 0);
std::size_t remains = buffer_.size();
std::copy(buffer_.begin(), buffer_.end(), temp);
temp[remains] = 0x80;
if (remains > 55) {
std::fill(temp + remains + 1, temp + 64, 0);
detail::hash256_block(h_, temp, temp + 64);
std::fill(temp, temp + 64 - 4, 0);
} else {
std::fill(temp + remains + 1, temp + 64 - 4, 0);
}
write_data_bit_length(&(temp[56]));
detail::hash256_block(h_, temp, temp + 64);
}
template <typename OutIter>
void get_hash_bytes(OutIter first, OutIter last) const {
for (const word_t* iter = h_; iter != h_ + 8; ++iter) {
for (std::size_t i = 0; i < 4 && first != last; ++i) {
*(first++) = detail::mask_8bit(
static_cast<byte_t>((*iter >> (24 - 8 * i))));
}
}
}
private:
void add_to_data_length(word_t n) {
word_t carry = 0;
data_length_digits_[0] += n;
for (std::size_t i = 0; i < 4; ++i) {
data_length_digits_[i] += carry;
if (data_length_digits_[i] >= 65536u) {
carry = data_length_digits_[i] >> 16;
data_length_digits_[i] &= 65535u;
} else {
break;
}
}
}
void write_data_bit_length(byte_t* begin) {
word_t data_bit_length_digits[4];
std::copy(data_length_digits_, data_length_digits_ + 4,
data_bit_length_digits);
// convert byte length to bit length (multiply 8 or shift 3 times left)
word_t carry = 0;
for (std::size_t i = 0; i < 4; ++i) {
word_t before_val = data_bit_length_digits[i];
data_bit_length_digits[i] <<= 3;
data_bit_length_digits[i] |= carry;
data_bit_length_digits[i] &= 65535u;
carry = (before_val >> (16 - 3)) & 65535u;
}
// write data_bit_length
for (int i = 3; i >= 0; --i) {
(*begin++) = static_cast<byte_t>(data_bit_length_digits[i] >> 8);
(*begin++) = static_cast<byte_t>(data_bit_length_digits[i]);
}
}
std::vector<byte_t> buffer_;
word_t data_length_digits_[4]; // as 64bit integer (16bit x 4 integer)
word_t h_[8];
};
inline void get_hash_hex_string(const hash256_one_by_one& hasher,
std::string& hex_str) {
byte_t hash[k_digest_size];
hasher.get_hash_bytes(hash, hash + k_digest_size);
return bytes_to_hex_string(hash, hash + k_digest_size, hex_str);
}
inline std::string get_hash_hex_string(const hash256_one_by_one& hasher) {
std::string hex_str;
get_hash_hex_string(hasher, hex_str);
return hex_str;
}
namespace impl {
template <typename RaIter, typename OutIter>
void hash256_impl(RaIter first, RaIter last, OutIter first2, OutIter last2, int,
std::random_access_iterator_tag) {
hash256_one_by_one hasher;
// hasher.init();
hasher.process(first, last);
hasher.finish();
hasher.get_hash_bytes(first2, last2);
}
template <typename InputIter, typename OutIter>
void hash256_impl(InputIter first, InputIter last, OutIter first2,
OutIter last2, int buffer_size, std::input_iterator_tag) {
std::vector<byte_t> buffer(buffer_size);
hash256_one_by_one hasher;
// hasher.init();
while (first != last) {
int size = buffer_size;
for (int i = 0; i != buffer_size; ++i, ++first) {
if (first == last) {
size = i;
break;
}
buffer[i] = *first;
}
hasher.process(buffer.begin(), buffer.begin() + size);
}
hasher.finish();
hasher.get_hash_bytes(first2, last2);
}
}
template <typename InIter, typename OutIter>
void hash256(InIter first, InIter last, OutIter first2, OutIter last2,
int buffer_size = PICOSHA2_BUFFER_SIZE_FOR_INPUT_ITERATOR) {
picosha2::impl::hash256_impl(
first, last, first2, last2, buffer_size,
typename std::iterator_traits<InIter>::iterator_category());
}
template <typename InIter, typename OutContainer>
void hash256(InIter first, InIter last, OutContainer& dst) {
hash256(first, last, dst.begin(), dst.end());
}
template <typename InContainer, typename OutIter>
void hash256(const InContainer& src, OutIter first, OutIter last) {
hash256(src.begin(), src.end(), first, last);
}
template <typename InContainer, typename OutContainer>
void hash256(const InContainer& src, OutContainer& dst) {
hash256(src.begin(), src.end(), dst.begin(), dst.end());
}
template <typename InIter>
void hash256_hex_string(InIter first, InIter last, std::string& hex_str) {
byte_t hashed[k_digest_size];
hash256(first, last, hashed, hashed + k_digest_size);
std::ostringstream oss;
output_hex(hashed, hashed + k_digest_size, oss);
hex_str.assign(oss.str());
}
template <typename InIter>
std::string hash256_hex_string(InIter first, InIter last) {
std::string hex_str;
hash256_hex_string(first, last, hex_str);
return hex_str;
}
inline void hash256_hex_string(const std::string& src, std::string& hex_str) {
hash256_hex_string(src.begin(), src.end(), hex_str);
}
template <typename InContainer>
void hash256_hex_string(const InContainer& src, std::string& hex_str) {
hash256_hex_string(src.begin(), src.end(), hex_str);
}
template <typename InContainer>
std::string hash256_hex_string(const InContainer& src) {
return hash256_hex_string(src.begin(), src.end());
}
template<typename OutIter>void hash256(std::ifstream& f, OutIter first, OutIter last){
hash256(std::istreambuf_iterator<char>(f), std::istreambuf_iterator<char>(), first,last);
}
}// namespace picosha2
#endif // PICOSHA2_H

Wyświetl plik

@ -0,0 +1,14 @@
#pragma once
#include <string>
#include <map>
namespace habdec
{
std::string HabitatUploadSentence(
const std::string& i_sentence,
const std::string& i_listener_callsign
);
} // namespace

Wyświetl plik

@ -0,0 +1,160 @@
#include "habitat_interface.h"
#include "common/http_request.h"
#include <string>
#include <vector>
#include <iostream>
#include <map>
#include <sstream>
#include <chrono>
#include <thread>
#include <boost/date_time/local_time/local_time.hpp>
#include <boost/date_time/c_local_time_adjustor.hpp>
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>
#include "common/picosha2.h"
namespace
{
std::string sha256(const std::string& i_str)
{
using namespace std;
vector<unsigned char> hash(picosha2::k_digest_size);
picosha2::hash256(i_str.cbegin(), i_str.cend(), hash.begin(), hash.end());
return picosha2::bytes_to_hex_string(hash.begin(), hash.end());
}
const char encodeLookup[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
const char padCharacter = '=';
std::string base64Encode(std::string inputBuffer)
{
using namespace std;
string encodedString;
encodedString.reserve(((inputBuffer.size()/3) + (inputBuffer.size() % 3 > 0)) * 4);
long temp;
auto cursor = inputBuffer.begin();
for(size_t idx = 0; idx < inputBuffer.size()/3; idx++)
{
temp = (*cursor++) << 16; //Convert to big endian
temp += (*cursor++) << 8;
temp += (*cursor++);
encodedString.append(1,encodeLookup[(temp & 0x00FC0000) >> 18]);
encodedString.append(1,encodeLookup[(temp & 0x0003F000) >> 12]);
encodedString.append(1,encodeLookup[(temp & 0x00000FC0) >> 6 ]);
encodedString.append(1,encodeLookup[(temp & 0x0000003F) ]);
}
switch(inputBuffer.size() % 3)
{
case 1:
temp = (*cursor++) << 16; //Convert to big endian
encodedString.append(1,encodeLookup[(temp & 0x00FC0000) >> 18]);
encodedString.append(1,encodeLookup[(temp & 0x0003F000) >> 12]);
encodedString.append(2,padCharacter);
break;
case 2:
temp = (*cursor++) << 16; //Convert to big endian
temp += (*cursor++) << 8;
encodedString.append(1,encodeLookup[(temp & 0x00FC0000) >> 18]);
encodedString.append(1,encodeLookup[(temp & 0x0003F000) >> 12]);
encodedString.append(1,encodeLookup[(temp & 0x00000FC0) >> 6 ]);
encodedString.append(1,padCharacter);
break;
}
return encodedString;
}
std::string UtcNow()
{
using namespace boost::posix_time;
ptime now = second_clock::universal_time();
return to_iso_extended_string(now) + "Z"; // habitat needs Z at the end
}
std::string SentenceToHabJson(
const std::string& i_sentence,
const std::string& i_listener_callsign
)
{
using namespace std;
using namespace boost::posix_time;
ptime now = second_clock::universal_time();
string utc_now_str = to_iso_extended_string(now) + "Z"; // habitat needs Z at the end
boost::property_tree::ptree root;
root.put( "type", "payload_telemetry" );
root.put( "data._raw", i_sentence );
root.put( "receivers." + i_listener_callsign + ".time_created", utc_now_str );
root.put( "receivers." + i_listener_callsign + ".time_uploaded", utc_now_str );
stringstream ss;
boost::property_tree::json_parser::write_json(ss, root);
return ss.str();
}
} // namespace
namespace habdec
{
std::string HabitatUploadSentence(
const std::string& i_sentence,
const std::string& i_listener_callsign
)
{
using namespace std;
string sentence = i_sentence;
if( sentence.compare(0, 2, "$$") )
sentence = "$$" + sentence;
if( sentence.compare(sentence.size()-1, 1, "\n") )
sentence += "\n";
string sentence_b64 = base64Encode(sentence);
string sentence_sha256 = sha256(sentence_b64);
string sentence_json = SentenceToHabJson(sentence_b64, i_listener_callsign);
string result;
int retries_left = 5;
while( retries_left-- )
{
int http_result = HttpRequest(
"habitat.habhub.org",
"/habitat/_design/payload_telemetry/_update/add_listener/" + sentence_sha256,
80,
habdec::HTTP_VERB::kPost,
"application/json",
sentence_json,
result
);
switch( http_result )
{
case 201: // Success
case 403: // Success, sentence has already seen by others
retries_left = 0;
break;
case 409: // Upload conflict (server busy). Sleep for a moment, then retry
default:
this_thread::sleep_for( ( chrono::duration<double, milli>(300) ) );
break;
}
}
return result;
}
} // namespace habdec

Wyświetl plik

@ -42,6 +42,8 @@ endif()
set ( websocketServer_src
program_options.h program_options.cpp
server.h server.cpp
${CMAKE_SOURCE_DIR}/habitat/habitat_upload.cpp
${CMAKE_SOURCE_DIR}/common/http_request.cpp
main.cpp
)

Wyświetl plik

@ -32,6 +32,7 @@
#include "IQSource/IQSource_SoapySDR.h"
#include "Decoder/Decoder.h"
#include "common/console_colors.h"
#include "habitat/habitat_interface.h"
#include "GLOBALS.h"
#include "server.h"
@ -234,6 +235,13 @@ void DECODER_FEED_THREAD()
void HAB_UPLOAD_THREAD()
{
using namespace std;
using namespace habdec;
if(GLOBALS::get().station_callsign_ == "")
{
cout<<C_RED<<"No --station parameter set. HAB Upload disabled."<<C_OFF<<endl;
return;
}
while(!G_DO_EXIT)
{
@ -244,18 +252,12 @@ void HAB_UPLOAD_THREAD()
lock_guard<mutex> _lock( GLOBALS::get().senteces_to_log_mtx_ );
sentences = move( GLOBALS::get().senteces_to_log_ );
}
if(GLOBALS::get().station_callsign_ != "")
{
for(auto& sentence : sentences)
{
string cmd = "python ./habLogger.py ";
cmd += sentence;
cmd += " ";
cmd += GLOBALS::get().station_callsign_;
auto res = system(cmd.c_str());
cout<<"HAB upload res: "<<res<<endl;
}
for(auto& sentence : sentences)
{
string res = HabitatUploadSentence( sentence, GLOBALS::get().station_callsign_ );
if( res != "OK" )
cout<<C_CLEAR<<"HAB Upload result: "<<res<<endl;
}
}
}
@ -293,9 +295,6 @@ int main(int argc, char** argv)
return 1;
}
if(GLOBALS::get().station_callsign_ == "")
cout<<C_RED<<"No --station parameter set. HAB Upload disabled."<<C_OFF<<endl;
// initial options
auto& DECODER = GLOBALS::get().decoder_;
DECODER.baud(GLOBALS::get().baud_);