Merge pull request #124 from jaywink/wip

Add fetching of IP and country

CHANGELOG.md

@@ -16,6 +16,12 @@
 
 * Add NodeInfo2 generator and Django view. See documentation for details. ([related issue](https://github.com/jaywink/federation/issues/32))
 
+* Added new network utilities to fetch IP and country information from a host.
+
+  The country information is fetched using the free `ip-api.com` service. NOTE! This service is rate limited to 150 requests per minute and requires a paid plan for commercial usage. Please make sure to respect the terms.
+  
+  Country information is automatically fetched when using the NodeInfo, NodeInfo2 and StatisticsJSON parsers.
+
 ### Changed
 
 * Send outbound Diaspora payloads in new format. Remove possibility to generate legacy MagicEnvelope payloads. ([related issue](https://github.com/jaywink/federation/issues/82))

docs/usage.rst

@@ -177,7 +177,9 @@ Diaspora
 Network
 .......
 
+.. autofunction:: federation.utils.network.fetch_country_by_ip
 .. autofunction:: federation.utils.network.fetch_document
+.. autofunction:: federation.utils.network.fetch_host_ip_and_country
 .. autofunction:: federation.utils.network.send_document
 
 

federation/tests/hostmeta/test_parsers.py

@@ -1,11 +1,13 @@
 import json
+from unittest.mock import patch
 
 from federation.hostmeta.parsers import parse_nodeinfo_document, parse_nodeinfo2_document, parse_statisticsjson_document
 from federation.tests.fixtures.hostmeta import NODEINFO2_10_DOC, NODEINFO_10_DOC, NODEINFO_20_DOC, STATISTICS_JSON_DOC
 
 
+@patch('federation.hostmeta.parsers.fetch_host_ip_and_country', return_value=("", ""))
 class TestParseNodeInfoDocument:
-    def test_parse_nodeinfo_10_document(self):
+    def test_parse_nodeinfo_10_document(self, mock_ip):
         result = parse_nodeinfo_document(json.loads(NODEINFO_10_DOC), 'iliketoast.net')
         assert result == {
             'organization': {
@@ -46,7 +48,7 @@ class TestParseNodeInfoDocument:
             },
         }
 
-    def test_parse_nodeinfo_20_document(self):
+    def test_parse_nodeinfo_20_document(self, mock_ip):
         result = parse_nodeinfo_document(json.loads(NODEINFO_20_DOC), 'iliketoast.net')
         assert result == {
             'organization': {
@@ -88,8 +90,9 @@ class TestParseNodeInfoDocument:
         }
 
 
+@patch('federation.hostmeta.parsers.fetch_host_ip_and_country', return_value=("", ""))
 class TestParseNodeInfo2Document:
-    def test_parse_nodeinfo2_10_document(self):
+    def test_parse_nodeinfo2_10_document(self, mock_ip):
         result = parse_nodeinfo2_document(json.loads(NODEINFO2_10_DOC), 'example.com')
         assert result == {
             'organization': {
@@ -122,8 +125,9 @@ class TestParseNodeInfo2Document:
         }
 
 
+@patch('federation.hostmeta.parsers.fetch_host_ip_and_country', return_value=("", ""))
 class TestParseStatisticsJSONDocument:
-    def test_parse_statisticsjson_document(self):
+    def test_parse_statisticsjson_document(self, mock_ip):
         result = parse_statisticsjson_document(json.loads(STATISTICS_JSON_DOC), 'example.com')
         assert result == {
             'organization': {

federation/tests/utils/test_network.py

@@ -1,14 +1,27 @@
-# -*- coding: utf-8 -*-
 from unittest.mock import patch, Mock, call
 
 import pytest
 from requests import HTTPError
 from requests.exceptions import SSLError, RequestException
 
-from federation.utils.network import fetch_document, USER_AGENT, send_document
+from federation.utils.network import (
+    fetch_document, USER_AGENT, send_document, fetch_country_by_ip, fetch_host_ip_and_country)
 
 
-class TestFetchDocument(object):
+@patch('federation.utils.network.requests.get', autospec=True, return_value=Mock(
+    status_code=200, json=Mock(return_value={'countryCode': 'FI'}),
+))
+class TestFetchCountryByIp:
+    def test_calls_ip_api_endpoint(self, mock_get):
+        fetch_country_by_ip('127.0.0.1')
+        mock_get.assert_called_once_with('http://ip-api.com/json/127.0.0.1"')
+
+    def test_returns_country_code(self, mock_get):
+        result = fetch_country_by_ip('127.0.0.1')
+        assert result == 'FI'
+
+
+class TestFetchDocument:
     call_args = {"timeout": 10, "headers": {'user-agent': USER_AGENT}}
 
     def test_raises_without_url_and_host(self):
@@ -88,7 +101,17 @@ class TestFetchDocument(object):
         assert exc.__class__ == RequestException
 
 
-class TestSendDocument(object):
+class TestHostIpAndCountry:
+    @patch('federation.utils.network.socket.gethostbyname', autospec=True, return_value='127.0.0.1')
+    @patch('federation.utils.network.fetch_country_by_ip', autospec=True, return_value='FI')
+    def test_calls(self, mock_fetch_country, mock_get_ip):
+        result = fetch_host_ip_and_country('domain.local')
+        assert result == ('127.0.0.1', 'FI')
+        mock_get_ip.assert_called_once_with('domain.local')
+        mock_fetch_country.assert_called_once_with('127.0.0.1')
+
+
+class TestSendDocument:
     call_args = {"timeout": 10, "headers": {'user-agent': USER_AGENT}}
 
     @patch("federation.utils.network.requests.post", return_value=Mock(status_code=200))

federation/utils/network.py

@@ -1,5 +1,5 @@
-# -*- coding: utf-8 -*-
 import logging
+import socket
 
 import requests
 from requests.exceptions import RequestException, HTTPError, SSLError
@@ -13,6 +13,27 @@ logger = logging.getLogger("federation")
 USER_AGENT = "python/federation/%s" % __version__
 
 
+def fetch_country_by_ip(ip):
+    """
+    Fetches country code by IP
+
+    Returns empty string if the request fails in non-200 code.
+
+    Uses the ip-api.com service which has the following rules:
+
+    * Max 150 requests per minute
+    * Non-commercial use only without a paid plan!
+
+    See: http://ip-api.com/docs/api:json
+    """
+    result = requests.get("http://ip-api.com/json/%s" % ip)
+    if result.status_code != 200:
+        return ''
+
+    result = result.json()
+    return result['countryCode']
+
+
 def fetch_document(url=None, host=None, path="/", timeout=10, raise_ssl_errors=True):
     """Helper method to fetch remote document.
 
@@ -74,6 +95,20 @@ def fetch_document(url=None, host=None, path="/", timeout=10, raise_ssl_errors=T
         return None, None, ex
 
 
+def fetch_host_ip_and_country(host):
+    """
+    Fetch ip and country by host
+    """
+    try:
+        ip = socket.gethostbyname(host)
+    except socket.gaierror:
+        return '', ''
+
+    country = fetch_country_by_ip(ip)
+
+    return ip, country
+
+
 def send_document(url, data, timeout=10, *args, **kwargs):
     """Helper method to send a document via POST.
 
@@ -101,8 +136,3 @@ def send_document(url, data, timeout=10, *args, **kwargs):
     except RequestException as ex:
         logger.debug("send_document: exception %s", ex)
         return None, ex
-
-
-def fetch_host_ip_and_country(host):
-    # TODO implement
-    return '', ''