Merge pull request #125 from jaywink/mastodon-parser

Add parser for Mastodon server info and activity documents
2018-05-27 01:58:05 +03:00 · 2018-05-27 01:58:05 +03:00 · 5e3e363714
commit 5e3e363714
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -12,7 +12,7 @@

  Also provided is a Django view and url configuration for easy addition into Django projects. Django is not a hard dependency of this library, usage of the Django view obviously requires installing Django itself. For configuration details see documentation.

-* Add fetchers and parsers for NodeInfo, NodeInfo2 and StatisticsJSON server metainfo documents.
+* Add fetchers and parsers for NodeInfo, NodeInfo2, StatisticsJSON and Mastodon server metainfo documents.

 * Add NodeInfo2 generator and Django view. See documentation for details. ([related issue](https://github.com/jaywink/federation/issues/32))

--- a/federation/hostmeta/fetchers.py
+++ b/federation/hostmeta/fetchers.py
@ -1,11 +1,23 @@
 import json

-from federation.hostmeta.parsers import parse_nodeinfo_document, parse_nodeinfo2_document, parse_statisticsjson_document
+from federation.hostmeta.parsers import (
+    parse_nodeinfo_document, parse_nodeinfo2_document, parse_statisticsjson_document, parse_mastodon_document)
 from federation.utils.network import fetch_document

 HIGHEST_SUPPORTED_NODEINFO_VERSION = 2.0


+def fetch_mastodon_document(host):
+    doc, status_code, error = fetch_document(host=host, path='/api/v1/instance')
+    if not doc:
+        return
+    try:
+        doc = json.loads(doc)
+    except json.JSONDecodeError:
+        return
+    return parse_mastodon_document(doc, host)
+
+
 def fetch_nodeinfo_document(host):
    doc, status_code, error = fetch_document(host=host, path='/.well-known/nodeinfo')
    if not doc:
--- a/federation/hostmeta/parsers.py
+++ b/federation/hostmeta/parsers.py
@ -1,7 +1,14 @@
+import json
 import re
 from copy import deepcopy

 from federation.utils.diaspora import generate_diaspora_profile_id
+from federation.utils.network import fetch_document
+
+WEEKLY_USERS_HALFYEAR_MULTIPLIER = 10.34
+WEEKLY_USERS_MONTHLY_MULTIPLIER = 3.17
+MONTHLY_USERS_WEEKLY_MULTIPLIER = 0.316
+HALFYEAR_USERS_WEEKLY_MULTIPLIER = 0.097

 defaults = {
    'organization': {
@ -44,6 +51,45 @@ def int_or_none(value):
        return None


+def parse_mastodon_document(doc, host):
+    result = deepcopy(defaults)
+    result['host'] = host
+    result['name'] = doc.get('title', host)
+    result['platform'] = 'mastodon'
+    result['version'] = doc.get('version', '')
+    # TODO parse about page
+    # https://github.com/TheKinrar/mastodon-instances/blob/master/tasks/update_instances.js#L508
+    # result['open_signups']
+    version = [int(part) for part in doc.get('version', '').split('.')]
+    if version >= [1, 6, 0]:
+        result['protocols'] = ['ostatus', 'activitypub']
+    else:
+        result['protocols'] = ['ostatus']
+    result['relay'] = False
+
+    result['activity']['users']['total'] = int_or_none(doc.get('stats', {}).get('user_count'))
+    # TODO figure out what to do with posts vs comments vs statuses
+    #result['activity']['users']['local_posts'] = int_or_none(doc.get('stats', {}).get('status_count'))
+
+    activity_doc, _status_code, _error = fetch_document(host=host, path='/api/v1/instance/activity')
+    if activity_doc:
+        try:
+            activity_doc = json.loads(activity_doc)
+        except json.JSONDecodeError:
+            return
+        else:
+            weekly_count = int_or_none(activity_doc[0].get('logins'))
+            result['activity']['users']['weekly'] = weekly_count
+            if weekly_count:
+                result['activity']['users']['half_year'] = int(weekly_count * WEEKLY_USERS_HALFYEAR_MULTIPLIER)
+                result['activity']['users']['monthly'] = int(weekly_count * WEEKLY_USERS_MONTHLY_MULTIPLIER)
+
+    result['organization']['account'] = doc.get('contact_account', {}).get('url', '')
+    result['organization']['contact'] = doc.get('email', '')
+    result['organization']['name'] = doc.get('contact_account', {}).get('display_name', '')
+    return result
+
+
 def parse_nodeinfo_document(doc, host):
    result = deepcopy(defaults)
    nodeinfo_version = doc.get('version', '1.0')
@ -65,7 +111,10 @@ def parse_nodeinfo_document(doc, host):
    result['open_signups'] = doc.get('openRegistrations', False)
    result['activity']['users']['total'] = int_or_none(doc.get('usage', {}).get('users', {}).get('total'))
    result['activity']['users']['half_year'] = int_or_none(doc.get('usage', {}).get('users', {}).get('activeHalfyear'))
-    result['activity']['users']['monthly'] = int_or_none(doc.get('usage', {}).get('users', {}).get('activeMonth'))
+    monthly = int_or_none(doc.get('usage', {}).get('users', {}).get('activeMonth'))
+    result['activity']['users']['monthly'] = monthly
+    if monthly:
+        result['activity']['users']['weekly'] = int(monthly * MONTHLY_USERS_WEEKLY_MULTIPLIER)
    result['activity']['local_posts'] = int_or_none(doc.get('usage', {}).get('localPosts'))
    result['activity']['local_comments'] = int_or_none(doc.get('usage', {}).get('localComments'))
    result['features'] = doc.get('metadata', {})
@ -118,7 +167,10 @@ def parse_statisticsjson_document(doc, host):
    result['protocols'] = ['diaspora']  # Reasonable default
    result['activity']['users']['total'] = int_or_none(doc.get('total_users'))
    result['activity']['users']['half_year'] = int_or_none(doc.get('active_users_halfyear'))
-    result['activity']['users']['monthly'] = int_or_none(doc.get('active_users_monthly'))
+    monthly = int_or_none(doc.get('active_users_monthly'))
+    result['activity']['users']['monthly'] = monthly
+    if monthly:
+        result['activity']['users']['weekly'] = int(monthly * MONTHLY_USERS_WEEKLY_MULTIPLIER)
    result['activity']['local_posts'] = int_or_none(doc.get('local_posts'))
    result['activity']['local_comments'] = int_or_none(doc.get('local_comments'))
    return result
--- a/federation/tests/fixtures/hostmeta.py
+++ b/federation/tests/fixtures/hostmeta.py
@ -1,3 +1,29 @@
+MASTODON_DOC = """
+{"uri": "mastodon.local", "title": "Mastodon",
+ "description": "This page describes the mastodon.local",
+ "email": "hello@mastodon.local", "version": "2.4.0", "urls": {"streaming_api": "wss://mastodon.local"},
+ "stats": {"user_count": 159726, "status_count": 6059606, "domain_count": 4703},
+ "thumbnail": "https://files.mastodon.local/site_uploads/files/000/000/001/original/file.jpeg",
+ "languages": ["en"],
+ "contact_account": {"id": "1", "username": "Admin", "acct": "Admin", "display_name": "Admin dude", "locked": false,
+                     "bot": false, "created_at": "2016-03-16T14:34:26.392Z",
+                     "note": "\u003cp\u003eSuperuser\u003c/p\u003e",
+                     "url": "https://mastodon.local/@Admin",
+                     "avatar": "https://files.mastodon.local/accounts/avatars/000/000/001/original/file.png",
+                     "avatar_static": "https://files.mastodon.local/accounts/avatars/000/000/001/original/file.png",
+                     "header": "https://files.mastodon.local/accounts/headers/000/000/001/original/file.jpeg",
+                     "header_static": "https://files.mastodon.local/accounts/headers/000/000/001/original/file.jpeg",
+                     "followers_count": 81779, "following_count": 506, "statuses_count": 36722, "emojis": [],
+                     "fields": []}}
+"""
+
+MASTODON_ACTIVITY_DOC = """
+    [
+        {"week":"1526860800","statuses":"121188","logins":"8779","registrations":"1143"},
+        {"week":"1526256000","statuses":"200229","logins":"10034","registrations":"1379"}
+    ]
+"""
+
 NODEINFO_10_DOC = '{"version":"1.0","software":{"name":"diaspora","version":"0.7.4.0-pd0313756"},"protocols":' \
                  '{"inbound":["diaspora"],"outbound":["diaspora"]},"services":{"inbound":[],"outbound":["twi' \
                  'tter","tumblr"]},"openRegistrations":true,"usage":{"users":{"total":348,"activeHalfyear":1' \
--- a/federation/tests/hostmeta/test_fetchers.py
+++ b/federation/tests/hostmeta/test_fetchers.py
@ -2,10 +2,21 @@ import json
 from unittest.mock import patch

 from federation.hostmeta.fetchers import (
-    fetch_nodeinfo_document, fetch_nodeinfo2_document, fetch_statisticsjson_document)
+    fetch_nodeinfo_document, fetch_nodeinfo2_document, fetch_statisticsjson_document, fetch_mastodon_document)
 from federation.tests.fixtures.hostmeta import NODEINFO_WELL_KNOWN_BUGGY, NODEINFO_WELL_KNOWN_BUGGY_2


+class TestFetchMastodonDocument:
+    @patch("federation.hostmeta.fetchers.fetch_document", return_value=('{"foo": "bar"}', 200, None), autospec=True)
+    @patch("federation.hostmeta.fetchers.parse_mastodon_document", autospec=True)
+    def test_makes_right_calls(self, mock_parse, mock_fetch):
+        fetch_mastodon_document('example.com')
+        args, kwargs = mock_fetch.call_args
+        assert kwargs['host'] == 'example.com'
+        assert kwargs['path'] == '/api/v1/instance'
+        mock_parse.assert_called_once_with({"foo": "bar"}, 'example.com')
+
+
 class TestFetchNodeInfoDocument:
    dummy_doc = json.dumps({"links": [
        {"href": "https://example.com/1.0", "rel": "http://nodeinfo.diaspora.software/ns/schema/1.0"},
--- a/federation/tests/hostmeta/test_parsers.py
+++ b/federation/tests/hostmeta/test_parsers.py
@ -1,8 +1,11 @@
 import json
+from unittest.mock import patch

 from federation.hostmeta.parsers import (
-    parse_nodeinfo_document, parse_nodeinfo2_document, parse_statisticsjson_document, int_or_none)
-from federation.tests.fixtures.hostmeta import NODEINFO2_10_DOC, NODEINFO_10_DOC, NODEINFO_20_DOC, STATISTICS_JSON_DOC
+    parse_nodeinfo_document, parse_nodeinfo2_document, parse_statisticsjson_document, int_or_none,
+    parse_mastodon_document)
+from federation.tests.fixtures.hostmeta import (
+    NODEINFO2_10_DOC, NODEINFO_10_DOC, NODEINFO_20_DOC, STATISTICS_JSON_DOC, MASTODON_DOC, MASTODON_ACTIVITY_DOC)


 class TestIntOrNone:
@ -10,6 +13,40 @@ class TestIntOrNone:
        assert int_or_none(-1) is None


+class TestParseMastodonDocument:
+    @patch('federation.hostmeta.parsers.fetch_document')
+    def test_parse_mastodon_document(self, mock_fetch):
+        mock_fetch.return_value = MASTODON_ACTIVITY_DOC, 200, None
+        result = parse_mastodon_document(json.loads(MASTODON_DOC), 'example.com')
+        assert result == {
+            'organization': {
+                'account': 'https://mastodon.local/@Admin',
+                'contact': 'hello@mastodon.local',
+                'name': 'Admin dude',
+            },
+            'host': 'example.com',
+            'name': 'Mastodon',
+            'open_signups': False,
+            'protocols': ["ostatus", "activitypub"],
+            'relay': False,
+            'server_meta': {},
+            'services': [],
+            'platform': 'mastodon',
+            'version': '2.4.0',
+            'features': {},
+            'activity': {
+                'users': {
+                    'total': 159726,
+                    'half_year': 90774,
+                    'monthly': 27829,
+                    'weekly': 8779,
+                },
+                'local_posts': None,
+                'local_comments': None,
+            },
+        }
+
+
 class TestParseNodeInfoDocument:
    def test_parse_nodeinfo_10_document(self):
        result = parse_nodeinfo_document(json.loads(NODEINFO_10_DOC), 'iliketoast.net')
@ -43,7 +80,7 @@ class TestParseNodeInfoDocument:
                    'total': 348,
                    'half_year': 123,
                    'monthly': 62,
-                    'weekly': None,
+                    'weekly': 19,
                },
                'local_posts': 8522,
                'local_comments': 17671,
@ -82,7 +119,7 @@ class TestParseNodeInfoDocument:
                    'total': 348,
                    'half_year': 123,
                    'monthly': 62,
-                    'weekly': None,
+                    'weekly': 19,
                },
                'local_posts': 8522,
                'local_comments': 17671,