diff --git a/CHANGELOG.md b/CHANGELOG.md index 02f48d9..561ce7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ Also provided is a Django view and url configuration for easy addition into Django projects. Django is not a hard dependency of this library, usage of the Django view obviously requires installing Django itself. For configuration details see documentation. -* Add fetchers and parsers for NodeInfo, NodeInfo2 and StatisticsJSON server metainfo documents. +* Add fetchers and parsers for NodeInfo, NodeInfo2, StatisticsJSON and Mastodon server metainfo documents. * Add NodeInfo2 generator and Django view. See documentation for details. ([related issue](https://github.com/jaywink/federation/issues/32)) diff --git a/federation/hostmeta/fetchers.py b/federation/hostmeta/fetchers.py index eb2e939..e6396d2 100644 --- a/federation/hostmeta/fetchers.py +++ b/federation/hostmeta/fetchers.py @@ -1,11 +1,23 @@ import json -from federation.hostmeta.parsers import parse_nodeinfo_document, parse_nodeinfo2_document, parse_statisticsjson_document +from federation.hostmeta.parsers import ( + parse_nodeinfo_document, parse_nodeinfo2_document, parse_statisticsjson_document, parse_mastodon_document) from federation.utils.network import fetch_document HIGHEST_SUPPORTED_NODEINFO_VERSION = 2.0 +def fetch_mastodon_document(host): + doc, status_code, error = fetch_document(host=host, path='/api/v1/instance') + if not doc: + return + try: + doc = json.loads(doc) + except json.JSONDecodeError: + return + return parse_mastodon_document(doc, host) + + def fetch_nodeinfo_document(host): doc, status_code, error = fetch_document(host=host, path='/.well-known/nodeinfo') if not doc: diff --git a/federation/hostmeta/parsers.py b/federation/hostmeta/parsers.py index 047d265..c2ef606 100644 --- a/federation/hostmeta/parsers.py +++ b/federation/hostmeta/parsers.py @@ -1,7 +1,14 @@ +import json import re from copy import deepcopy from federation.utils.diaspora import generate_diaspora_profile_id +from federation.utils.network import fetch_document + +WEEKLY_USERS_HALFYEAR_MULTIPLIER = 10.34 +WEEKLY_USERS_MONTHLY_MULTIPLIER = 3.17 +MONTHLY_USERS_WEEKLY_MULTIPLIER = 0.316 +HALFYEAR_USERS_WEEKLY_MULTIPLIER = 0.097 defaults = { 'organization': { @@ -44,6 +51,45 @@ def int_or_none(value): return None +def parse_mastodon_document(doc, host): + result = deepcopy(defaults) + result['host'] = host + result['name'] = doc.get('title', host) + result['platform'] = 'mastodon' + result['version'] = doc.get('version', '') + # TODO parse about page + # https://github.com/TheKinrar/mastodon-instances/blob/master/tasks/update_instances.js#L508 + # result['open_signups'] + version = [int(part) for part in doc.get('version', '').split('.')] + if version >= [1, 6, 0]: + result['protocols'] = ['ostatus', 'activitypub'] + else: + result['protocols'] = ['ostatus'] + result['relay'] = False + + result['activity']['users']['total'] = int_or_none(doc.get('stats', {}).get('user_count')) + # TODO figure out what to do with posts vs comments vs statuses + #result['activity']['users']['local_posts'] = int_or_none(doc.get('stats', {}).get('status_count')) + + activity_doc, _status_code, _error = fetch_document(host=host, path='/api/v1/instance/activity') + if activity_doc: + try: + activity_doc = json.loads(activity_doc) + except json.JSONDecodeError: + return + else: + weekly_count = int_or_none(activity_doc[0].get('logins')) + result['activity']['users']['weekly'] = weekly_count + if weekly_count: + result['activity']['users']['half_year'] = int(weekly_count * WEEKLY_USERS_HALFYEAR_MULTIPLIER) + result['activity']['users']['monthly'] = int(weekly_count * WEEKLY_USERS_MONTHLY_MULTIPLIER) + + result['organization']['account'] = doc.get('contact_account', {}).get('url', '') + result['organization']['contact'] = doc.get('email', '') + result['organization']['name'] = doc.get('contact_account', {}).get('display_name', '') + return result + + def parse_nodeinfo_document(doc, host): result = deepcopy(defaults) nodeinfo_version = doc.get('version', '1.0') @@ -65,7 +111,10 @@ def parse_nodeinfo_document(doc, host): result['open_signups'] = doc.get('openRegistrations', False) result['activity']['users']['total'] = int_or_none(doc.get('usage', {}).get('users', {}).get('total')) result['activity']['users']['half_year'] = int_or_none(doc.get('usage', {}).get('users', {}).get('activeHalfyear')) - result['activity']['users']['monthly'] = int_or_none(doc.get('usage', {}).get('users', {}).get('activeMonth')) + monthly = int_or_none(doc.get('usage', {}).get('users', {}).get('activeMonth')) + result['activity']['users']['monthly'] = monthly + if monthly: + result['activity']['users']['weekly'] = int(monthly * MONTHLY_USERS_WEEKLY_MULTIPLIER) result['activity']['local_posts'] = int_or_none(doc.get('usage', {}).get('localPosts')) result['activity']['local_comments'] = int_or_none(doc.get('usage', {}).get('localComments')) result['features'] = doc.get('metadata', {}) @@ -118,7 +167,10 @@ def parse_statisticsjson_document(doc, host): result['protocols'] = ['diaspora'] # Reasonable default result['activity']['users']['total'] = int_or_none(doc.get('total_users')) result['activity']['users']['half_year'] = int_or_none(doc.get('active_users_halfyear')) - result['activity']['users']['monthly'] = int_or_none(doc.get('active_users_monthly')) + monthly = int_or_none(doc.get('active_users_monthly')) + result['activity']['users']['monthly'] = monthly + if monthly: + result['activity']['users']['weekly'] = int(monthly * MONTHLY_USERS_WEEKLY_MULTIPLIER) result['activity']['local_posts'] = int_or_none(doc.get('local_posts')) result['activity']['local_comments'] = int_or_none(doc.get('local_comments')) return result diff --git a/federation/tests/fixtures/hostmeta.py b/federation/tests/fixtures/hostmeta.py index 56b1436..d947770 100644 --- a/federation/tests/fixtures/hostmeta.py +++ b/federation/tests/fixtures/hostmeta.py @@ -1,3 +1,29 @@ +MASTODON_DOC = """ +{"uri": "mastodon.local", "title": "Mastodon", + "description": "This page describes the mastodon.local", + "email": "hello@mastodon.local", "version": "2.4.0", "urls": {"streaming_api": "wss://mastodon.local"}, + "stats": {"user_count": 159726, "status_count": 6059606, "domain_count": 4703}, + "thumbnail": "https://files.mastodon.local/site_uploads/files/000/000/001/original/file.jpeg", + "languages": ["en"], + "contact_account": {"id": "1", "username": "Admin", "acct": "Admin", "display_name": "Admin dude", "locked": false, + "bot": false, "created_at": "2016-03-16T14:34:26.392Z", + "note": "\u003cp\u003eSuperuser\u003c/p\u003e", + "url": "https://mastodon.local/@Admin", + "avatar": "https://files.mastodon.local/accounts/avatars/000/000/001/original/file.png", + "avatar_static": "https://files.mastodon.local/accounts/avatars/000/000/001/original/file.png", + "header": "https://files.mastodon.local/accounts/headers/000/000/001/original/file.jpeg", + "header_static": "https://files.mastodon.local/accounts/headers/000/000/001/original/file.jpeg", + "followers_count": 81779, "following_count": 506, "statuses_count": 36722, "emojis": [], + "fields": []}} +""" + +MASTODON_ACTIVITY_DOC = """ + [ + {"week":"1526860800","statuses":"121188","logins":"8779","registrations":"1143"}, + {"week":"1526256000","statuses":"200229","logins":"10034","registrations":"1379"} + ] +""" + NODEINFO_10_DOC = '{"version":"1.0","software":{"name":"diaspora","version":"0.7.4.0-pd0313756"},"protocols":' \ '{"inbound":["diaspora"],"outbound":["diaspora"]},"services":{"inbound":[],"outbound":["twi' \ 'tter","tumblr"]},"openRegistrations":true,"usage":{"users":{"total":348,"activeHalfyear":1' \ diff --git a/federation/tests/hostmeta/test_fetchers.py b/federation/tests/hostmeta/test_fetchers.py index c7f4eca..68c50b3 100644 --- a/federation/tests/hostmeta/test_fetchers.py +++ b/federation/tests/hostmeta/test_fetchers.py @@ -2,10 +2,21 @@ import json from unittest.mock import patch from federation.hostmeta.fetchers import ( - fetch_nodeinfo_document, fetch_nodeinfo2_document, fetch_statisticsjson_document) + fetch_nodeinfo_document, fetch_nodeinfo2_document, fetch_statisticsjson_document, fetch_mastodon_document) from federation.tests.fixtures.hostmeta import NODEINFO_WELL_KNOWN_BUGGY, NODEINFO_WELL_KNOWN_BUGGY_2 +class TestFetchMastodonDocument: + @patch("federation.hostmeta.fetchers.fetch_document", return_value=('{"foo": "bar"}', 200, None), autospec=True) + @patch("federation.hostmeta.fetchers.parse_mastodon_document", autospec=True) + def test_makes_right_calls(self, mock_parse, mock_fetch): + fetch_mastodon_document('example.com') + args, kwargs = mock_fetch.call_args + assert kwargs['host'] == 'example.com' + assert kwargs['path'] == '/api/v1/instance' + mock_parse.assert_called_once_with({"foo": "bar"}, 'example.com') + + class TestFetchNodeInfoDocument: dummy_doc = json.dumps({"links": [ {"href": "https://example.com/1.0", "rel": "http://nodeinfo.diaspora.software/ns/schema/1.0"}, diff --git a/federation/tests/hostmeta/test_parsers.py b/federation/tests/hostmeta/test_parsers.py index 45fbdab..b7e2174 100644 --- a/federation/tests/hostmeta/test_parsers.py +++ b/federation/tests/hostmeta/test_parsers.py @@ -1,8 +1,11 @@ import json +from unittest.mock import patch from federation.hostmeta.parsers import ( - parse_nodeinfo_document, parse_nodeinfo2_document, parse_statisticsjson_document, int_or_none) -from federation.tests.fixtures.hostmeta import NODEINFO2_10_DOC, NODEINFO_10_DOC, NODEINFO_20_DOC, STATISTICS_JSON_DOC + parse_nodeinfo_document, parse_nodeinfo2_document, parse_statisticsjson_document, int_or_none, + parse_mastodon_document) +from federation.tests.fixtures.hostmeta import ( + NODEINFO2_10_DOC, NODEINFO_10_DOC, NODEINFO_20_DOC, STATISTICS_JSON_DOC, MASTODON_DOC, MASTODON_ACTIVITY_DOC) class TestIntOrNone: @@ -10,6 +13,40 @@ class TestIntOrNone: assert int_or_none(-1) is None +class TestParseMastodonDocument: + @patch('federation.hostmeta.parsers.fetch_document') + def test_parse_mastodon_document(self, mock_fetch): + mock_fetch.return_value = MASTODON_ACTIVITY_DOC, 200, None + result = parse_mastodon_document(json.loads(MASTODON_DOC), 'example.com') + assert result == { + 'organization': { + 'account': 'https://mastodon.local/@Admin', + 'contact': 'hello@mastodon.local', + 'name': 'Admin dude', + }, + 'host': 'example.com', + 'name': 'Mastodon', + 'open_signups': False, + 'protocols': ["ostatus", "activitypub"], + 'relay': False, + 'server_meta': {}, + 'services': [], + 'platform': 'mastodon', + 'version': '2.4.0', + 'features': {}, + 'activity': { + 'users': { + 'total': 159726, + 'half_year': 90774, + 'monthly': 27829, + 'weekly': 8779, + }, + 'local_posts': None, + 'local_comments': None, + }, + } + + class TestParseNodeInfoDocument: def test_parse_nodeinfo_10_document(self): result = parse_nodeinfo_document(json.loads(NODEINFO_10_DOC), 'iliketoast.net') @@ -43,7 +80,7 @@ class TestParseNodeInfoDocument: 'total': 348, 'half_year': 123, 'monthly': 62, - 'weekly': None, + 'weekly': 19, }, 'local_posts': 8522, 'local_comments': 17671, @@ -82,7 +119,7 @@ class TestParseNodeInfoDocument: 'total': 348, 'half_year': 123, 'monthly': 62, - 'weekly': None, + 'weekly': 19, }, 'local_posts': 8522, 'local_comments': 17671,