kopia lustrzana https://github.com/halcy/Mastodon.py
335 wiersze
13 KiB
Plaintext
335 wiersze
13 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Imports"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Super special from this directory for sure and guaranteed import\n",
|
|
"import importlib.util\n",
|
|
"import sys\n",
|
|
"\n",
|
|
"spec = importlib.util.spec_from_file_location(\"mastodon\", \"../mastodon/__init__.py\")\n",
|
|
"\n",
|
|
"mastodon = importlib.util.module_from_spec(spec)\n",
|
|
"sys.modules[\"mastodon\"] = mastodon\n",
|
|
"spec.loader.exec_module(mastodon)\n",
|
|
"Mastodon = mastodon.Mastodon\n",
|
|
"print(mastodon.__file__)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Regular normal person imports\n",
|
|
"import json\n",
|
|
"from datetime import datetime, timedelta, timezone\n",
|
|
"import copy\n",
|
|
"from typing import List, Union\n",
|
|
"import pickle as pkl\n",
|
|
"\n",
|
|
"# Mastodon.py imports\n",
|
|
"from mastodon.types import *\n",
|
|
"from mastodon.types_base import real_issubclass"
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Setup"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"mastodon_soc = Mastodon(access_token=\"mastosoc_credentials.secret\", debug_requests=True)\n",
|
|
"mastodon_ico_admin = Mastodon(access_token = \"../../pytooter_usercred_ADMIN_DANGER.secret\", debug_requests=True)"
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Manual test zone"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Here you can test things manually during development\n",
|
|
"# results = {}\n",
|
|
"import pickle as pkl\n",
|
|
"#results = pkl.load(open(\"temp_entities.pkl\", 'rb'))\n",
|
|
"#mastodon_soc.status(110447003454258227)\n",
|
|
"#mastodon_soc.status(110447012773105565).media_attachments[0].meta.original"
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Entity verification"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"entities = json.load(open(\"return_types_all_current_fixed_bu.json\", \"r\"))\n",
|
|
"update_only = \"PreviewCardAuthor\"\n",
|
|
"update_only = None\n",
|
|
"\n",
|
|
"if update_only is None:\n",
|
|
" results = {}\n",
|
|
"for entity in entities:\n",
|
|
" name = entity[\"python_name\"]\n",
|
|
" if update_only is None and name in results:\n",
|
|
" continue\n",
|
|
" if not update_only is None and name != update_only:\n",
|
|
" continue\n",
|
|
" if entity.get(\"manual_update\") == True:\n",
|
|
" continue\n",
|
|
" func_call = entity.get(\"func_call_real\")\n",
|
|
" if func_call is None:\n",
|
|
" func_call = entity[\"func_call\"]\n",
|
|
" if func_call == \"TODO_TO_BE_IMPLEMENTED\":\n",
|
|
" continue\n",
|
|
" mastodon = mastodon_soc\n",
|
|
" if entity.get(\"func_alternate_acc\") == True:\n",
|
|
" mastodon = mastodon_ico_admin\n",
|
|
" print(\"Checking\", name)\n",
|
|
" print(\" *\", func_call)\n",
|
|
" results[name] = [eval(func_call)]\n",
|
|
" func_call = entity.get(\"func_call_additional\")\n",
|
|
" if not func_call is None:\n",
|
|
" print(\" *\", func_call)\n",
|
|
" results[name].append(eval(func_call))\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"entities = json.load(open(\"return_types_all_current_fixed_bu.json\", \"r\"))\n",
|
|
"\n",
|
|
"entities_by_name = {}\n",
|
|
"for entity in entities:\n",
|
|
" entities_by_name[entity[\"python_name\"]] = entity\n",
|
|
"\n",
|
|
"for result_name in results:\n",
|
|
" entity = entities_by_name[result_name]\n",
|
|
" entity_fields = entity[\"fields\"]\n",
|
|
" field_types_ok = {}\n",
|
|
" field_types_found = {}\n",
|
|
" for result in results[result_name]:\n",
|
|
" for field in result:\n",
|
|
" if not field in field_types_ok:\n",
|
|
" field_types_ok[field] = True\n",
|
|
" field_types_found[field] = []\n",
|
|
" entity_field = entity_fields.get(field)\n",
|
|
" if entity_field is None:\n",
|
|
" entity_field = entity_fields.get(field.replace(\"_\", \":\")) # hack for fields with colons. the actual json has this documented, but we don't care here\n",
|
|
" if entity_field is None:\n",
|
|
" print(result_name + \":\", field, \"not documented\")\n",
|
|
" continue\n",
|
|
" if result[field] is None and not (entity_field[\"is_nullable\"] or entity_field[\"is_optional\"]):\n",
|
|
" print(result_name + \":\", field, \"documented as not nullable/optional but is None\") \n",
|
|
" else:\n",
|
|
" field_types_found[field].append(type(result[field]))\n",
|
|
" try:\n",
|
|
" if not real_issubclass(type(result[field]), eval(entity_field[\"field_type\"])):\n",
|
|
" if not (entity_field[\"is_nullable\"] or entity_field[\"is_optional\"]) and result[field] is None:\n",
|
|
" field_types_ok[field] = False\n",
|
|
" except Exception as e:\n",
|
|
" field_types_ok[field] = False\n",
|
|
" for field in field_types_ok:\n",
|
|
" if not field_types_ok[field]:\n",
|
|
" if not set(field_types_found[field]) == set([type(None)]):\n",
|
|
" entity_fields_real = set(field_types_found[field]) - set([type(None)])\n",
|
|
" if not (entity_fields[field][\"field_type\"] == \"EntityList\" and len(entity_fields_real) == 1 and list(entity_fields_real)[0] == NonPaginatableList):\n",
|
|
" print(result_name + \":\", field, \"documented as\", entity_fields[field][\"field_type\"], \"but does not parse as such in all cases (found types:\", field_types_found[field], \")\")\n",
|
|
" if set(field_types_found[field]) == set(str(type(None))):\n",
|
|
" print(result_name + \":\", field, \"documented as\", entity_fields[field][\"field_type\"], \"but only found as None\")\n",
|
|
"for entity_name in entities_by_name:\n",
|
|
" entity = entities_by_name[entity_name]\n",
|
|
" entity_fields = entity[\"fields\"]\n",
|
|
" if not entity_name in results:\n",
|
|
" print(\"entity\", entity_name + \":\", \"documented but never retrieved\")\n",
|
|
" continue\n",
|
|
" for field in entity_fields:\n",
|
|
" found = False\n",
|
|
" for result in results[entity_name]:\n",
|
|
" if field in result:\n",
|
|
" found = True\n",
|
|
" else:\n",
|
|
" if not entity_fields[field][\"is_optional\"] and not entity_fields[field].get(\"api_version\") is None:\n",
|
|
" print(entity_name + \": field\", field, \"documented as not optional but missing from some retrieved entities\")\n",
|
|
" if not found:\n",
|
|
" print(entity_name + \": field\", field, \"documented but missing from all retrieved entities\")\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"mastodon_soc.featured_tags()[0]"
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### JSON normalization"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"entities = json.load(open(\"return_types_all_current_fixed_bu.json\", \"r\"))\n",
|
|
"for entity in entities:\n",
|
|
" print(entity)"
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Python generation"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from mastodon.utility import max_version\n",
|
|
"\n",
|
|
"def add_dot(str):\n",
|
|
" if str[-1] == \".\":\n",
|
|
" return str\n",
|
|
" return str + \".\"\n",
|
|
"\n",
|
|
"entities = json.load(open(\"return_types_all_current_fixed_bu.json\", \"r\"))\n",
|
|
"\n",
|
|
"all_entities_text = \"\"\n",
|
|
"for entity in entities:\n",
|
|
" entity_version = \"0.0.0\"\n",
|
|
" all_entities_text += f\"class {entity['python_name']}(AttribAccessDict):\\n\"\n",
|
|
" all_entities_text += f\" \\\"\\\"\\\"\\n {add_dot(entity['description'])}\\n\\n\"\n",
|
|
" all_entities_text += f\" See also (Mastodon API documentation): {entity['masto_doc_link']}\\n\"\n",
|
|
" all_entities_text += f\" \\\"\\\"\\\"\\n\"\n",
|
|
" all_entities_text += \"\\n\"\n",
|
|
" rename_map = {}\n",
|
|
" access_map = {}\n",
|
|
" for field in entity[\"fields\"]:\n",
|
|
" if \"moved_path\" in entity[\"fields\"][field]:\n",
|
|
" access_map[field] = entity[\"fields\"][field][\"moved_path\"]\n",
|
|
" field_name = field\n",
|
|
" if \"python_name\" in entity[\"fields\"][field]:\n",
|
|
" field_name = entity[\"fields\"][field][\"python_name\"]\n",
|
|
" rename_map[field] = field_name\n",
|
|
" type_str = entity[\"fields\"][field][\"field_type\"]\n",
|
|
" if entity[\"fields\"][field][\"field_subtype\"] is not None:\n",
|
|
" type_str += f\"[{entity['fields'][field]['field_subtype']}]\"\n",
|
|
" if entity[\"fields\"][field][\"is_optional\"] or entity[\"fields\"][field][\"is_nullable\"]:\n",
|
|
" type_str = f\"Optional[{type_str}]\"\n",
|
|
" type_str = f\"\\\"{type_str}\\\"\" \n",
|
|
" all_entities_text += f\" {field_name}: {type_str}\\n\"\n",
|
|
" all_entities_text += f\" \\\"\\\"\\\"\\n\"\n",
|
|
" if \"is_deprecated\" in entity[\"fields\"][field] and entity[\"fields\"][field][\"is_deprecated\"] == True:\n",
|
|
" all_entities_text += f\" THIS FIELD IS DEPRECATED. IT IS RECOMMENDED THAT YOU DO NOT USE IT.\\n\\n\"\n",
|
|
" all_entities_text += f\" {add_dot(entity['fields'][field]['description'])}\"\n",
|
|
" if entity[\"fields\"][field][\"is_optional\"]:\n",
|
|
" if entity[\"fields\"][field][\"is_nullable\"]:\n",
|
|
" all_entities_text += \" (optional, nullable)\"\n",
|
|
" else:\n",
|
|
" all_entities_text += \" (optional)\"\n",
|
|
" elif entity[\"fields\"][field][\"is_nullable\"]:\n",
|
|
" all_entities_text += \" (nullable)\"\n",
|
|
" all_entities_text += \"\\n\"\n",
|
|
" if entity[\"fields\"][field].get(\"field_structuretype\", None) is not None:\n",
|
|
" all_entities_text += f\" Should contain (as text): {entity['fields'][field]['field_structuretype']}\\n\"\n",
|
|
" all_entities_text += \"\\n Version history:\\n\"\n",
|
|
" for version, changed in entity[\"fields\"][field][\"version_history\"]:\n",
|
|
" entity_version = max_version(entity_version, version)\n",
|
|
" all_entities_text += f\" * {version}: {changed}\\n\"\n",
|
|
" all_entities_text += \" \\\"\\\"\\\"\\n\\n\"\n",
|
|
" all_entities_text += f\" _version = \\\"{entity_version}\\\"\\n\"\n",
|
|
" if len(rename_map) > 0:\n",
|
|
" all_entities_text += \" _rename_map = {\\n\"\n",
|
|
" for field in rename_map:\n",
|
|
" all_entities_text += f\" \\\"{rename_map[field]}\\\": \\\"{field}\\\",\\n\"\n",
|
|
" all_entities_text += \" }\\n\"\n",
|
|
" if len(access_map) > 0:\n",
|
|
" all_entities_text += \" _access_map = {\\n\"\n",
|
|
" for field in access_map:\n",
|
|
" all_entities_text += f\" \\\"{field}\\\": \\\"{access_map[field]}\\\",\\n\"\n",
|
|
" all_entities_text += \" }\\n\"\n",
|
|
" all_entities_text += \"\\n\"\n",
|
|
"print(\"\"\"from __future__ import annotations # python< 3.9 compat\n",
|
|
"from datetime import datetime\n",
|
|
"from typing import Union, Optional, Tuple, List, IO, Dict\n",
|
|
"from mastodon.types_base import AttribAccessDict, IdType, MaybeSnowflakeIdType, PaginationInfo, PrimitiveIdType, EntityList, PaginatableList, NonPaginatableList, PathOrFile, WebpushCryptoParamsPubkey, WebpushCryptoParamsPrivkey, try_cast_recurse, try_cast, real_issubclass\n",
|
|
"\"\"\") \n",
|
|
"print(all_entities_text)"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "mastopy_39",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.10.11"
|
|
},
|
|
"orig_nbformat": 4
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|