{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Super special from this directory for sure and guaranteed import\n",
    "import importlib.util\n",
    "import sys\n",
    "\n",
    "spec = importlib.util.spec_from_file_location(\"mastodon\", \"../mastodon/__init__.py\")\n",
    "\n",
    "mastodon = importlib.util.module_from_spec(spec)\n",
    "sys.modules[\"mastodon\"] = mastodon\n",
    "spec.loader.exec_module(mastodon)\n",
    "Mastodon = mastodon.Mastodon\n",
    "print(mastodon.__file__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Regular normal person imports\n",
    "import json\n",
    "from datetime import datetime, timedelta, timezone\n",
    "import copy\n",
    "from typing import List, Union\n",
    "import pickle as pkl\n",
    "\n",
    "# Mastodon.py imports\n",
    "from mastodon.types import *\n",
    "from mastodon.types_base import real_issubclass"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mastodon_soc = Mastodon(access_token=\"mastosoc_credentials.secret\", debug_requests=True)\n",
    "mastodon_ico_admin = Mastodon(access_token = \"../../pytooter_usercred_ADMIN_DANGER.secret\", debug_requests=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Manual test zone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Here you can test things manually during development\n",
    "# results = {}\n",
    "import pickle as pkl\n",
    "#results = pkl.load(open(\"temp_entities.pkl\", 'rb'))\n",
    "#mastodon_soc.status(110447003454258227)\n",
    "#mastodon_soc.status(110447012773105565).media_attachments[0].meta.original"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Entity verification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "entities = json.load(open(\"return_types_all_current_fixed_bu.json\", \"r\"))\n",
    "update_only = \"PreviewCardAuthor\"\n",
    "update_only = None\n",
    "\n",
    "if update_only is None:\n",
    "    results = {}\n",
    "for entity in entities:\n",
    "    name = entity[\"python_name\"]\n",
    "    if update_only is None and name in results:\n",
    "        continue\n",
    "    if not update_only is None and name != update_only:\n",
    "        continue\n",
    "    if entity.get(\"manual_update\") == True:\n",
    "        continue\n",
    "    func_call = entity.get(\"func_call_real\")\n",
    "    if func_call is None:\n",
    "        func_call = entity[\"func_call\"]\n",
    "    if func_call == \"TODO_TO_BE_IMPLEMENTED\":\n",
    "        continue\n",
    "    mastodon = mastodon_soc\n",
    "    if entity.get(\"func_alternate_acc\") == True:\n",
    "        mastodon = mastodon_ico_admin\n",
    "    print(\"Checking\", name)\n",
    "    print(\"    *\", func_call)\n",
    "    results[name] = [eval(func_call)]\n",
    "    func_call = entity.get(\"func_call_additional\")\n",
    "    if not func_call is None:\n",
    "        print(\"    *\", func_call)\n",
    "        results[name].append(eval(func_call))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "entities = json.load(open(\"return_types_all_current_fixed_bu.json\", \"r\"))\n",
    "\n",
    "entities_by_name = {}\n",
    "for entity in entities:\n",
    "    entities_by_name[entity[\"python_name\"]] = entity\n",
    "\n",
    "for result_name in results:\n",
    "    entity = entities_by_name[result_name]\n",
    "    entity_fields = entity[\"fields\"]\n",
    "    field_types_ok = {}\n",
    "    field_types_found = {}\n",
    "    for result in results[result_name]:\n",
    "        for field in result:\n",
    "            if not field in field_types_ok:\n",
    "                field_types_ok[field] = True\n",
    "                field_types_found[field] = []\n",
    "            entity_field = entity_fields.get(field)\n",
    "            if entity_field is None:\n",
    "                entity_field = entity_fields.get(field.replace(\"_\", \":\")) # hack for fields with colons. the actual json has this documented, but we don't care here\n",
    "            if entity_field is None:\n",
    "                print(result_name + \":\", field, \"not documented\")\n",
    "                continue\n",
    "            if result[field] is None and not (entity_field[\"is_nullable\"] or entity_field[\"is_optional\"]):\n",
    "                print(result_name + \":\", field, \"documented as not nullable/optional but is None\")                \n",
    "            else:\n",
    "                field_types_found[field].append(type(result[field]))\n",
    "                try:\n",
    "                    if not real_issubclass(type(result[field]), eval(entity_field[\"field_type\"])):\n",
    "                        if not (entity_field[\"is_nullable\"] or entity_field[\"is_optional\"]) and result[field] is None:\n",
    "                            field_types_ok[field] = False\n",
    "                except Exception as e:\n",
    "                    field_types_ok[field] = False\n",
    "    for field in field_types_ok:\n",
    "        if not field_types_ok[field]:\n",
    "            if not set(field_types_found[field]) == set([type(None)]):\n",
    "                entity_fields_real = set(field_types_found[field]) - set([type(None)])\n",
    "                if not (entity_fields[field][\"field_type\"] == \"EntityList\" and len(entity_fields_real) == 1 and list(entity_fields_real)[0] == NonPaginatableList):\n",
    "                    print(result_name + \":\", field, \"documented as\", entity_fields[field][\"field_type\"], \"but does not parse as such in all cases (found types:\", field_types_found[field], \")\")\n",
    "        if set(field_types_found[field]) == set(str(type(None))):\n",
    "            print(result_name + \":\", field, \"documented as\", entity_fields[field][\"field_type\"], \"but only found as None\")\n",
    "for entity_name in entities_by_name:\n",
    "    entity = entities_by_name[entity_name]\n",
    "    entity_fields = entity[\"fields\"]\n",
    "    if not entity_name in results:\n",
    "        print(\"entity\", entity_name + \":\", \"documented but never retrieved\")\n",
    "        continue\n",
    "    for field in entity_fields:\n",
    "        found = False\n",
    "        for result in results[entity_name]:\n",
    "            if field in result:\n",
    "                found = True\n",
    "            else:\n",
    "                if not entity_fields[field][\"is_optional\"] and not entity_fields[field].get(\"api_version\") is None:\n",
    "                    print(entity_name + \": field\", field, \"documented as not optional but missing from some retrieved entities\")\n",
    "        if not found:\n",
    "            print(entity_name + \": field\", field, \"documented but missing from all retrieved entities\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mastodon_soc.featured_tags()[0]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### JSON normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "entities = json.load(open(\"return_types_all_current_fixed_bu.json\", \"r\"))\n",
    "for entity in entities:\n",
    "    print(entity)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Python generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mastodon.utility import max_version\n",
    "\n",
    "def add_dot(str):\n",
    "    if str[-1] == \".\":\n",
    "        return str\n",
    "    return str + \".\"\n",
    "\n",
    "entities = json.load(open(\"return_types_all_current_fixed_bu.json\", \"r\"))\n",
    "\n",
    "all_entities_text = \"\"\n",
    "for entity in entities:\n",
    "    entity_version = \"0.0.0\"\n",
    "    all_entities_text += f\"class {entity['python_name']}(AttribAccessDict):\\n\"\n",
    "    all_entities_text += f\"    \\\"\\\"\\\"\\n    {add_dot(entity['description'])}\\n\\n\"\n",
    "    all_entities_text += f\"    See also (Mastodon API documentation): {entity['masto_doc_link']}\\n\"\n",
    "    all_entities_text += f\"    \\\"\\\"\\\"\\n\"\n",
    "    all_entities_text += \"\\n\"\n",
    "    rename_map = {}\n",
    "    access_map = {}\n",
    "    for field in entity[\"fields\"]:\n",
    "        if \"moved_path\" in entity[\"fields\"][field]:\n",
    "            access_map[field] = entity[\"fields\"][field][\"moved_path\"]\n",
    "        field_name = field\n",
    "        if \"python_name\" in entity[\"fields\"][field]:\n",
    "            field_name = entity[\"fields\"][field][\"python_name\"]\n",
    "            rename_map[field] = field_name\n",
    "        type_str = entity[\"fields\"][field][\"field_type\"]\n",
    "        if entity[\"fields\"][field][\"field_subtype\"] is not None:\n",
    "            type_str += f\"[{entity['fields'][field]['field_subtype']}]\"\n",
    "        if entity[\"fields\"][field][\"is_optional\"] or entity[\"fields\"][field][\"is_nullable\"]:\n",
    "            type_str = f\"Optional[{type_str}]\"\n",
    "        type_str = f\"\\\"{type_str}\\\"\"            \n",
    "        all_entities_text += f\"    {field_name}: {type_str}\\n\"\n",
    "        all_entities_text += f\"    \\\"\\\"\\\"\\n\"\n",
    "        if \"is_deprecated\" in entity[\"fields\"][field] and entity[\"fields\"][field][\"is_deprecated\"] == True:\n",
    "            all_entities_text += f\"    THIS FIELD IS DEPRECATED. IT IS RECOMMENDED THAT YOU DO NOT USE IT.\\n\\n\"\n",
    "        all_entities_text += f\"    {add_dot(entity['fields'][field]['description'])}\"\n",
    "        if entity[\"fields\"][field][\"is_optional\"]:\n",
    "            if entity[\"fields\"][field][\"is_nullable\"]:\n",
    "                all_entities_text += \" (optional, nullable)\"\n",
    "            else:\n",
    "                all_entities_text += \" (optional)\"\n",
    "        elif entity[\"fields\"][field][\"is_nullable\"]:\n",
    "            all_entities_text += \" (nullable)\"\n",
    "        all_entities_text += \"\\n\"\n",
    "        if entity[\"fields\"][field].get(\"field_structuretype\", None) is not None:\n",
    "            all_entities_text += f\"    Should contain (as text): {entity['fields'][field]['field_structuretype']}\\n\"\n",
    "        all_entities_text += \"\\n    Version history:\\n\"\n",
    "        for version, changed in entity[\"fields\"][field][\"version_history\"]:\n",
    "            entity_version = max_version(entity_version, version)\n",
    "            all_entities_text += f\"      * {version}: {changed}\\n\"\n",
    "        all_entities_text += \"    \\\"\\\"\\\"\\n\\n\"\n",
    "    all_entities_text += f\"    _version = \\\"{entity_version}\\\"\\n\"\n",
    "    if len(rename_map) > 0:\n",
    "        all_entities_text += \"    _rename_map = {\\n\"\n",
    "        for field in rename_map:\n",
    "            all_entities_text += f\"        \\\"{rename_map[field]}\\\": \\\"{field}\\\",\\n\"\n",
    "        all_entities_text += \"    }\\n\"\n",
    "    if len(access_map) > 0:\n",
    "        all_entities_text += \"    _access_map = {\\n\"\n",
    "        for field in access_map:\n",
    "            all_entities_text += f\"        \\\"{field}\\\": \\\"{access_map[field]}\\\",\\n\"\n",
    "        all_entities_text += \"    }\\n\"\n",
    "    all_entities_text += \"\\n\"\n",
    "print(\"\"\"from __future__ import annotations # python< 3.9 compat\n",
    "from datetime import datetime\n",
    "from typing import Union, Optional, Tuple, List, IO, Dict\n",
    "from mastodon.types_base import AttribAccessDict, IdType, MaybeSnowflakeIdType, PaginationInfo, PrimitiveIdType, EntityList, PaginatableList, NonPaginatableList, PathOrFile, WebpushCryptoParamsPubkey, WebpushCryptoParamsPrivkey, try_cast_recurse, try_cast, real_issubclass\n",
    "\"\"\")    \n",
    "print(all_entities_text)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mastopy_39",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}