Experimental opt-in description generator

2023-12-27 23:33:59 +08:00 · 2023-12-27 23:33:59 +08:00 · fe54eb11a7
commit fe54eb11a7
--- a/src/components/compose.css
+++ b/src/components/compose.css
@ -501,6 +501,19 @@
    padding-inline: 24px;
  }
 }
+
+@keyframes breathe {
+  0% {
+    opacity: 1;
+  }
+  40% {
+    opacity: 0.4;
+  }
+  100% {
+    opacity: 1;
+  }
+}
+
 #media-sheet {
  .media-form {
    flex: 1;
@ -514,6 +527,10 @@
      resize: none;
      width: 100%;
      /* height: 10em; */
+
+      &.loading {
+        animation: skeleton-breathe 1.5s linear infinite;
+      }
    }

    footer {
--- a/src/components/compose.jsx
+++ b/src/components/compose.jsx
@ -1,6 +1,7 @@
 import './compose.css';

 import '@github/text-expander-element';
+import { MenuItem } from '@szhsin/react-menu';
 import equal from 'fast-deep-equal';
 import { forwardRef } from 'preact/compat';
 import { useEffect, useMemo, useRef, useState } from 'preact/hooks';
@ -11,6 +12,7 @@ import { uid } from 'uid/single';
 import { useDebouncedCallback, useThrottledCallback } from 'use-debounce';
 import { useSnapshot } from 'valtio';

+import Menu2 from '../components/menu2';
 import supportedLanguages from '../data/status-supported-languages';
 import urlRegex from '../data/url-regex';
 import { api } from '../utils/api';
@ -19,6 +21,7 @@ import emojifyText from '../utils/emojify-text';
 import localeMatch from '../utils/locale-match';
 import openCompose from '../utils/open-compose';
 import shortenNumber from '../utils/shorten-number';
+import showToast from '../utils/show-toast';
 import states, { saveStatus } from '../utils/states';
 import store from '../utils/store';
 import {
@ -39,6 +42,8 @@ import Loader from './loader';
 import Modal from './modal';
 import Status from './status';

+const { PHANPY_IMG_ALT_API_URL: IMG_ALT_API_URL } = import.meta.env;
+
 const supportedLanguagesMap = supportedLanguages.reduce((acc, l) => {
  const [code, common, native] = l;
  acc[code] = {
@ -1291,7 +1296,7 @@ const Textarea = forwardRef((props, ref) => {
  const { masto } = api();
  const [text, setText] = useState(ref.current?.value || '');
  const { maxCharacters, performSearch = () => {}, ...textareaProps } = props;
-  const snapStates = useSnapshot(states);
+  // const snapStates = useSnapshot(states);
  // const charCount = snapStates.composerCharacterCount;

  const customEmojis = useRef();
@ -1645,6 +1650,7 @@ function MediaAttachment({
  onDescriptionChange = () => {},
  onRemove = () => {},
 }) {
+  const [uiState, setUIState] = useState('default');
  const supportsEdit = supports('@mastodon/edit-media-attributes');
  const { type, id, file } = attachment;
  const url = useMemo(
@ -1653,7 +1659,7 @@ function MediaAttachment({
  );
  console.log({ attachment });
  const [description, setDescription] = useState(attachment.description);
-  const suffixType = type.split('/')[0];
+  const [suffixType, subtype] = type.split('/');
  const debouncedOnDescriptionChange = useDebouncedCallback(
    onDescriptionChange,
    250,
@ -1699,7 +1705,8 @@ function MediaAttachment({
          autoCorrect="on"
          spellCheck="true"
          dir="auto"
-          disabled={disabled}
+          disabled={disabled || uiState === 'loading'}
+          class={uiState === 'loading' ? 'loading' : ''}
          maxlength="1500" // Not unicode-aware :(
          // TODO: Un-hard-code this maxlength, ref: https://github.com/mastodon/mastodon/blob/b59fb28e90bc21d6fd1a6bafd13cfbd81ab5be54/app/models/media_attachment.rb#L39
          onInput={(e) => {
@ -1712,6 +1719,13 @@ function MediaAttachment({
    </>
  );

+  const toastRef = useRef(null);
+  useEffect(() => {
+    return () => {
+      toastRef.current?.hideToast?.();
+    };
+  }, []);
+
  return (
    <>
      <div class="media-attachment">
@ -1785,12 +1799,68 @@ function MediaAttachment({
              <div class="media-form">
                {descTextarea}
                <footer>
+                  {suffixType === 'image' &&
+                    /^(png|jpe?g|gif|webp)$/i.test(subtype) &&
+                    !!states.settings.mediaAltGenerator &&
+                    !!IMG_ALT_API_URL && (
+                      <Menu2
+                        portal={{
+                          target: document.body,
+                        }}
+                        containerProps={{
+                          style: {
+                            zIndex: 1001,
+                          },
+                        }}
+                        align="center"
+                        position="anchor"
+                        overflow="auto"
+                        menuButton={
+                          <button type="button" title="More" class="plain">
+                            <Icon icon="more" size="l" alt="More" />
+                          </button>
+                        }
+                      >
+                        <MenuItem
+                          disabled={uiState === 'loading'}
+                          onClick={() => {
+                            setUIState('loading');
+                            toastRef.current = showToast({
+                              text: 'Generating description. Please wait...',
+                              duration: -1,
+                            });
+                            // POST with multipart
+                            (async function () {
+                              try {
+                                const body = new FormData();
+                                body.append('image', file);
+                                const response = await fetch(IMG_ALT_API_URL, {
+                                  method: 'POST',
+                                  body,
+                                }).then((r) => r.json());
+                                setDescription(response.description);
+                              } catch (e) {
+                                console.error(e);
+                                showToast('Failed to generate description');
+                              } finally {
+                                setUIState('default');
+                                toastRef.current?.hideToast?.();
+                              }
+                            })();
+                          }}
+                        >
+                          <Icon icon="sparkles2" />
+                          <span>Generate description…</span>
+                        </MenuItem>
+                      </Menu2>
+                    )}
                  <button
                    type="button"
                    class="light block"
                    onClick={() => {
                      setShowModal(false);
                    }}
+                    disabled={uiState === 'loading'}
                  >
                    Done
                  </button>
--- a/src/components/icon.jsx
+++ b/src/components/icon.jsx
@ -69,6 +69,7 @@ export const ICONS = {
  history: () => import('@iconify-icons/mingcute/history-line'),
  share: () => import('@iconify-icons/mingcute/share-2-line'),
  sparkles: () => import('@iconify-icons/mingcute/sparkles-line'),
+  sparkles2: () => import('@iconify-icons/mingcute/sparkles-2-line'),
  exit: () => import('@iconify-icons/mingcute/exit-line'),
  translate: () => import('@iconify-icons/mingcute/translate-line'),
  play: () => import('@iconify-icons/mingcute/play-fill'),
--- a/src/components/media-modal.jsx
+++ b/src/components/media-modal.jsx
@ -1,4 +1,4 @@
-import { Menu } from '@szhsin/react-menu';
+import { MenuDivider, MenuItem } from '@szhsin/react-menu';
 import { getBlurHashAverageColor } from 'fast-blurhash';
 import {
  useEffect,
@ -10,6 +10,7 @@ import {
 import { useHotkeys } from 'react-hotkeys-hook';

 import { oklab2rgb, rgb2oklab } from '../utils/color-utils';
+import showToast from '../utils/show-toast';
 import states from '../utils/states';

 import Icon from './icon';
@ -18,6 +19,8 @@ import Media from './media';
 import Menu2 from './menu2';
 import MenuLink from './menu-link';

+const { PHANPY_IMG_ALT_API_URL: IMG_ALT_API_URL } = import.meta.env;
+
 function MediaModal({
  mediaAttachments,
  statusID,
@ -26,6 +29,7 @@ function MediaModal({
  index = 0,
  onClose = () => {},
 }) {
+  const [uiState, setUIState] = useState('default');
  const carouselRef = useRef(null);

  const [currentIndex, setCurrentIndex] = useState(index);
@ -144,6 +148,13 @@ function MediaModal({
    );
  }, [mediaAccentColors]);

+  let toastRef = useRef(null);
+  useEffect(() => {
+    return () => {
+      toastRef.current?.hideToast?.();
+    };
+  }, []);
+
  return (
    <div
      class={`media-modal-container media-modal-count-${mediaAttachments?.length}`}
@ -284,6 +295,47 @@ function MediaModal({
              <Icon icon="popout" />
              <span>Open original media</span>
            </MenuLink>
+            {import.meta.env.DEV && // Only dev for now
+              !!states.settings.mediaAltGenerator &&
+              !!IMG_ALT_API_URL &&
+              !!mediaAttachments[currentIndex]?.url &&
+              !mediaAttachments[currentIndex]?.description &&
+              mediaAttachments[currentIndex]?.type === 'image' && (
+                <>
+                  <MenuDivider />
+                  <MenuItem
+                    disabled={uiState === 'loading'}
+                    onClick={() => {
+                      setUIState('loading');
+                      toastRef.current = showToast({
+                        text: 'Attempting to describe image. Please wait...',
+                        duration: -1,
+                      });
+                      (async function () {
+                        try {
+                          const response = await fetch(
+                            `${IMG_ALT_API_URL}?image=${encodeURIComponent(
+                              mediaAttachments[currentIndex]?.url,
+                            )}`,
+                          ).then((r) => r.json());
+                          states.showMediaAlt = {
+                            alt: response.description,
+                          };
+                        } catch (e) {
+                          console.error(e);
+                          showToast('Failed to describe image');
+                        } finally {
+                          setUIState('default');
+                          toastRef.current?.hideToast?.();
+                        }
+                      })();
+                    }}
+                  >
+                    <Icon icon="sparkles2" />
+                    <span>Describe image…</span>
+                  </MenuItem>
+                </>
+              )}
          </Menu2>{' '}
          <Link
            to={`${instance ? `/${instance}` : ''}/s/${statusID}${
--- a/src/index.css
+++ b/src/index.css
@ -338,6 +338,9 @@ button.large {
  font-size: 125%;
  padding: 12px;
 }
+textarea:disabled {
+  background-color: var(--bg-faded-color);
+}

 :is(input[type='text'], textarea, select).block {
  display: block;
--- a/src/pages/settings.jsx
+++ b/src/pages/settings.jsx
@ -27,6 +27,7 @@ const TEXT_SIZES = [15, 16, 17, 18, 19, 20];
 const {
  PHANPY_WEBSITE: WEBSITE,
  PHANPY_PRIVACY_POLICY_URL: PRIVACY_POLICY_URL,
+  PHANPY_IMG_ALT_API_URL: IMG_ALT_API_URL,
 } = import.meta.env;

 function Settings({ onClose }) {
@ -432,6 +433,34 @@ function Settings({ onClose }) {
                </div>
              </div>
            </li>
+            {!!IMG_ALT_API_URL && (
+              <li>
+                <label>
+                  <input
+                    type="checkbox"
+                    checked={snapStates.settings.mediaAltGenerator}
+                    onChange={(e) => {
+                      states.settings.mediaAltGenerator = e.target.checked;
+                    }}
+                  />{' '}
+                  Image description generator{' '}
+                  <Icon icon="sparkles2" class="more-insignificant" />
+                </label>
+                <div class="sub-section insignificant">
+                  <small>
+                    Note: This feature uses external AI service, powered by{' '}
+                    <a
+                      href="https://github.com/cheeaun/img-alt-api"
+                      target="_blank"
+                      rel="noopener noreferrer"
+                    >
+                      img-alt-api
+                    </a>
+                    . May not work well. Only for images and in English.
+                  </small>
+                </div>
+              </li>
+            )}
            <li>
              <label>
                <input
--- a/src/utils/show-toast.js
+++ b/src/utils/show-toast.js
@ -23,6 +23,7 @@ function showToast(props) {
  } else {
    toast.showToast();
  }
+  return toast;
 }

 export default showToast;
--- a/src/utils/states.js
+++ b/src/utils/states.js
@ -59,6 +59,7 @@ const states = proxy({
    contentTranslationTargetLanguage: null,
    contentTranslationHideLanguages: [],
    contentTranslationAutoInline: false,
+    mediaAltGenerator: false,
    cloakMode: false,
  },
 });
@ -87,6 +88,8 @@ export function initStates() {
    store.account.get('settings-contentTranslationHideLanguages') || [];
  states.settings.contentTranslationAutoInline =
    store.account.get('settings-contentTranslationAutoInline') ?? false;
+  states.settings.mediaAltGenerator =
+    store.account.get('settings-mediaAltGenerator') ?? false;
  states.settings.cloakMode = store.account.get('settings-cloakMode') ?? false;
 }

@ -122,6 +125,9 @@ subscribe(states, (changes) => {
        states.settings.contentTranslationHideLanguages,
      );
    }
+    if (path.join('.') === 'settings.mediaAltGenerator') {
+      store.account.set('settings-mediaAltGenerator', !!value);
+    }
    if (path?.[0] === 'shortcuts') {
      store.account.set('shortcuts', states.shortcuts);
    }