feat(media): Blurhash (#1381)

* chore(npm): Install blurhash

* feat(media): Show blurhash

* fix(media/blurhash): Better sensitive video handling

* feat(media): Preference for using blurhash

* chore(utils/blurhash): Add performance marks

* fix(utils/blurhash): Performance marks

* fix(utils/blurhash): Use correct dimension

* refactor(utils/blurhash): Use constant for number of pixels

* refactor(media): Simplify logic for displaying blurhash

* chore(tests/spec): Attempt to adjust sensitivity tests for blurhash

* chore(tests/spec): Update sensitivity tests for blurhash

* chore(tests/spec): Check for sensitive

* fix(media/blurhash): Handle videos

* fix: Video handling

* fix: Videos

* minor refactoring, fix Svelte warning

* fix: Large inline images and videos

* feat(settings): Rename blurhash setting

* refactor: Use toBlob, block media rendering until blurhash ready

* refactor: Move computations to Web Worker

* fix(workers/blurhash): More error handling

* feat(workers/blurhash): Use quick-lru for caching

* fix: Don't create Context2D needlessly

* fix(workers/blurhash): Increase cache size to 100

* fix(workers/blurhash): Don't resolve promise twice

* fix(utils/decode-image): Ignore data URLs

Throws exception which prevents the image from loading.
blurhash-color
Sorin Davidoi 2019-08-17 20:54:45 +03:00 zatwierdzone przez Nolan Lawson
rodzic d52049cca5
commit 77bb784efd
15 zmienionych plików z 221 dodań i 33 usunięć

Wyświetl plik

@ -48,6 +48,7 @@
"@webcomponents/custom-elements": "^1.2.4",
"babel-loader": "^8.0.6",
"babel-plugin-transform-react-remove-prop-types": "^0.4.24",
"blurhash": "^1.1.3",
"cheerio": "^1.0.0-rc.2",
"child-process-promise": "^2.2.1",
"chokidar": "^3.0.1",
@ -109,7 +110,8 @@
"mocha": "^6.1.4",
"now": "^15.7.0",
"standard": "^13.1.0",
"testcafe": "^1.2.1"
"testcafe": "^1.2.1",
"worker-loader": "^2.0.0"
},
"engines": {
"node": ">= 8"
@ -150,7 +152,10 @@
"customElements",
"AbortController",
"matchMedia",
"MessageChannel"
"MessageChannel",
"ImageData",
"OffscreenCanvas",
"postMessage"
],
"ignore": [
"dist",

Wyświetl plik

@ -53,7 +53,8 @@
height: void 0,
ariaHidden: false,
alt: '',
title: ''
title: '',
blurhash: void 0
}),
computed: {
computedStyle: ({ background }) => {
@ -71,7 +72,7 @@
return `object-position: ${coordsToPercent(focus.x)}% ${100 - coordsToPercent(focus.y)}%;`
},
fillFixSize: ({ forceSize, $largeInlineMedia }) => !$largeInlineMedia && !forceSize,
displaySrc: ({ error, src, fallback }) => ((error && fallback) || src)
displaySrc: ({ blurhash, error, src, fallback }) => (blurhash || (error && fallback) || src)
}
}
</script>

Wyświetl plik

@ -17,6 +17,7 @@
alt={label || ''}
title={label || ''}
src={staticSrc}
blurhash={blurhash}
fallback={oneTransparentPixel}
{width}
{height}
@ -24,7 +25,9 @@
{focus}
/>
{/if}
<PlayVideoIcon className={playing ? 'hidden' : ''}/>
{#if !blurhash}
<PlayVideoIcon className={playing ? 'hidden' : ''}/>
{/if}
</div>
<style>
.non-autoplay-gifv {

Wyświetl plik

@ -1,23 +1,40 @@
{#if type === 'video' || type === 'audio'}
<button id={elementId}
type="button"
class="play-video-button focus-after {$largeInlineMedia ? '' : 'fixed-size'} {type === 'audio' ? 'play-audio-button' : ''}"
aria-label="Play video: {description}"
style="width: {inlineWidth}px; height: {inlineHeight}px;">
<PlayVideoIcon />
{#if blurhash}
{#if type === 'video'}
<LazyImage
alt={description}
title={description}
src={previewUrl}
fallback={oneTransparentPixel}
blurhash={blurhash}
width={inlineWidth}
height={inlineHeight}
background="var(--loading-bg)"
{focus}
/>
{/if}
</button>
{:else}
<button id={elementId}
type="button"
class="play-video-button focus-after {$largeInlineMedia ? '' : 'fixed-size'} {type === 'audio' ? 'play-audio-button' : ''}"
aria-label="Play video: {description}"
style="width: {inlineWidth}px; height: {inlineHeight}px;">
<PlayVideoIcon />
{#if type === 'video'}
<LazyImage
alt={description}
title={description}
src={previewUrl}
fallback={oneTransparentPixel}
blurhash={blurhash}
width={inlineWidth}
height={inlineHeight}
background="var(--loading-bg)"
{focus}
/>
{/if}
</button>
{/if}
{:else}
<button id={elementId}
type="button"
@ -40,6 +57,7 @@
class={noNativeWidthHeight ? 'no-native-width-height' : ''}
label="Animated GIF: {description}"
poster={previewUrl}
blurhash={blurhash}
src={url}
staticSrc={previewUrl}
width={inlineWidth}
@ -53,6 +71,7 @@
title={description}
src={previewUrl}
fallback={oneTransparentPixel}
blurhash={blurhash}
width={inlineWidth}
height={inlineHeight}
background="var(--loading-bg)"
@ -91,11 +110,16 @@
import LazyImage from '../LazyImage.html'
import AutoplayVideo from '../AutoplayVideo.html'
import { registerClickDelegate } from '../../_utils/delegate'
import { decode } from '../../_utils/blurhash'
export default {
oncreate () {
const { elementId } = this.get()
export default {
async oncreate () {
const { elementId, media } = this.get()
registerClickDelegate(this, elementId, () => this.onClick())
if (media.blurhash) {
this.set({ decodedBlurhash: await decode(media.blurhash) })
}
},
computed: {
focus: ({ meta }) => meta && meta.focus,
@ -126,6 +150,7 @@
elementId: ({ media, uuid }) => `media-${uuid}-${media.id}`,
description: ({ media }) => media.description || '',
previewUrl: ({ media }) => media.preview_url,
blurhash: ({ showBlurhash, decodedBlurhash }) => showBlurhash && decodedBlurhash,
url: ({ media }) => media.url,
type: ({ media }) => media.type
},
@ -141,6 +166,7 @@
},
data: () => ({
oneTransparentPixel: ONE_TRANSPARENT_PIXEL,
decodedBlurhash: ONE_TRANSPARENT_PIXEL,
mouseover: void 0
}),
store: () => store,

Wyświetl plik

@ -1,7 +1,7 @@
<div class={computedClass}
style="grid-template-columns: repeat({nCols}, 1fr);" >
{#each mediaAttachments as media, index}
<Media {media} {uuid} {mediaAttachments} {index} />
<Media {media} {uuid} {mediaAttachments} {index} {showBlurhash} />
{/each}
</div>
<style>
@ -39,6 +39,7 @@
}
.status-media.status-media-is-sensitive {
height: inherit;
margin: 0;
}
</style>
@ -55,6 +56,10 @@
twoCols && 'two-cols',
!$largeInlineMedia && 'grouped-images'
),
showBlurhash:
({ sensitive, sensitiveShown, mediaAttachments }) => {
return sensitive && mediaAttachments.every(attachment => !!attachment.blurhash) ? !sensitiveShown : false
},
nCols:
({ mediaAttachments, $largeInlineMedia }) => {
return (!$largeInlineMedia && mediaAttachments.length > 1) ? 2 : 1

Wyświetl plik

@ -10,28 +10,32 @@
<SvgIcon className="status-sensitive-media-svg" href="#fa-eye-slash" />
</div>
</button>
<MediaAttachments {mediaAttachments} {sensitive} {uuid} />
{:else}
<button id={elementId}
type="button"
class="status-sensitive-media-button"
aria-label="Show sensitive media" >
<div class="status-sensitive-media-warning">
Sensitive content. Click to show.
<div class="{customWarningClass}">
<div class="status-sensitive-media-warning-text">
Sensitive content. Click to show.
</div>
</div>
<div class="svg-wrapper">
<SvgIcon className="status-sensitive-media-svg" href="#fa-eye" />
</div>
</button>
{/if}
{#if sensitiveShown || canUseBlurhash}
<MediaAttachments {mediaAttachments} {sensitive} {sensitiveShown} {uuid} />
{/if}
</div>
</div>
{#if enableShortcuts}
<Shortcut scope={shortcutScope} key="y" on:pressed="toggleSensitiveMedia()"/>
{/if}
{:else}
<MediaAttachments {mediaAttachments} {sensitive} {uuid} />
<MediaAttachments {mediaAttachments} {sensitive} {sensitiveShown} {uuid} />
{/if}
<style>
.status-sensitive-media-container {
@ -81,6 +85,7 @@
}
.status-sensitive-media-hidden .status-sensitive-media-button {
position: absolute;
right: 0;
bottom: 0;
width: 100%;
@ -96,7 +101,6 @@
}
.status-sensitive-media-container .status-sensitive-media-warning {
position: absolute;
top: 0;
left: 0;
right: 0;
@ -109,6 +113,21 @@
padding: 0 10px;
}
.status-sensitive-media-container .status-sensitive-media-warning-transparent {
position: absolute;
}
.status-sensitive-media-container .status-sensitive-media-warning-opaque {
background: var(--mask-bg);
height: 100%;
}
.status-sensitive-media-container .status-sensitive-media-warning-transparent .status-sensitive-media-warning-text {
background: var(--mask-bg);
padding: 10px;
border-radius: 6px;
}
.status-sensitive-media-container .svg-wrapper {
display: flex;
align-items: flex-start;
@ -119,6 +138,7 @@
}
.status-sensitive-media-hidden .svg-wrapper {
position: absolute;
background: none;
top: 0;
left: 0;
right: 0;
@ -171,6 +191,7 @@
$largeInlineMedia ? 'not-grouped-images' : 'grouped-images'
),
mediaAttachments: ({ originalStatus }) => originalStatus.media_attachments,
canUseBlurhash: ({ $ignoreBlurhash, mediaAttachments }) => !$ignoreBlurhash && mediaAttachments && mediaAttachments.every(media => !!media.blurhash),
sensitiveShown: ({ $sensitivesShown, uuid }) => !!$sensitivesShown[uuid],
sensitive: ({ originalStatus, $markMediaAsSensitive, $neverMarkMediaAsSensitive }) => (
!$neverMarkMediaAsSensitive && ($markMediaAsSensitive || originalStatus.sensitive)
@ -181,7 +202,11 @@
return ''
}
return `padding-bottom: ${Math.ceil(mediaAttachments.length / 2) * 29}%;`
}
},
customWarningClass: ({ canUseBlurhash }) => classname(
'status-sensitive-media-warning',
canUseBlurhash ? 'status-sensitive-media-warning-transparent' : 'status-sensitive-media-warning-opaque'
)
},
methods: {
toggleSensitiveMedia () {

Wyświetl plik

@ -8,6 +8,11 @@
bind:checked="$neverMarkMediaAsSensitive" on:change="onChange(event)">
<label for="choice-never-mark-media-sensitive">Show sensitive media by default</label>
</div>
<div class="setting-group">
<input type="checkbox" id="choice-use-blurhash"
bind:checked="$ignoreBlurhash" on:change="onChange(event)">
<label for="choice-use-blurhash">Show a plain gray color for sensitive media</label>
</div>
<div class="setting-group">
<input type="checkbox" id="choice-mark-media-sensitive"
bind:checked="$markMediaAsSensitive" on:change="onChange(event)">

Wyświetl plik

@ -30,6 +30,7 @@ const persistedState = {
loggedInInstancesInOrder: [],
markMediaAsSensitive: false,
neverMarkMediaAsSensitive: false,
ignoreBlurhash: false,
omitEmojiInDisplayNames: undefined,
pinnedPages: {},
pushSubscriptions: {},

Wyświetl plik

@ -0,0 +1,51 @@
import BlurhashWorker from 'worker-loader!../_workers/blurhash' // eslint-disable-line
const RESOLUTION = 32
let worker
let canvas
let canvasContext2D
export function init () {
worker = worker || new BlurhashWorker()
}
export async function decode (blurhash) {
return new Promise((resolve, reject) => {
try {
init()
const onMessage = ({ data: { encoded, decoded, imageData, error } }) => {
if (encoded !== blurhash) {
return
}
worker.removeEventListener('message', onMessage)
if (error) {
return reject(error)
}
if (decoded) {
resolve(decoded)
} else {
if (!canvas) {
canvas = document.createElement('canvas')
canvas.height = RESOLUTION
canvas.width = RESOLUTION
canvasContext2D = canvas.getContext('2d')
}
canvasContext2D.putImageData(imageData, 0, 0)
canvas.toBlob(blob => {
resolve(URL.createObjectURL(blob))
})
}
}
worker.addEventListener('message', onMessage)
worker.postMessage({ encoded: blurhash })
} catch (e) {
reject(e)
}
})
}

Wyświetl plik

@ -4,7 +4,7 @@ export function decodeImage (img) {
// Remove this UA sniff when the Firefox bug is fixed
// https://github.com/nolanlawson/pinafore/issues/1344#issuecomment-514312672
// https://bugzilla.mozilla.org/show_bug.cgi?id=1565542
if (!IS_FIREFOX && typeof img.decode === 'function') {
if (!IS_FIREFOX && typeof img.decode === 'function' && !img.src.startsWith('data:image/png;base64,')) {
return img.decode()
}

Wyświetl plik

@ -0,0 +1,45 @@
import { decode as decodeBlurHash } from 'blurhash'
import QuickLRU from 'quick-lru'
const RESOLUTION = 32
const OFFSCREEN_CANVAS = typeof OffscreenCanvas === 'function'
? new OffscreenCanvas(RESOLUTION, RESOLUTION) : null
const OFFSCREEN_CANVAS_CONTEXT_2D = OFFSCREEN_CANVAS
? OFFSCREEN_CANVAS.getContext('2d') : null
const CACHE = new QuickLRU({ maxSize: 100 })
self.addEventListener('message', ({ data: { encoded } }) => {
try {
if (CACHE.has(encoded)) {
if (OFFSCREEN_CANVAS) {
postMessage({ encoded, decoded: CACHE.get(encoded), imageData: null, error: null })
} else {
postMessage({ encoded, imageData: CACHE.get(encoded), decoded: null, error: null })
}
} else {
const pixels = decodeBlurHash(encoded, RESOLUTION, RESOLUTION)
if (pixels) {
const imageData = new ImageData(pixels, RESOLUTION, RESOLUTION)
if (OFFSCREEN_CANVAS) {
OFFSCREEN_CANVAS_CONTEXT_2D.putImageData(imageData, 0, 0)
OFFSCREEN_CANVAS.convertToBlob().then(blob => {
const decoded = URL.createObjectURL(blob)
CACHE.set(encoded, decoded)
postMessage({ encoded, decoded, imageData: null, error: null })
}).catch(error => {
postMessage({ encoded, decoded: null, imageData: null, error })
})
} else {
CACHE.set(encoded, imageData)
postMessage({ encoded, imageData, decoded: null, error: null })
}
} else {
postMessage({ encoded, decoded: null, imageData: null, error: new Error('decode did not return any pixels') })
}
}
} catch (error) {
postMessage({ encoded, decoded: null, imageData: null, error })
}
})

Wyświetl plik

@ -13,10 +13,10 @@ test('shows sensitive images and videos', async t => {
const videoIdx = homeTimeline.findIndex(_ => _.content === 'secret video')
await scrollToStatus(t, 1 + kittenIdx)
await t.expect($(`${getNthStatusSelector(1 + kittenIdx)} .status-media img`).exists).notOk()
await t.expect($(`${getNthStatusSelector(1 + kittenIdx)} .status-media img`).getAttribute('src')).match(/^blob:http:\/\/localhost/)
.click($(`${getNthStatusSelector(1 + kittenIdx)} .status-sensitive-media-button`))
.expect($(`${getNthStatusSelector(1 + kittenIdx)} .status-media img`).getAttribute('alt')).eql('kitten')
.expect($(`${getNthStatusSelector(1 + kittenIdx)} .status-media img`).hasAttribute('src')).ok()
.expect($(`${getNthStatusSelector(1 + kittenIdx)} .status-media img`).getAttribute('src')).match(/^http:\/\//)
.hover(getNthStatus(1 + videoIdx))
.expect($(`${getNthStatusSelector(1 + videoIdx)} .status-media .play-video-button`).exists).notOk()
.click($(`${getNthStatusSelector(1 + videoIdx)} .status-sensitive-media-button`))

Wyświetl plik

@ -1,7 +1,7 @@
import { loginAsFoobar } from '../roles'
import {
generalSettingsButton,
getNthStatus, getNthStatusMedia, getNthStatusSensitiveMediaButton, homeNavButton, markMediaSensitiveInput,
getNthStatus, getNthStatusMediaImg, getNthStatusSensitiveMediaButton, homeNavButton, markMediaSensitiveInput,
scrollToStatus, settingsNavButton, neverMarkMediaSensitiveInput
} from '../utils'
@ -14,11 +14,11 @@ async function checkSensitivityForStatus (t, idx, sensitive) {
if (sensitive) {
await t
.expect(getNthStatusSensitiveMediaButton(1 + idx).exists).ok()
.expect(getNthStatusMedia(1 + idx).exists).notOk()
.expect(getNthStatusMediaImg(1 + idx).getAttribute('src')).match(/^blob:http:\/\/localhost/)
} else {
await t
.expect(getNthStatusSensitiveMediaButton(1 + idx).exists).notOk()
.expect(getNthStatusMedia(1 + idx).exists).ok()
.expect(getNthStatusMediaImg(1 + idx).getAttribute('src')).match(/^http:\/\//)
}
}

Wyświetl plik

@ -4,7 +4,7 @@ import {
getNthFavorited,
getNthStatus,
getNthStatusContent,
getNthStatusMedia,
getNthStatusMediaImg,
getNthStatusSensitiveMediaButton,
getNthStatusSpoiler,
getUrl, modalDialog,
@ -104,11 +104,11 @@ test('Shortcut y shows/hides sensitive image', async t => {
await t
.expect(isNthStatusActive(1 + idx)()).ok()
.expect(getNthStatusSensitiveMediaButton(1 + idx).exists).ok()
.expect(getNthStatusMedia(1 + idx).exists).notOk()
.expect(getNthStatusMediaImg(1 + idx).getAttribute('src')).match(/^blob:http:\/\/localhost/)
.pressKey('y')
.expect(getNthStatusMedia(1 + idx).exists).ok()
.expect(getNthStatusMediaImg(1 + idx).getAttribute('src')).match(/^http:\/\//)
.pressKey('y')
.expect(getNthStatusMedia(1 + idx).exists).notOk()
.expect(getNthStatusMediaImg(1 + idx).getAttribute('src')).match(/^blob:http:\/\/localhost/)
})
test('Shortcut f toggles favorite status', async t => {

Wyświetl plik

@ -1489,6 +1489,11 @@ bluebird@^3.5.5:
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.5.tgz#a8d0afd73251effbbd5fe384a77d73003c17a71f"
integrity sha512-5am6HnnfN+urzt4yfg7IgTbotDjIT/u8AJpEt0sIU9FtXfVeezXAPKswrG+xKUCOYAINpSdgZVDU6QFh+cuH3w==
blurhash@^1.1.3:
version "1.1.3"
resolved "https://registry.yarnpkg.com/blurhash/-/blurhash-1.1.3.tgz#dc325af7da836d07a0861d830bdd63694382483e"
integrity sha512-yUhPJvXexbqbyijCIE/T2NCXcj9iNPhWmOKbPTuR/cm7Q5snXYIfnVnz6m7MWOXxODMz/Cr3UcVkRdHiuDVRDw==
bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.1.1, bn.js@^4.4.0:
version "4.11.8"
resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.11.8.tgz#2cde09eb5ee341f484746bb0309b3253b1b1442f"
@ -4602,7 +4607,7 @@ loader-runner@^2.4.0:
resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-2.4.0.tgz#ed47066bfe534d7e84c4c7b9998c2a75607d9357"
integrity sha512-Jsmr89RcXGIwivFY21FcRrisYZfvLMTWx5kOLc+JTxtpBOG6xML0vzbc6SEQG2FO9/4Fc3wW4LVcB5DmGflaRw==
loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.3:
loader-utils@^1.0.0, loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.3:
version "1.2.3"
resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.2.3.tgz#1ff5dc6911c9f0a062531a4c04b609406108c2c7"
integrity sha512-fkpz8ejdnEMG3s37wGL07iSBDg99O9D5yflE9RGNH3hRdx9SOwYfnGYdZOUIZitN8E+E2vkq3MUMYMvPYl5ZZA==
@ -6600,6 +6605,14 @@ sax@^1.2.4, sax@~1.2.4:
resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
schema-utils@^0.4.0:
version "0.4.7"
resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-0.4.7.tgz#ba74f597d2be2ea880131746ee17d0a093c68187"
integrity sha512-v/iwU6wvwGK8HbU9yi3/nhGzP0yGSuhQMzL6ySiec1FSrZZDkhm4noOSWzrNFo/jEc+SJY6jRTwuwbSXJPDUnQ==
dependencies:
ajv "^6.1.0"
ajv-keywords "^3.1.0"
schema-utils@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-1.0.0.tgz#0b79a93204d7b600d4b2850d1f66c2a34951c770"
@ -8044,6 +8057,14 @@ worker-farm@^1.7.0:
dependencies:
errno "~0.1.7"
worker-loader@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/worker-loader/-/worker-loader-2.0.0.tgz#45fda3ef76aca815771a89107399ee4119b430ac"
integrity sha512-tnvNp4K3KQOpfRnD20m8xltE3eWh89Ye+5oj7wXEEHKac1P4oZ6p9oTj8/8ExqoSBnk9nu5Pr4nKfQ1hn2APJw==
dependencies:
loader-utils "^1.0.0"
schema-utils "^0.4.0"
wrap-ansi@^2.0.0, wrap-ansi@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-2.1.0.tgz#d8fc3d284dd05794fe84973caecdd1cf824fdd85"