From e2685ccc81f04e1a63a97af80686bf85027418a6 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 14 Jul 2017 19:47:53 +0200 Subject: [PATCH] Fix #4149, fix #1199 - Store emojis as unicode (#4189) - Use unicode when selecting emoji through picker - Convert shortcodes to unicode when storing text input server-side - Do not convert shortcodes in JS anymore --- Gemfile | 1 + Gemfile.lock | 4 ++- app/helpers/emoji_helper.rb | 19 ++++++++++ app/javascript/mastodon/actions/compose.js | 7 ++-- app/javascript/mastodon/emoji.js | 24 ++----------- .../compose/components/compose_form.js | 3 +- .../components/emoji_picker_dropdown.js | 5 +-- app/javascript/mastodon/reducers/compose.js | 2 +- app/javascript/styles/components.scss | 1 + app/models/account.rb | 10 ++++++ app/models/status.rb | 6 +++- app/services/post_status_service.rb | 2 ++ spec/helpers/emoji_helper_spec.rb | 15 ++++++++ spec/helpers/routing_helper.rb | 5 --- spec/javascript/components/emojify.test.js | 36 ------------------- 15 files changed, 69 insertions(+), 71 deletions(-) create mode 100644 app/helpers/emoji_helper.rb create mode 100644 spec/helpers/emoji_helper_spec.rb delete mode 100644 spec/helpers/routing_helper.rb diff --git a/Gemfile b/Gemfile index 988b4d6b9..531d01ae0 100644 --- a/Gemfile +++ b/Gemfile @@ -28,6 +28,7 @@ gem 'devise', '~> 4.2' gem 'devise-two-factor', '~> 3.0' gem 'doorkeeper', '~> 4.2' gem 'fast_blank', '~> 1.0' +gem 'gemoji', '~> 3.0' gem 'goldfinger', '~> 1.2' gem 'hiredis', '~> 0.6' gem 'redis-namespace', '~> 1.5' diff --git a/Gemfile.lock b/Gemfile.lock index 5599e1db1..83202189d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -106,9 +106,9 @@ GEM rack (>= 1.0.0) rack-test (>= 0.5.4) xpath (~> 2.0) - charlock_holmes (0.7.3) case_transform (0.2) activesupport + charlock_holmes (0.7.3) chunky_png (1.3.8) cld3 (3.1.3) ffi (>= 1.1.0, < 1.10.0) @@ -163,6 +163,7 @@ GEM fuubar (2.2.0) rspec-core (~> 3.0) ruby-progressbar (~> 1.4) + gemoji (3.0.0) globalid (0.4.0) activesupport (>= 4.2.0) goldfinger (1.2.0) @@ -518,6 +519,7 @@ DEPENDENCIES faker (~> 1.7) fast_blank (~> 1.0) fuubar (~> 2.2) + gemoji (~> 3.0) goldfinger (~> 1.2) hamlit-rails (~> 0.2) hiredis (~> 0.6) diff --git a/app/helpers/emoji_helper.rb b/app/helpers/emoji_helper.rb new file mode 100644 index 000000000..c1595851f --- /dev/null +++ b/app/helpers/emoji_helper.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module EmojiHelper + EMOJI_PATTERN = /(?<=[^[:alnum:]:]|\n|^):([\w+-]+):(?=[^[:alnum:]:]|$)/x + + def emojify(text) + return text if text.blank? + + text.gsub(EMOJI_PATTERN) do |match| + emoji = Emoji.find_by_alias($1) # rubocop:disable Rails/DynamicFindBy,Style/PerlBackrefs + + if emoji + emoji.raw + else + match + end + end + end +end diff --git a/app/javascript/mastodon/actions/compose.js b/app/javascript/mastodon/actions/compose.js index 647a52b93..9f05a53e9 100644 --- a/app/javascript/mastodon/actions/compose.js +++ b/app/javascript/mastodon/actions/compose.js @@ -2,8 +2,6 @@ import api from '../api'; import { updateTimeline } from './timelines'; -import * as emojione from 'emojione'; - export const COMPOSE_CHANGE = 'COMPOSE_CHANGE'; export const COMPOSE_SUBMIT_REQUEST = 'COMPOSE_SUBMIT_REQUEST'; export const COMPOSE_SUBMIT_SUCCESS = 'COMPOSE_SUBMIT_SUCCESS'; @@ -73,11 +71,14 @@ export function mentionCompose(account, router) { export function submitCompose() { return function (dispatch, getState) { - const status = emojione.shortnameToUnicode(getState().getIn(['compose', 'text'], '')); + const status = getState().getIn(['compose', 'text'], ''); + if (!status || !status.length) { return; } + dispatch(submitComposeRequest()); + api(getState).post('/api/v1/statuses', { status, in_reply_to_id: getState().getIn(['compose', 'in_reply_to'], null), diff --git a/app/javascript/mastodon/emoji.js b/app/javascript/mastodon/emoji.js index 7043d5f3a..ed2180cd1 100644 --- a/app/javascript/mastodon/emoji.js +++ b/app/javascript/mastodon/emoji.js @@ -6,36 +6,18 @@ const trie = new Trie(Object.keys(emojione.jsEscapeMap)); function emojify(str) { // This walks through the string from start to end, ignoring any tags (

,
, etc.) - // and replacing valid shortnames like :smile: and :wink: as well as unicode strings + // and replacing valid unicode strings // that _aren't_ within tags with an version. - // The goal is to be the same as an emojione.regShortNames/regUnicode replacement, but faster. + // The goal is to be the same as an emojione.regUnicode replacement, but faster. let i = -1; let insideTag = false; - let insideShortname = false; - let shortnameStartIndex = -1; let match; while (++i < str.length) { const char = str.charAt(i); - if (insideShortname && char === ':') { - const shortname = str.substring(shortnameStartIndex, i + 1); - if (shortname in emojione.emojioneList) { - const unicode = emojione.emojioneList[shortname].unicode[emojione.emojioneList[shortname].unicode.length - 1]; - const alt = emojione.convert(unicode.toUpperCase()); - const replacement = `${alt}`; - str = str.substring(0, shortnameStartIndex) + replacement + str.substring(i + 1); - i += (replacement.length - shortname.length - 1); // jump ahead the length we've added to the string - } else { - i--; // stray colon, try again - } - insideShortname = false; - } else if (insideTag && char === '>') { + if (insideTag && char === '>') { insideTag = false; } else if (char === '<') { insideTag = true; - insideShortname = false; - } else if (!insideTag && char === ':') { - insideShortname = true; - shortnameStartIndex = i; } else if (!insideTag && (match = trie.search(str.substring(i)))) { const unicodeStr = match; if (unicodeStr in emojione.jsEscapeMap) { diff --git a/app/javascript/mastodon/features/compose/components/compose_form.js b/app/javascript/mastodon/features/compose/components/compose_form.js index f7eeedc69..f07552947 100644 --- a/app/javascript/mastodon/features/compose/components/compose_form.js +++ b/app/javascript/mastodon/features/compose/components/compose_form.js @@ -136,7 +136,8 @@ export default class ComposeForm extends ImmutablePureComponent { handleEmojiPick = (data) => { const position = this.autosuggestTextarea.textarea.selectionStart; - this._restoreCaret = position + data.shortname.length + 1; + const emojiChar = String.fromCodePoint(parseInt(data.unicode, 16)); + this._restoreCaret = position + emojiChar.length + 1; this.props.onPickEmoji(position, data); } diff --git a/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js b/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js index 83c66a5d5..acc584f20 100644 --- a/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js +++ b/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js @@ -109,11 +109,12 @@ export default class EmojiPickerDropdown extends React.PureComponent { 🙂 + { this.state.active && !this.state.loading && diff --git a/app/javascript/mastodon/reducers/compose.js b/app/javascript/mastodon/reducers/compose.js index a92b5aa23..ea3b78b67 100644 --- a/app/javascript/mastodon/reducers/compose.js +++ b/app/javascript/mastodon/reducers/compose.js @@ -118,7 +118,7 @@ const insertSuggestion = (state, position, token, completion) => { }; const insertEmoji = (state, position, emojiData) => { - const emoji = emojiData.shortname; + const emoji = String.fromCodePoint(parseInt(emojiData.unicode, 16)); return state.withMutations(map => { map.update('text', oldText => `${oldText.slice(0, position)}${emoji} ${oldText.slice(position)}`); diff --git a/app/javascript/styles/components.scss b/app/javascript/styles/components.scss index 0face646d..0420a2bed 100644 --- a/app/javascript/styles/components.scss +++ b/app/javascript/styles/components.scss @@ -2708,6 +2708,7 @@ button.icon-button.active i.fa-retweet { margin-left: 2px; width: 24px; outline: 0; + cursor: pointer; &:active, &:focus { diff --git a/app/models/account.rb b/app/models/account.rb index 2b54cee5f..7243cb1a5 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -47,6 +47,7 @@ class Account < ApplicationRecord include AccountInteractions include Attachmentable include Remotable + include EmojiHelper # Local users has_one :user, inverse_of: :account @@ -240,9 +241,18 @@ class Account < ApplicationRecord before_create :generate_keys before_validation :normalize_domain + before_validation :prepare_contents, if: :local? private + def prepare_contents + display_name&.strip! + note&.strip! + + self.display_name = emojify(display_name) + self.note = emojify(note) + end + def generate_keys return unless local? diff --git a/app/models/status.rb b/app/models/status.rb index 65db7579a..24eaf7071 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -29,6 +29,7 @@ class Status < ApplicationRecord include Streamable include Cacheable include StatusThreadingConcern + include EmojiHelper enum visibility: [:public, :unlisted, :private, :direct], _suffix: :visibility @@ -120,7 +121,7 @@ class Status < ApplicationRecord !sensitive? && media_attachments.any? end - before_validation :prepare_contents + before_validation :prepare_contents, if: :local? before_validation :set_reblog before_validation :set_visibility before_validation :set_conversation @@ -241,6 +242,9 @@ class Status < ApplicationRecord def prepare_contents text&.strip! spoiler_text&.strip! + + self.text = emojify(text) + self.spoiler_text = emojify(spoiler_text) end def set_reblog diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb index 2e6fbb5c3..951a38e19 100644 --- a/app/services/post_status_service.rb +++ b/app/services/post_status_service.rb @@ -21,6 +21,7 @@ class PostStatusService < BaseService media = validate_media!(options[:media_ids]) status = nil + ApplicationRecord.transaction do status = account.statuses.create!(text: text, thread: in_reply_to, @@ -31,6 +32,7 @@ class PostStatusService < BaseService application: options[:application]) attach_media(status, media) end + process_mentions_service.call(status) process_hashtags_service.call(status) diff --git a/spec/helpers/emoji_helper_spec.rb b/spec/helpers/emoji_helper_spec.rb new file mode 100644 index 000000000..1eedfb719 --- /dev/null +++ b/spec/helpers/emoji_helper_spec.rb @@ -0,0 +1,15 @@ +require 'rails_helper' + +RSpec.describe EmojiHelper, type: :helper do + describe '#emojify' do + it 'converts shortcodes to unicode' do + text = ':book: Book' + expect(emojify(text)).to eq '📖 Book' + end + + it 'does not convert shortcodes that are part of a string into unicode' do + text = ':see_no_evil::hear_no_evil::speak_no_evil:' + expect(emojify(text)).to eq text + end + end +end diff --git a/spec/helpers/routing_helper.rb b/spec/helpers/routing_helper.rb deleted file mode 100644 index 3cd397397..000000000 --- a/spec/helpers/routing_helper.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'rails_helper' - -RSpec.describe RoutingHelper, type: :helper do - -end diff --git a/spec/javascript/components/emojify.test.js b/spec/javascript/components/emojify.test.js index 3e8b25af9..e165b4519 100644 --- a/spec/javascript/components/emojify.test.js +++ b/spec/javascript/components/emojify.test.js @@ -2,32 +2,6 @@ import { expect } from 'chai'; import emojify from '../../../app/javascript/mastodon/emoji'; describe('emojify', () => { - it('does a basic emojify', () => { - expect(emojify(':smile:')).to.equal( - '😄'); - }); - - it('does a double emojify', () => { - expect(emojify(':smile: and :wink:')).to.equal( - '😄 and 😉'); - }); - - it('works with random colons', () => { - expect(emojify(':smile: : :wink:')).to.equal( - '😄 : 😉'); - expect(emojify(':smile::::wink:')).to.equal( - '😄::😉'); - expect(emojify(':smile:::::wink:')).to.equal( - '😄:::😉'); - }); - - it('works with tags', () => { - expect(emojify('

:smile:

')).to.equal( - '

😄

'); - expect(emojify('

:smile:

and

:wink:

')).to.equal( - '

😄

and

😉

'); - }); - it('ignores unknown shortcodes', () => { expect(emojify(':foobarbazfake:')).to.equal(':foobarbazfake:'); }); @@ -46,11 +20,6 @@ describe('emojify', () => { expect(emojify(':smile')).to.equal(':smile'); }); - it('does two emoji next to each other', () => { - expect(emojify(':smile::wink:')).to.equal( - '😄😉'); - }); - it('does unicode', () => { expect(emojify('\uD83D\uDC69\u200D\uD83D\uDC69\u200D\uD83D\uDC66\u200D\uD83D\uDC66')).to.equal( '👩‍👩‍👦‍👦'); @@ -72,12 +41,7 @@ describe('emojify', () => { 'foo ❗ #️⃣ bar'); }); - it('does mixed unicode and shortnames', () => { - expect(emojify(':smile:#\uFE0F\u20E3:wink:\u2757')).to.equal('😄#️⃣😉❗'); - }); - it('ignores unicode inside of tags', () => { expect(emojify('

')).to.equal('

'); }); - });