Browse Source

Only offer translation for supported languages (#23879)

closed-social-glitch-2
Christian Schmidt 1 year ago
committed by GitHub
parent
commit
5a8c651e8f
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 336 additions and 31 deletions
  1. +4
    -0
      .rubocop.yml
  2. +2
    -2
      app/javascript/mastodon/components/status_content.jsx
  3. +0
    -2
      app/javascript/mastodon/initial_state.js
  4. +4
    -0
      app/lib/translation_service.rb
  5. +33
    -13
      app/lib/translation_service/deepl.rb
  6. +27
    -11
      app/lib/translation_service/libre_translate.rb
  7. +10
    -0
      app/models/status.rb
  8. +0
    -1
      app/serializers/initial_state_serializer.rb
  9. +5
    -1
      app/serializers/rest/status_serializer.rb
  10. +1
    -1
      app/services/translate_status_service.rb
  11. +100
    -0
      spec/lib/translation_service/deepl_spec.rb
  12. +71
    -0
      spec/lib/translation_service/libre_translate_spec.rb
  13. +79
    -0
      spec/models/status_spec.rb

+ 4
- 0
.rubocop.yml View File

@ -97,6 +97,10 @@ Rails/Exit:
- 'lib/mastodon/cli_helper.rb' - 'lib/mastodon/cli_helper.rb'
- 'lib/cli.rb' - 'lib/cli.rb'
RSpec/FilePath:
CustomTransform:
DeepL: deepl
RSpec/NotToNot: RSpec/NotToNot:
EnforcedStyle: to_not EnforcedStyle: to_not

+ 2
- 2
app/javascript/mastodon/components/status_content.jsx View File

@ -6,7 +6,7 @@ import { Link } from 'react-router-dom';
import classnames from 'classnames'; import classnames from 'classnames';
import PollContainer from 'mastodon/containers/poll_container'; import PollContainer from 'mastodon/containers/poll_container';
import Icon from 'mastodon/components/icon'; import Icon from 'mastodon/components/icon';
import { autoPlayGif, languages as preloadedLanguages, translationEnabled } from 'mastodon/initial_state';
import { autoPlayGif, languages as preloadedLanguages } from 'mastodon/initial_state';
const MAX_HEIGHT = 706; // 22px * 32 (+ 2px padding at the top) const MAX_HEIGHT = 706; // 22px * 32 (+ 2px padding at the top)
@ -220,7 +220,7 @@ class StatusContent extends React.PureComponent {
const hidden = this.props.onExpandedToggle ? !this.props.expanded : this.state.hidden; const hidden = this.props.onExpandedToggle ? !this.props.expanded : this.state.hidden;
const renderReadMore = this.props.onClick && status.get('collapsed'); const renderReadMore = this.props.onClick && status.get('collapsed');
const renderTranslate = translationEnabled && this.context.identity.signedIn && this.props.onTranslate && ['public', 'unlisted'].includes(status.get('visibility')) && status.get('contentHtml').length > 0 && status.get('language') !== null && intl.locale !== status.get('language');
const renderTranslate = this.props.onTranslate && status.get('translatable');
const content = { __html: status.get('translation') ? status.getIn(['translation', 'content']) : status.get('contentHtml') }; const content = { __html: status.get('translation') ? status.getIn(['translation', 'content']) : status.get('contentHtml') };
const spoilerContent = { __html: status.get('spoilerHtml') }; const spoilerContent = { __html: status.get('spoilerHtml') };

+ 0
- 2
app/javascript/mastodon/initial_state.js View File

@ -80,7 +80,6 @@
* @property {boolean} use_blurhash * @property {boolean} use_blurhash
* @property {boolean=} use_pending_items * @property {boolean=} use_pending_items
* @property {string} version * @property {string} version
* @property {boolean} translation_enabled
*/ */
/** /**
@ -132,7 +131,6 @@ export const unfollowModal = getMeta('unfollow_modal');
export const useBlurhash = getMeta('use_blurhash'); export const useBlurhash = getMeta('use_blurhash');
export const usePendingItems = getMeta('use_pending_items'); export const usePendingItems = getMeta('use_pending_items');
export const version = getMeta('version'); export const version = getMeta('version');
export const translationEnabled = getMeta('translation_enabled');
export const languages = initialState?.languages; export const languages = initialState?.languages;
export const statusPageUrl = getMeta('status_page_url'); export const statusPageUrl = getMeta('status_page_url');

+ 4
- 0
app/lib/translation_service.rb View File

@ -21,6 +21,10 @@ class TranslationService
ENV['DEEPL_API_KEY'].present? || ENV['LIBRE_TRANSLATE_ENDPOINT'].present? ENV['DEEPL_API_KEY'].present? || ENV['LIBRE_TRANSLATE_ENDPOINT'].present?
end end
def supported?(_source_language, _target_language)
false
end
def translate(_text, _source_language, _target_language) def translate(_text, _source_language, _target_language)
raise NotImplementedError raise NotImplementedError
end end

+ 33
- 13
app/lib/translation_service/deepl.rb View File

@ -11,33 +11,53 @@ class TranslationService::DeepL < TranslationService
end end
def translate(text, source_language, target_language) def translate(text, source_language, target_language)
request(text, source_language, target_language).perform do |res|
form = { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' }
request(:post, '/v2/translate', form: form) do |res|
transform_response(res.body_with_limit)
end
end
def supported?(source_language, target_language)
source_language.in?(languages('source')) && target_language.in?(languages('target'))
end
private
def languages(type)
Rails.cache.fetch("translation_service/deepl/languages/#{type}", expires_in: 7.days, race_condition_ttl: 1.minute) do
request(:get, "/v2/languages?type=#{type}") do |res|
# In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so
# they are supported but not returned by the API.
extra = type == 'source' ? [nil] : %w(en pt)
languages = Oj.load(res.body_with_limit).map { |language| language['language'].downcase }
languages + extra
end
end
end
def request(verb, path, **options)
req = Request.new(verb, "#{base_url}#{path}", **options)
req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}")
req.perform do |res|
case res.code case res.code
when 429 when 429
raise TooManyRequestsError raise TooManyRequestsError
when 456 when 456
raise QuotaExceededError raise QuotaExceededError
when 200...300 when 200...300
transform_response(res.body_with_limit)
yield res
else else
raise UnexpectedResponseError raise UnexpectedResponseError
end end
end end
end end
private
def request(text, source_language, target_language)
req = Request.new(:post, endpoint_url, form: { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' })
req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}")
req
end
def endpoint_url
def base_url
if @plan == 'free' if @plan == 'free'
'https://api-free.deepl.com/v2/translate'
'https://api-free.deepl.com'
else else
'https://api.deepl.com/v2/translate'
'https://api.deepl.com'
end end
end end

+ 27
- 11
app/lib/translation_service/libre_translate.rb View File

@ -9,29 +9,45 @@ class TranslationService::LibreTranslate < TranslationService
end end
def translate(text, source_language, target_language) def translate(text, source_language, target_language)
request(text, source_language, target_language).perform do |res|
body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key)
request(:post, '/translate', body: body) do |res|
transform_response(res.body_with_limit, source_language)
end
end
def supported?(source_language, target_language)
languages.key?(source_language) && languages[source_language].include?(target_language)
end
private
def languages
Rails.cache.fetch('translation_service/libre_translate/languages', expires_in: 7.days, race_condition_ttl: 1.minute) do
request(:get, '/languages') do |res|
languages = Oj.load(res.body_with_limit).to_h { |language| [language['code'], language['targets']] }
languages[nil] = languages.values.flatten.uniq
languages
end
end
end
def request(verb, path, **options)
req = Request.new(verb, "#{@base_url}#{path}", allow_local: true, **options)
req.add_headers('Content-Type': 'application/json')
req.perform do |res|
case res.code case res.code
when 429 when 429
raise TooManyRequestsError raise TooManyRequestsError
when 403 when 403
raise QuotaExceededError raise QuotaExceededError
when 200...300 when 200...300
transform_response(res.body_with_limit, source_language)
yield res
else else
raise UnexpectedResponseError raise UnexpectedResponseError
end end
end end
end end
private
def request(text, source_language, target_language)
body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key)
req = Request.new(:post, "#{@base_url}/translate", body: body, allow_local: true)
req.add_headers('Content-Type': 'application/json')
req
end
def transform_response(str, source_language) def transform_response(str, source_language)
json = Oj.load(str, mode: :strict) json = Oj.load(str, mode: :strict)

+ 10
- 0
app/models/status.rb View File

@ -232,6 +232,16 @@ class Status < ApplicationRecord
public_visibility? || unlisted_visibility? public_visibility? || unlisted_visibility?
end end
def translatable?
translate_target_locale = I18n.locale.to_s.split(/[_-]/).first
distributable? &&
content.present? &&
language != translate_target_locale &&
TranslationService.configured? &&
TranslationService.configured.supported?(language, translate_target_locale)
end
alias sign? distributable? alias sign? distributable?
def with_media? def with_media?

+ 0
- 1
app/serializers/initial_state_serializer.rb View File

@ -30,7 +30,6 @@ class InitialStateSerializer < ActiveModel::Serializer
timeline_preview: Setting.timeline_preview, timeline_preview: Setting.timeline_preview,
activity_api_enabled: Setting.activity_api_enabled, activity_api_enabled: Setting.activity_api_enabled,
single_user_mode: Rails.configuration.x.single_user_mode, single_user_mode: Rails.configuration.x.single_user_mode,
translation_enabled: TranslationService.configured?,
trends_as_landing_page: Setting.trends_as_landing_page, trends_as_landing_page: Setting.trends_as_landing_page,
status_page_url: Setting.status_page_url, status_page_url: Setting.status_page_url,
} }

+ 5
- 1
app/serializers/rest/status_serializer.rb View File

@ -4,7 +4,7 @@ class REST::StatusSerializer < ActiveModel::Serializer
include FormattingHelper include FormattingHelper
attributes :id, :created_at, :in_reply_to_id, :in_reply_to_account_id, attributes :id, :created_at, :in_reply_to_id, :in_reply_to_account_id,
:sensitive, :spoiler_text, :visibility, :language,
:sensitive, :spoiler_text, :visibility, :language, :translatable,
:uri, :url, :replies_count, :reblogs_count, :uri, :url, :replies_count, :reblogs_count,
:favourites_count, :edited_at :favourites_count, :edited_at
@ -50,6 +50,10 @@ class REST::StatusSerializer < ActiveModel::Serializer
object.account.user_shows_application? || (current_user? && current_user.account_id == object.account_id) object.account.user_shows_application? || (current_user? && current_user.account_id == object.account_id)
end end
def translatable
current_user? && object.translatable?
end
def visibility def visibility
# This visibility is masked behind "private" # This visibility is masked behind "private"
# to avoid API changes because there are no # to avoid API changes because there are no

+ 1
- 1
app/services/translate_status_service.rb View File

@ -6,7 +6,7 @@ class TranslateStatusService < BaseService
include FormattingHelper include FormattingHelper
def call(status, target_language) def call(status, target_language)
raise Mastodon::NotPermittedError unless status.public_visibility? || status.unlisted_visibility?
raise Mastodon::NotPermittedError unless status.translatable?
@status = status @status = status
@content = status_content_format(@status) @content = status_content_format(@status)

+ 100
- 0
spec/lib/translation_service/deepl_spec.rb View File

@ -0,0 +1,100 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe TranslationService::DeepL do
subject(:service) { described_class.new(plan, 'my-api-key') }
let(:plan) { 'advanced' }
before do
stub_request(:get, 'https://api.deepl.com/v2/languages?type=source').to_return(
body: '[{"language":"EN","name":"English"},{"language":"UK","name":"Ukrainian"}]'
)
stub_request(:get, 'https://api.deepl.com/v2/languages?type=target').to_return(
body: '[{"language":"EN-GB","name":"English (British)"},{"language":"ZH","name":"Chinese"}]'
)
end
describe '#supported?' do
it 'supports included languages as source and target languages' do
expect(service.supported?('uk', 'en')).to be true
end
it 'supports auto-detecting source language' do
expect(service.supported?(nil, 'en')).to be true
end
it 'supports "en" and "pt" as target languages though not included in language list' do
expect(service.supported?('uk', 'en')).to be true
expect(service.supported?('uk', 'pt')).to be true
end
it 'does not support non-included language as target language' do
expect(service.supported?('uk', 'nl')).to be false
end
it 'does not support non-included language as source language' do
expect(service.supported?('da', 'en')).to be false
end
end
describe '#translate' do
it 'returns translation with specified source language' do
stub_request(:post, 'https://api.deepl.com/v2/translate')
.with(body: 'text=Hasta+la+vista&source_lang=ES&target_lang=en&tag_handling=html')
.to_return(body: '{"translations":[{"detected_source_language":"ES","text":"See you soon"}]}')
translation = service.translate('Hasta la vista', 'es', 'en')
expect(translation.detected_source_language).to eq 'es'
expect(translation.provider).to eq 'DeepL.com'
expect(translation.text).to eq 'See you soon'
end
it 'returns translation with auto-detected source language' do
stub_request(:post, 'https://api.deepl.com/v2/translate')
.with(body: 'text=Guten+Tag&source_lang&target_lang=en&tag_handling=html')
.to_return(body: '{"translations":[{"detected_source_language":"DE","text":"Good Morning"}]}')
translation = service.translate('Guten Tag', nil, 'en')
expect(translation.detected_source_language).to eq 'de'
expect(translation.provider).to eq 'DeepL.com'
expect(translation.text).to eq 'Good Morning'
end
end
describe '#languages?' do
it 'returns source languages' do
expect(service.send(:languages, 'source')).to eq ['en', 'uk', nil]
end
it 'returns target languages' do
expect(service.send(:languages, 'target')).to eq %w(en-gb zh en pt)
end
end
describe '#request' do
before do
stub_request(:any, //)
# rubocop:disable Lint/EmptyBlock
service.send(:request, :get, '/v2/languages') { |res| }
# rubocop:enable Lint/EmptyBlock
end
it 'uses paid plan base URL' do
expect(a_request(:get, 'https://api.deepl.com/v2/languages')).to have_been_made.once
end
context 'with free plan' do
let(:plan) { 'free' }
it 'uses free plan base URL' do
expect(a_request(:get, 'https://api-free.deepl.com/v2/languages')).to have_been_made.once
end
end
it 'sends API key' do
expect(a_request(:get, 'https://api.deepl.com/v2/languages').with(headers: { Authorization: 'DeepL-Auth-Key my-api-key' })).to have_been_made.once
end
end
end

+ 71
- 0
spec/lib/translation_service/libre_translate_spec.rb View File

@ -0,0 +1,71 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe TranslationService::LibreTranslate do
subject(:service) { described_class.new('https://libretranslate.example.com', 'my-api-key') }
before do
stub_request(:get, 'https://libretranslate.example.com/languages').to_return(
body: '[{"code": "en","name": "English","targets": ["de","es"]},{"code": "da","name": "Danish","targets": ["en","de"]}]'
)
end
describe '#supported?' do
it 'supports included language pair' do
expect(service.supported?('en', 'de')).to be true
end
it 'does not support reversed language pair' do
expect(service.supported?('de', 'en')).to be false
end
it 'supports auto-detecting source language' do
expect(service.supported?(nil, 'de')).to be true
end
it 'does not support auto-detecting for unsupported target language' do
expect(service.supported?(nil, 'pt')).to be false
end
end
describe '#languages' do
subject(:languages) { service.send(:languages) }
it 'includes supported source languages' do
expect(languages.keys).to eq ['en', 'da', nil]
end
it 'includes supported target languages for source language' do
expect(languages['en']).to eq %w(de es)
end
it 'includes supported target languages for auto-detected language' do
expect(languages[nil]).to eq %w(de es en)
end
end
describe '#translate' do
it 'returns translation with specified source language' do
stub_request(:post, 'https://libretranslate.example.com/translate')
.with(body: '{"q":"Hasta la vista","source":"es","target":"en","format":"html","api_key":"my-api-key"}')
.to_return(body: '{"translatedText": "See you"}')
translation = service.translate('Hasta la vista', 'es', 'en')
expect(translation.detected_source_language).to eq 'es'
expect(translation.provider).to eq 'LibreTranslate'
expect(translation.text).to eq 'See you'
end
it 'returns translation with auto-detected source language' do
stub_request(:post, 'https://libretranslate.example.com/translate')
.with(body: '{"q":"Guten Morgen","source":"auto","target":"en","format":"html","api_key":"my-api-key"}')
.to_return(body: '{"detectedLanguage":{"confidence":92,"language":"de"},"translatedText":"Good morning"}')
translation = service.translate('Guten Morgen', nil, 'en')
expect(translation.detected_source_language).to be_nil
expect(translation.provider).to eq 'LibreTranslate'
expect(translation.text).to eq 'Good morning'
end
end
end

+ 79
- 0
spec/models/status_spec.rb View File

@ -114,6 +114,85 @@ RSpec.describe Status, type: :model do
end end
end end
describe '#translatable?' do
before do
allow(TranslationService).to receive(:configured?).and_return(true)
allow(TranslationService).to receive(:configured).and_return(TranslationService.new)
allow(TranslationService.configured).to receive(:supported?).with('es', 'en').and_return(true)
subject.language = 'es'
subject.visibility = :public
end
context 'all conditions are satisfied' do
it 'returns true' do
expect(subject.translatable?).to be true
end
end
context 'translation service is not configured' do
it 'returns false' do
allow(TranslationService).to receive(:configured?).and_return(false)
allow(TranslationService).to receive(:configured).and_raise(TranslationService::NotConfiguredError)
expect(subject.translatable?).to be false
end
end
context 'status language is nil' do
it 'returns true' do
subject.language = nil
allow(TranslationService.configured).to receive(:supported?).with(nil, 'en').and_return(true)
expect(subject.translatable?).to be true
end
end
context 'status language is same as default locale' do
it 'returns false' do
subject.language = I18n.locale
expect(subject.translatable?).to be false
end
end
context 'status language is unsupported' do
it 'returns false' do
subject.language = 'af'
allow(TranslationService.configured).to receive(:supported?).with('af', 'en').and_return(false)
expect(subject.translatable?).to be false
end
end
context 'default locale is unsupported' do
it 'returns false' do
allow(TranslationService.configured).to receive(:supported?).with('es', 'af').and_return(false)
I18n.with_locale('af') do
expect(subject.translatable?).to be false
end
end
end
context 'default locale has region' do
it 'returns true' do
I18n.with_locale('en-GB') do
expect(subject.translatable?).to be true
end
end
end
context 'status text is blank' do
it 'returns false' do
subject.text = ' '
expect(subject.translatable?).to be false
end
end
context 'status visiblity is hidden' do
it 'returns false' do
subject.visibility = 'limited'
expect(subject.translatable?).to be false
end
end
end
describe '#content' do describe '#content' do
it 'returns the text of the status if it is not a reblog' do it 'returns the text of the status if it is not a reblog' do
expect(subject.content).to eql subject.text expect(subject.content).to eql subject.text

Loading…
Cancel
Save