From 1b167707c2ae5ebc272afc3a7515f5b236083135 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 15 Mar 2019 05:07:09 +0100 Subject: [PATCH] Fix language detection of non-latin alphabets even at few characters (#10276) --- app/lib/language_detector.rb | 31 +++++++++++++++++++++++------- spec/lib/language_detector_spec.rb | 4 ++-- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/app/lib/language_detector.rb b/app/lib/language_detector.rb index 58c8e20694..70a9084d1b 100644 --- a/app/lib/language_detector.rb +++ b/app/lib/language_detector.rb @@ -3,7 +3,8 @@ class LanguageDetector include Singleton - CHARACTER_THRESHOLD = 140 + CHARACTER_THRESHOLD = 140 + RELIABLE_CHARACTERS_RE = /[\p{Hebrew}\p{Arabic}\p{Syriac}\p{Thaana}\p{Nko}\p{Han}\p{Katakana}\p{Hiragana}\p{Hangul}]+/m def initialize @identifier = CLD3::NNetLanguageIdentifier.new(1, 2048) @@ -11,15 +12,14 @@ class LanguageDetector def detect(text, account) input_text = prepare_text(text) + return if input_text.blank? detect_language_code(input_text) || default_locale(account) end def language_names - @language_names = - CLD3::TaskContextParams::LANGUAGE_NAMES.map { |name| iso6391(name.to_s).to_sym } - .uniq + @language_names = CLD3::TaskContextParams::LANGUAGE_NAMES.map { |name| iso6391(name.to_s).to_sym }.uniq end private @@ -29,12 +29,29 @@ class LanguageDetector end def unreliable_input?(text) - text.size < CHARACTER_THRESHOLD + !reliable_input?(text) + end + + def reliable_input?(text) + sufficient_text_length?(text) || language_specific_character_set?(text) + end + + def sufficient_text_length?(text) + text.size >= CHARACTER_THRESHOLD + end + + def language_specific_character_set?(text) + words = text.scan(RELIABLE_CHARACTERS_RE) + + if words.present? + words.reduce(0) { |acc, elem| acc + elem.size }.to_f / text.size.to_f > 0.3 + else + false + end end def detect_language_code(text) return if unreliable_input?(text) - result = @identifier.find_language(text) iso6391(result.language.to_s).to_sym if result.reliable? end @@ -77,6 +94,6 @@ class LanguageDetector end def default_locale(account) - return account.user_locale&.to_sym || I18n.default_locale if account.local? + account.user_locale&.to_sym || I18n.default_locale if account.local? end end diff --git a/spec/lib/language_detector_spec.rb b/spec/lib/language_detector_spec.rb index 0fa2a59eff..0cb70605ad 100644 --- a/spec/lib/language_detector_spec.rb +++ b/spec/lib/language_detector_spec.rb @@ -106,11 +106,11 @@ describe LanguageDetector do end describe 'remote user' do - it 'nil for foreign user when language is not present' do + it 'detects Korean language' do string = '안녕하세요' result = described_class.instance.detect(string, account_remote) - expect(result).to eq nil + expect(result).to eq :ko end end