You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

99 lines
2.4 KiB

  1. # frozen_string_literal: true
  2. class LanguageDetector
  3. include Singleton
  4. WORDS_THRESHOLD = 4
  5. RELIABLE_CHARACTERS_RE = /[\p{Hebrew}\p{Arabic}\p{Syriac}\p{Thaana}\p{Nko}\p{Han}\p{Katakana}\p{Hiragana}\p{Hangul}]+/m
  6. def initialize
  7. @identifier = CLD3::NNetLanguageIdentifier.new(1, 2048)
  8. end
  9. def detect(text, account)
  10. input_text = prepare_text(text)
  11. return if input_text.blank?
  12. detect_language_code(input_text) || default_locale(account)
  13. end
  14. def language_names
  15. @language_names = CLD3::TaskContextParams::LANGUAGE_NAMES.map { |name| iso6391(name.to_s).to_sym }.uniq
  16. end
  17. private
  18. def prepare_text(text)
  19. simplify_text(text).strip
  20. end
  21. def unreliable_input?(text)
  22. !reliable_input?(text)
  23. end
  24. def reliable_input?(text)
  25. sufficient_text_length?(text) || language_specific_character_set?(text)
  26. end
  27. def sufficient_text_length?(text)
  28. text.split(/\s+/).size >= WORDS_THRESHOLD
  29. end
  30. def language_specific_character_set?(text)
  31. words = text.scan(RELIABLE_CHARACTERS_RE)
  32. if words.present?
  33. words.reduce(0) { |acc, elem| acc + elem.size }.to_f / text.size.to_f > 0.3
  34. else
  35. false
  36. end
  37. end
  38. def detect_language_code(text)
  39. return if unreliable_input?(text)
  40. result = @identifier.find_language(text)
  41. iso6391(result.language.to_s).to_sym if result.reliable?
  42. end
  43. def iso6391(bcp47)
  44. iso639 = bcp47.split('-').first
  45. # CLD3 returns grandfathered language code for Hebrew
  46. return 'he' if iso639 == 'iw'
  47. ISO_639.find(iso639).alpha2
  48. end
  49. def simplify_text(text)
  50. new_text = remove_html(text)
  51. new_text.gsub!(FetchLinkCardService::URL_PATTERN, '')
  52. new_text.gsub!(Account::MENTION_RE, '')
  53. new_text.gsub!(Tag::HASHTAG_RE) { |string| string.gsub(/[#_]/, '#' => '', '_' => ' ').gsub(/[a-z][A-Z]|[a-zA-Z][\d]/) { |s| s.insert(1, ' ') }.downcase }
  54. new_text.gsub!(/:#{CustomEmoji::SHORTCODE_RE_FRAGMENT}:/, '')
  55. new_text.gsub!(/\s+/, ' ')
  56. new_text
  57. end
  58. def new_scrubber
  59. scrubber = Rails::Html::PermitScrubber.new
  60. scrubber.tags = %w(br p)
  61. scrubber
  62. end
  63. def scrubber
  64. @scrubber ||= new_scrubber
  65. end
  66. def remove_html(text)
  67. text = Loofah.fragment(text).scrub!(scrubber).to_s
  68. text.gsub!('<br>', "\n")
  69. text.gsub!('</p><p>', "\n\n")
  70. text.gsub!(/(^<p>|<\/p>$)/, '')
  71. text
  72. end
  73. def default_locale(account)
  74. account.user_locale&.to_sym || I18n.default_locale if account.local?
  75. end
  76. end