Ignore low-confidence CharlockHolmes guesses when parsing link cards (#9510)

* Add failing test for windows-1251 link cards * Ignore low-confidence CharlockHolmes guesses Fixes #9466 * Fix no method error when charlock holmes cannot detect charset
5 years ago · e709b8da0d
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@ -137,7 +137,8 @@ class FetchLinkCardService < BaseService
    detector.strip_tags = true

    guess      = detector.detect(@html, @html_charset)
    page       = Nokogiri::HTML(@html, nil, guess&.fetch(:encoding, nil))
    encoding   = guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
    page       = Nokogiri::HTML(@html, nil, encoding)
    player_url = meta_property(page, 'twitter:player')

    if player_url && !bad_url?(Addressable::URI.parse(player_url))
--- a/spec/fixtures/requests/windows-1251.txt
+++ b/spec/fixtures/requests/windows-1251.txt
@ -0,0 +1,17 @@
 HTTP/1.1 200 OK
 server: nginx
 date: Wed, 12 Dec 2018 13:14:03 GMT
 content-type: text/html
 content-length: 190
 accept-ranges: bytes

 <!DOCTYPE html>
 <html>
 <head>
  <meta http-equiv="Content-Type" content="text/html; charset=windows-1251" />
  <title>ñýìïë òåêñò</title>
 </head>
 <body>
  <p>ñýìïë òåêñò</p>
 </body>
 </html>
--- a/spec/services/fetch_link_card_service_spec.rb
+++ b/spec/services/fetch_link_card_service_spec.rb
@ -17,6 +17,8 @@ RSpec.describe FetchLinkCardService, type: :service do
    stub_request(:head, 'https://github.com/qbi/WannaCry').to_return(status: 404)
    stub_request(:head, 'http://example.com/test-').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
    stub_request(:get, 'http://example.com/test-').to_return(request_fixture('idn.txt'))
    stub_request(:head, 'http://example.com/windows-1251').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
    stub_request(:get, 'http://example.com/windows-1251').to_return(request_fixture('windows-1251.txt'))

    subject.call(status)
  end
@ -57,6 +59,15 @@ RSpec.describe FetchLinkCardService, type: :service do
      end
    end

    context do
      let(:status) { Fabricate(:status, text: 'Check out http://example.com/windows-1251') }

      it 'works with windows-1251' do
        expect(a_request(:get, 'http://example.com/windows-1251')).to have_been_made.at_least_once
        expect(status.preview_cards.first.title).to eq('сэмпл текст')
      end
    end

    context do
      let(:status) { Fabricate(:status, text: 'テストhttp://example.com/日本語') }