From 8777c4b3e75df2daa2c1777ce7e3455b36c6a2ed Mon Sep 17 00:00:00 2001
From: ckins <492159680@qq.com>
Date: Fri, 24 Nov 2017 23:56:32 +0800
Subject: [PATCH 1/2] add feature for title extract

---
 newspaper/extractors.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/newspaper/extractors.py b/newspaper/extractors.py
index 12875c6..a168581 100644
--- a/newspaper/extractors.py
+++ b/newspaper/extractors.py
@@ -17,6 +17,7 @@ import re
 import re
 from collections import defaultdict
 
+from difflib import SequenceMatcher
 from dateutil.parser import parse as date_parser
 from tldextract import tldextract
 from urllib.parse import urljoin, urlparse, urlunparse
@@ -342,11 +343,17 @@ class ContentExtractor(object):
         # (either it differs for case, for special chars, or it's truncated)
         # in these cases, we prefer the title_text_h1
         filter_title = filter_regex.sub('', title).lower()
-        if filter_title_text_h1 == filter_title:
+        if self.is_similar(filter_title_text_h1, filter_title):
             title = title_text_h1
 
         return title
 
+    def is_similar(self, text_a, text_b):
+        """used for comparison between the final title and title_text_h1
+            0.6 is an empirical value
+        """
+        return SequenceMatcher(None, text_a, text_b).ratio() > 0.6
+
     def split_title(self, title, splitter, hint=None):
         """Split the title to best part possible
         """

From 84f3f7380c1799e67093540ba753e43fe93c7712 Mon Sep 17 00:00:00 2001
From: ckins <492159680@qq.com>
Date: Wed, 29 Nov 2017 23:45:02 +0800
Subject: [PATCH 2/2] enhance filter

---
 newspaper/cleaners.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/newspaper/cleaners.py b/newspaper/cleaners.py
index 00fec2f..eff9126 100644
--- a/newspaper/cleaners.py
+++ b/newspaper/cleaners.py
@@ -28,6 +28,12 @@ class DocumentCleaner(object):
             "|date|^print$|popup|author-dropdown|tools|socialtools|byline"
             "|konafilter|KonaFilter|breadcrumbs|^fn$|wp-caption-text"
             "|legende|ajoutVideo|timestamp|js_replies"
+            "|date|^print$|popup|author-dropdown|tools|socialtools|byline"
+            "|konafilter|KonaFilter|breadcrumbs|^fn$|wp-caption-text"
+            "|legende|ajoutVideo|timestamp|js_replies|breadcrumb|^rating$""|^comment$|^share$|^like$|^icon$|^count$|^sharing$|^news-list$"
+            "|^vote$|^ad$|^Ad$|^rec$|^oneindia$|^inread$|^showmore$|^tags_scroll$""|^Share$|^date$|^related$|^fb-root$|^recommendation$|^recomment$"
+            "|^readalso$|^read-also$|^image_counter$|^yarp$|^navig$|^extranews$""|^arrow$|^slider__footer$|^socbuttons$|^see-more$|^subscribe$"
+            "|post-data|post-social|article__content__author-title|archive__posts__item""|user|^banner$|^flair$|^forlo$|append-news|^inject$|^rg-gallery_inj$|^tag$"
         )
         self.regexp_namespace = "http://exslt.org/regular-expressions"
         self.nauthy_ids_re = ("//*[re:test(@id, '%s', 'i')]" %