Replace pipfeed with news3k's summary option

Author: chris-y

bestof.py

@@ -6,14 +6,14 @@
 import random
 import mimetypes
 import tldr
-import pipfeed
 import yt
 import news
 import deepseek
 import requests
 import urllib.parse
 import datetime
 import issues
+import shorten
 from urllib.parse import urlparse
 from pythorhead import Lemmy
 from pythorhead.types import SortType
@@ -61,20 +61,6 @@ def extract_desc(ci):
 
   return "No description"
 
-def shorten_text(text, deepseek_key):
-  max_len = 300
-  if len(text) > max_len:
-    print('shorten...')
-    try:
-      t = deepseek.shorten(text, deepseek_key, max_len)
-      t = f'{t} 🖍🤖\n\n'
-      return t
-    except Exception as e:
-      print(f'deepseek raised exception: {e}')
-    return f'{text[:297]}...\n\n' # NB: three less than max_len
-  else:
-    return f'{text}\n\n'
-
 def add_embed(p):
   if "embed_title" in p:
     t = f'*{p["embed_title"]}*\n\n'
@@ -350,7 +336,7 @@ def run(user, pw, instance, postcomm, cfg, post_title, images_only, nsfw_b, modu
       posttext = posttext + f"![]({p['post']['url']})\n\n"
       if images_only is not True:
         if 'body' in p['post']:
-          posttext += shorten_text(p['post']['body'], rapidkey)
+          posttext += shorten.shorten_text(p['post']['body'], rapidkey)
     elif "url" in p['post']:
       print(f"* {p['post']['name']} - {p['post']['url']}")
       if "url_content_type" in p['post']:
@@ -367,20 +353,16 @@ def run(user, pw, instance, postcomm, cfg, post_title, images_only, nsfw_b, modu
             if t is not None:
               posttext += t
             else:
-              print('pipfeed...')
-              t = pipfeed.extract(rapidkey, p['post']['url'])
+              # use news3k to get an article image
+              print('news...')
+              try:
+                t = news.article(p['post']['url'], rapidkey)
+              except Exception as e:
+                print(f'failed to use news3k to get article: {e}')
+                t = None
               if t is not None:
                 posttext += t
               else:
-                # use news3k to get an article image
-                print('news...')
-                try:
-                  t = news.article_image(p['post']['url'])
-                except Exception as e:
-                  print(f'failed to use news3k to get article: {e}')
-                  t = None
-                if t is not None:
-                  posttext += t
                 # add title/desc from lemmy api
                 print('lemmy fallback 1...')
                 t = add_embed(p['post'])
@@ -389,12 +371,12 @@ def run(user, pw, instance, postcomm, cfg, post_title, images_only, nsfw_b, modu
                 else:
                   print('lemmy fallback 2...')
                   if 'body' in p['post']:
-                    posttext += shorten_text(p['post']['body'], rapidkey)
+                    posttext += shorten.shorten_text(p['post']['body'], rapidkey)
         elif p['post']['url_content_type'][:6] == 'video/':
           # embed video url
           posttext = posttext + f"![]({p['post']['url']})\n\n"
           if 'body' in p['post']:
-            posttext += shorten_text(p['post']['body'], rapidkey)
+            posttext += shorten.shorten_text(p['post']['body'], rapidkey)
       else:
         '''no content type'''
         t = add_embed(p['post'])
@@ -406,18 +388,18 @@ def run(user, pw, instance, postcomm, cfg, post_title, images_only, nsfw_b, modu
           if t is not None:
             posttext += t
           if 'body' in p['post']:
-            posttext += shorten_text(p['post']['body'], rapidkey)
+            posttext += shorten.shorten_text(p['post']['body'], rapidkey)
     else:
       '''not url'''
       if 'body' in p['post']:
-        posttext += shorten_text(p['post']['body'], rapidkey)
+        posttext += shorten.shorten_text(p['post']['body'], rapidkey)
 
     posttext = posttext + f"Posted by [{p['author']['name']}]({p['author']['actor_id']})\n\n"
 
   posttext += "\n\n----\n\nThe main links are using lemmyverse.link which should redirect to the post on your own instance. If you have not used this before, you may need to go direct to https://lemmyverse.link/ and click on 'configure instance'.  Some apps will open posts correctly when using the direct link."
 
   if images_only is not True:
-    posttext += "\n\n️🤖 indicates a summary generated using AI - 🖋️ TLDR This, 🖊️ Pipfeed, 🖍 Deepseek.  It is possible that the summary does not accurately convey the meaning of the original article, refer to the source material if in any doubt."
+    posttext += "\n\n️🤖 indicates a summary generated using AI - 🖋️ TLDR This, 🖊️ news3k, 🖍 Deepseek.  It is possible that the summary does not accurately convey the meaning of the original article, refer to the source material if in any doubt."
 
   print(posttext)
 

news.py

@@ -1,6 +1,8 @@
 #!/usr/bin/python3
 from newspaper import Article
 from newspaper import Config
+import shorten
+import nltk
 
 def article_image(url):
   config = Config()
@@ -20,3 +22,31 @@ def article_image(url):
   #print(article.text)
 
   return None
+
+
+def article(url, rapidkey):
+  config = Config()
+  config.browser_user_agent = "Mozilla/5.0"
+
+  art = Article(url)
+  try:
+    art.download()
+  except Exception as e:
+    print(f'unable to download article, {e}')
+    return None
+  art.parse()
+  nltk.download('punkt')
+  art.nlp()
+  t = ''
+  if art.top_image is not None:
+    t = f'![]({art.top_image})\n\n'
+  if art.title is not None:
+    t += f'*{art.title}*\n\n'
+  if art.summary is not None:
+    t += f'{art.summary} 🖊️️🤖\n\n'
+  elif art.text is not None:
+    t += shorten.shorten_text(art.text, rapidkey)
+  if len(t) > 0:
+    return t
+    
+  return None

pipfeed.py

@@ -6,4 +6,4 @@ aiohttp
 pytube
 newspaper3k
 lxml_html_clean
-
+nltk

requirements.txt

@@ -0,0 +1,16 @@
+#!/usr/bin/python3
+import deepseek
+
+def shorten_text(text, deepseek_key):
+  max_len = 300
+  if len(text) > max_len:
+    print('shorten...')
+    try:
+      t = deepseek.shorten(text, deepseek_key, max_len)
+      t = f'{t} 🖍🤖\n\n'
+      return t
+    except Exception as e:
+      print(f'deepseek raised exception: {e}')
+    return f'{text[:297]}...\n\n' # NB: three less than max_len
+  else:
+    return f'{text}\n\n'