push code from main to new modules

feed2toot/hashtags.py

@@ -0,0 +1,66 @@
+# vim:ts=4:sw=4:ft=python:fileencoding=utf-8
+# Copyright © 2015-2019 Carl Chenet <carl.chenet@ohmytux.com>
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+
+'''Manage a lock file'''
+
+# standard libraires imports
+import codecs
+
+def extract_hashtags_from_list(options):
+    '''extract hashtags from the the list''' 
+    if 'hashtaglist' in options and options['hashtaglist']:
+        severalwordshashtags = codecs.open(options['hashtaglist'],
+                                           encoding='utf-8').readlines()
+        severalwordshashtags = [i.rstrip('\n') for i in severalwordshashtags]
+    else:
+        severalwordshashtags = []
+    return severalwordshashtags
+
+def build_hashtags(entry, rss, options, severalwordshashtags):
+    '''build hashtags'''
+    severalwordsinhashtag = False
+    # has the the rss feed hashtag
+    if 'tags' in entry and options['addtags']:
+        hastags = True
+    else:
+        hastags = False
+    if hastags:
+        rss['hashtags'] = []
+        for i, _ in enumerate(entry['tags']):
+            if 'hashtaglist' in options:
+                prehashtags = entry['tags'][i]['term']
+                tmphashtags = entry['tags'][i]['term']
+                for element in severalwordshashtags:
+                    if element in prehashtags:
+                        severalwordsinhashtag = True
+                        tmphashtags = prehashtags.replace(element,
+                                                          ''.join(element.split()))
+            # replace characters stopping a word from being a hashtag
+            if severalwordsinhashtag:
+                # remove ' from hashtag
+                tmphashtags = tmphashtags.replace("'", "")
+                # remove - from hashtag
+                tmphashtags = tmphashtags.replace("-", "")
+                # remove . from hashtag
+                tmphashtags = tmphashtags.replace(".", "")
+                # remove space from hashtag
+                finalhashtags = tmphashtags.replace(" ", "")
+                rss['hashtags'].append('#{}'.format(finalhashtags))
+            else:
+                nospace = ''.join(entry['tags'][i]['term'])
+                # remove space from hashtag
+                nospace = nospace.replace(" ", "")
+                rss['hashtags'].append('#{}'.format(nospace))
+    return rss

feed2toot/main.py

@@ -24,16 +24,25 @@ import logging.handlers
 import sys
 import re
 
+# external liraries imports
+from bs4 import BeautifulSoup
+
 # app libraries imports
 from feed2toot.addtags import AddTags
 from feed2toot.cliparse import CliParse
 from feed2toot.confparse import ConfParse
-from feed2toot.filterentry import FilterEntry
-from feed2toot.removeduplicates import RemoveDuplicates
-from feed2toot.tootpost import TootPost
 from feed2toot.feedcache import FeedCache
+from feed2toot.filterentry import FilterEntry
+from feed2toot.hashtags import build_hashtags
+from feed2toot.hashtags import extract_hashtags_from_list
 from feed2toot.lock import LockFile
-from bs4 import BeautifulSoup
+from feed2toot.message import build_message
+from feed2toot.message import send_message_dry_run
+from feed2toot.message import send_message
+from feed2toot.removeduplicates import RemoveDuplicates
+from feed2toot.rss import populate_rss
+from feed2toot.sortentries import sort_entries
+from feed2toot.tootpost import TootPost
 
 class Main:
     '''Main class of Feed2toot'''
@@ -59,7 +68,7 @@ class Main:
         logging.debug('configured stdout level %s' % sh.level)
 
     def main(self):
-        """The main function."""
+        '''The main function'''
         clip = CliParse()
         clioptions = clip.options
         self.setup_logging(clioptions)
@@ -76,12 +85,7 @@ class Main:
             lockfile = LockFile(options['lockfile'], options['locktimeout'])
             # create link to the persistent list
             cache = FeedCache(options)
-            if 'hashtaglist' in options and options['hashtaglist']:
+            severalwordshashtags = extract_hashtags_from_list(options)
-                severalwordshashtags = codecs.open(options['hashtaglist'],
-                                                   encoding='utf-8').readlines()
-                severalwordshashtags = [i.rstrip('\n') for i in severalwordshashtags]
-            else:
-                severalwordshashtags = []
             # reverse feed entries because most recent one should be sent as the last one in Mastodon
             for feed in feeds:
                 # store the patterns by rss
@@ -96,73 +100,12 @@ class Main:
                         sys.exit(0)
                     else:
                         sys.exit('Could not parse the section of the rss feed')
-                totweet = []
+                # sort entries and check if they were not previously sent
-                # cache the ids of last rss feeds
+                totweet = sort_entries(clioptions.all, cache, entries)
-                if not clioptions.all:
-                    for i in entries:
-                        if 'id' in i:
-                            if i['id'] not in cache.getdeque():
-                                totweet.append(i)
-                        elif 'guid' in i:
-                            if i['guid'] not in cache.getdeque():
-                                totweet.append(i)
-                        else:
-                            # if id or guid not in the entry, use link
-                            if i['link'] not in cache.getdeque():
-                                totweet.append(i)
-                else:
-                    totweet = entries
-
                 for entry in totweet:
-                    if 'id' in entry:
+                    # populate rss with new entry to send
-                        logging.debug('found feed entry {entryid}'.format(entryid=entry['id']))
+                    rss = populate_rss(entry)
-                        rss = {
+                    rss = build_hashtags(entry, rss, options, severalwordshashtags)
-                            'id': entry['id'],
-                        }
-                    elif 'guid' in entry:
-                        logging.debug('found feed entry {entryid}'.format(entryid=entry['guid']))
-                        rss = {
-                            'id': entry['guid'],
-                        }
-                    else:
-                        logging.debug('found feed entry {entryid}'.format(entryid=entry['link']))
-                        rss = {
-                            'id': entry['link'],
-                        }
-
-                    severalwordsinhashtag = False
-                    # lets see if the rss feed has hashtag
-                    if 'tags' in entry and options['addtags']:
-                        hastags = True
-                    else:
-                        hastags = False
-                    if hastags:
-                        rss['hashtags'] = []
-                        for i, _ in enumerate(entry['tags']):
-                            if 'hashtaglist' in options:
-                                prehashtags = entry['tags'][i]['term']
-                                tmphashtags = entry['tags'][i]['term']
-                                for element in severalwordshashtags:
-                                    if element in prehashtags:
-                                        severalwordsinhashtag = True
-                                        tmphashtags = prehashtags.replace(element,
-                                                                          ''.join(element.split()))
-                            # replace characters stopping a word from being a hashtag
-                            if severalwordsinhashtag:
-                                # remove ' from hashtag
-                                tmphashtags = tmphashtags.replace("'", "")
-                                # remove - from hashtag
-                                tmphashtags = tmphashtags.replace("-", "")
-                                # remove . from hashtag
-                                tmphashtags = tmphashtags.replace(".", "")
-                                # remove space from hashtag
-                                finalhashtags = tmphashtags.replace(" ", "")
-                                rss['hashtags'].append('#{}'.format(finalhashtags))
-                            else:
-                                nospace = ''.join(entry['tags'][i]['term'])
-                                # remove space from hashtag
-                                nospace = nospace.replace(" ", "")
-                                rss['hashtags'].append('#{}'.format(nospace))
                     # parse tweetfomat to elements
                     elements = re.findall(r"\{(.*?)\}",tweetformat)
                     # strip : from elements to allow string formating, eg. {title:.20}
@@ -172,56 +115,14 @@ class Main:
                     fe = FilterEntry(elements, entry, options, feed['patterns'], feed['rssobject'], feed['feedname'])
                     entrytosend = fe.finalentry
                     if entrytosend:
-                        tweetwithnotag = tweetformat.format(**entrytosend)
+                        finaltweet = build_message(entrytosend, tweetformat, rss)
-                        # remove duplicates from the final tweet
-                        dedup = RemoveDuplicates(tweetwithnotag)
-                        # only append hashtags if they exist
-                        # remove last tags if tweet too long
-                        if 'hashtags' in rss:
-                            addtag = AddTags(dedup.finaltweet, rss['hashtags'])
-                            finaltweet = addtag.finaltweet
-                        else:
-                            finaltweet = dedup.finaltweet
-
-                        # strip html tags
-                        finaltweet = BeautifulSoup(finaltweet, 'html.parser').get_text()
-
                     if clioptions.dryrun:
-                        if entrytosend:
+                        send_message_dry_run(config, entrytosend, finaltweet)
-                            logging.warning('Would toot with visibility "{visibility}": {toot}'.format(
-                                toot=finaltweet,
-                                visibility=config.get(
-                                    'mastodon', 'toot_visibility',
-                                    fallback='public')))
                     else:
-                            logging.debug('This rss entry did not meet pattern criteria. Should have not been sent')
+                        send_message(config, clioptions, options, entrytosend, finaltweet, cache, rss)
-                    else:
-                        storeit = True
-                        if entrytosend and not clioptions.populate:
-                            logging.debug('Tooting with visibility "{visibility}": {toot}'.format(
-                                toot=finaltweet,
-                                visibility=config.get(
-                                    'mastodon', 'toot_visibility',
-                                    fallback='public')))
-                            twp = TootPost(config, options, finaltweet)
-                            storeit = twp.storeit()
-                        else:
-                            logging.debug('populating RSS entry {}'.format(rss['id']))
-                        # in both cas we store the id of the sent tweet
-                        if storeit:
-                            cache.append(rss['id'])
                         # plugins
                         if plugins and entrytosend:
-                            for plugin in plugins:
+                            activate_plugins(plugins)
-                                capitalizedplugin = plugin.title()
-                                pluginclassname = '{plugin}Plugin'.format(plugin=capitalizedplugin)
-                                pluginmodulename = 'feed2toot.plugins.{pluginmodule}'.format(pluginmodule=pluginclassname.lower())
-                                try:
-                                    pluginmodule = importlib.import_module(pluginmodulename)
-                                    pluginclass = getattr(pluginmodule, pluginclassname)
-                                    pluginclass(plugins[plugin], finaltweet)
-                                except ImportError as err:
-                                    print(err)
             # do not forget to close cache (shelf object)
             cache.close()
             # release the lock file

feed2toot/message.py

@@ -0,0 +1,72 @@
+# vim:ts=4:sw=4:ft=python:fileencoding=utf-8
+# Copyright © 2015-2019 Carl Chenet <carl.chenet@ohmytux.com>
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+
+'''Build the message'''
+
+# standard libraires imports
+import logging
+
+# external liraries imports
+from bs4 import BeautifulSoup
+
+# app libraries imports
+from feed2toot.addtags import AddTags
+from feed2toot.removeduplicates import RemoveDuplicates
+from feed2toot.tootpost import TootPost
+
+def build_message(entrytosend, tweetformat, rss):
+    '''populate the rss dict with the new entry'''
+    tweetwithnotag = tweetformat.format(**entrytosend)
+    # remove duplicates from the final tweet
+    dedup = RemoveDuplicates(tweetwithnotag)
+    # only append hashtags if they exist
+    # remove last tags if tweet too long
+    if 'hashtags' in rss:
+        addtag = AddTags(dedup.finaltweet, rss['hashtags'])
+        finaltweet = addtag.finaltweet
+    else:
+        finaltweet = dedup.finaltweet
+
+    # strip html tags
+    finaltweet = BeautifulSoup(finaltweet, 'html.parser').get_text()
+    return finaltweet
+
+def send_message_dry_run(config, entrytosend, finaltweet):
+    '''simulate sending message using dry run mode'''
+    if entrytosend:
+        logging.warning('Would toot with visibility "{visibility}": {toot}'.format(
+            toot=finaltweet,
+            visibility=config.get(
+                'mastodon', 'toot_visibility',
+                fallback='public')))
+    else:
+        logging.debug('This rss entry did not meet pattern criteria. Should have not been sent')
+
+def send_message(config, clioptions, options, entrytosend, finaltweet, cache, rss):
+    '''send message'''
+    storeit = True
+    if entrytosend and not clioptions.populate:
+        logging.debug('Tooting with visibility "{visibility}": {toot}'.format(
+            toot=finaltweet,
+            visibility=config.get(
+                'mastodon', 'toot_visibility',
+                fallback='public')))
+        twp = TootPost(config, options, finaltweet)
+        storeit = twp.storeit()
+    else:
+        logging.debug('populating RSS entry {}'.format(rss['id']))
+    # in both cas we store the id of the sent tweet
+    if storeit:
+        cache.append(rss['id'])

feed2toot/plugins/__init__.py

@@ -13,3 +13,19 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>
+
+# standard libraires imports
+import importlib
+
+def activate_plugins(entrytosend):
+    '''activate plugins'''
+    for plugin in plugins:
+        capitalizedplugin = plugin.title()
+        pluginclassname = '{plugin}Plugin'.format(plugin=capitalizedplugin)
+        pluginmodulename = 'feed2toot.plugins.{pluginmodule}'.format(pluginmodule=pluginclassname.lower())
+        try:
+            pluginmodule = importlib.import_module(pluginmodulename)
+            pluginclass = getattr(pluginmodule, pluginclassname)
+            pluginclass(plugins[plugin], finaltweet)
+        except ImportError as err:
+            print(err)

feed2toot/rss.py

@@ -0,0 +1,42 @@
+# vim:ts=4:sw=4:ft=python:fileencoding=utf-8
+# Copyright © 2015-2019 Carl Chenet <carl.chenet@ohmytux.com>
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+
+'''Manage a lock file'''
+
+# standard libraires imports
+import datetime
+import logging
+import os
+import os.path
+import sys
+
+def populate_rss(entry):
+    '''populate the rss dict with the new entry'''
+    if 'id' in entry:
+        logging.debug('found feed entry {entryid}'.format(entryid=entry['id']))
+        rss = {
+            'id': entry['id'],
+        }
+    elif 'guid' in entry:
+        logging.debug('found feed entry {entryid}'.format(entryid=entry['guid']))
+        rss = {
+            'id': entry['guid'],
+        }
+    else:
+        logging.debug('found feed entry {entryid}'.format(entryid=entry['link']))
+        rss = {
+            'id': entry['link'],
+        }
+    return rss

feed2toot/sortentries.py

@@ -0,0 +1,42 @@
+# vim:ts=4:sw=4:ft=python:fileencoding=utf-8
+# Copyright © 2015-2019 Carl Chenet <carl.chenet@ohmytux.com>
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+
+'''Manage a lock file'''
+
+# standard libraires imports
+import datetime
+import logging
+import os
+import os.path
+import sys
+
+def sort_entries(is_all, cache, entries):
+    '''sort entries before sending'''
+    totweet = []
+    if not is_all:
+        for i in entries:
+            if 'id' in i:
+                if i['id'] not in cache.getdeque():
+                    totweet.append(i)
+            elif 'guid' in i:
+                if i['guid'] not in cache.getdeque():
+                    totweet.append(i)
+            else:
+                # if id or guid not in the entry, use link
+                if i['link'] not in cache.getdeque():
+                    totweet.append(i)
+    else:
+        totweet = entries
+    return totweet