From 3b5e66af40b46460de55e9fb85be794e5c811593 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Mon, 26 Aug 2019 15:34:37 +0200 Subject: [PATCH] push code from main to new modules --- feed2toot/hashtags.py | 66 +++++++++++++++ feed2toot/main.py | 147 ++++++---------------------------- feed2toot/message.py | 72 +++++++++++++++++ feed2toot/plugins/__init__.py | 16 ++++ feed2toot/rss.py | 42 ++++++++++ feed2toot/sortentries.py | 42 ++++++++++ 6 files changed, 262 insertions(+), 123 deletions(-) create mode 100644 feed2toot/hashtags.py create mode 100644 feed2toot/message.py create mode 100644 feed2toot/rss.py create mode 100644 feed2toot/sortentries.py diff --git a/feed2toot/hashtags.py b/feed2toot/hashtags.py new file mode 100644 index 0000000..88405b0 --- /dev/null +++ b/feed2toot/hashtags.py @@ -0,0 +1,66 @@ +# vim:ts=4:sw=4:ft=python:fileencoding=utf-8 +# Copyright © 2015-2019 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see + +'''Manage a lock file''' + +# standard libraires imports +import codecs + +def extract_hashtags_from_list(options): + '''extract hashtags from the the list''' + if 'hashtaglist' in options and options['hashtaglist']: + severalwordshashtags = codecs.open(options['hashtaglist'], + encoding='utf-8').readlines() + severalwordshashtags = [i.rstrip('\n') for i in severalwordshashtags] + else: + severalwordshashtags = [] + return severalwordshashtags + +def build_hashtags(entry, rss, options, severalwordshashtags): + '''build hashtags''' + severalwordsinhashtag = False + # has the the rss feed hashtag + if 'tags' in entry and options['addtags']: + hastags = True + else: + hastags = False + if hastags: + rss['hashtags'] = [] + for i, _ in enumerate(entry['tags']): + if 'hashtaglist' in options: + prehashtags = entry['tags'][i]['term'] + tmphashtags = entry['tags'][i]['term'] + for element in severalwordshashtags: + if element in prehashtags: + severalwordsinhashtag = True + tmphashtags = prehashtags.replace(element, + ''.join(element.split())) + # replace characters stopping a word from being a hashtag + if severalwordsinhashtag: + # remove ' from hashtag + tmphashtags = tmphashtags.replace("'", "") + # remove - from hashtag + tmphashtags = tmphashtags.replace("-", "") + # remove . from hashtag + tmphashtags = tmphashtags.replace(".", "") + # remove space from hashtag + finalhashtags = tmphashtags.replace(" ", "") + rss['hashtags'].append('#{}'.format(finalhashtags)) + else: + nospace = ''.join(entry['tags'][i]['term']) + # remove space from hashtag + nospace = nospace.replace(" ", "") + rss['hashtags'].append('#{}'.format(nospace)) + return rss diff --git a/feed2toot/main.py b/feed2toot/main.py index cebb434..ff2041c 100644 --- a/feed2toot/main.py +++ b/feed2toot/main.py @@ -24,16 +24,25 @@ import logging.handlers import sys import re +# external liraries imports +from bs4 import BeautifulSoup + # app libraries imports from feed2toot.addtags import AddTags from feed2toot.cliparse import CliParse from feed2toot.confparse import ConfParse -from feed2toot.filterentry import FilterEntry -from feed2toot.removeduplicates import RemoveDuplicates -from feed2toot.tootpost import TootPost from feed2toot.feedcache import FeedCache +from feed2toot.filterentry import FilterEntry +from feed2toot.hashtags import build_hashtags +from feed2toot.hashtags import extract_hashtags_from_list from feed2toot.lock import LockFile -from bs4 import BeautifulSoup +from feed2toot.message import build_message +from feed2toot.message import send_message_dry_run +from feed2toot.message import send_message +from feed2toot.removeduplicates import RemoveDuplicates +from feed2toot.rss import populate_rss +from feed2toot.sortentries import sort_entries +from feed2toot.tootpost import TootPost class Main: '''Main class of Feed2toot''' @@ -59,7 +68,7 @@ class Main: logging.debug('configured stdout level %s' % sh.level) def main(self): - """The main function.""" + '''The main function''' clip = CliParse() clioptions = clip.options self.setup_logging(clioptions) @@ -76,12 +85,7 @@ class Main: lockfile = LockFile(options['lockfile'], options['locktimeout']) # create link to the persistent list cache = FeedCache(options) - if 'hashtaglist' in options and options['hashtaglist']: - severalwordshashtags = codecs.open(options['hashtaglist'], - encoding='utf-8').readlines() - severalwordshashtags = [i.rstrip('\n') for i in severalwordshashtags] - else: - severalwordshashtags = [] + severalwordshashtags = extract_hashtags_from_list(options) # reverse feed entries because most recent one should be sent as the last one in Mastodon for feed in feeds: # store the patterns by rss @@ -96,73 +100,12 @@ class Main: sys.exit(0) else: sys.exit('Could not parse the section of the rss feed') - totweet = [] - # cache the ids of last rss feeds - if not clioptions.all: - for i in entries: - if 'id' in i: - if i['id'] not in cache.getdeque(): - totweet.append(i) - elif 'guid' in i: - if i['guid'] not in cache.getdeque(): - totweet.append(i) - else: - # if id or guid not in the entry, use link - if i['link'] not in cache.getdeque(): - totweet.append(i) - else: - totweet = entries - + # sort entries and check if they were not previously sent + totweet = sort_entries(clioptions.all, cache, entries) for entry in totweet: - if 'id' in entry: - logging.debug('found feed entry {entryid}'.format(entryid=entry['id'])) - rss = { - 'id': entry['id'], - } - elif 'guid' in entry: - logging.debug('found feed entry {entryid}'.format(entryid=entry['guid'])) - rss = { - 'id': entry['guid'], - } - else: - logging.debug('found feed entry {entryid}'.format(entryid=entry['link'])) - rss = { - 'id': entry['link'], - } - - severalwordsinhashtag = False - # lets see if the rss feed has hashtag - if 'tags' in entry and options['addtags']: - hastags = True - else: - hastags = False - if hastags: - rss['hashtags'] = [] - for i, _ in enumerate(entry['tags']): - if 'hashtaglist' in options: - prehashtags = entry['tags'][i]['term'] - tmphashtags = entry['tags'][i]['term'] - for element in severalwordshashtags: - if element in prehashtags: - severalwordsinhashtag = True - tmphashtags = prehashtags.replace(element, - ''.join(element.split())) - # replace characters stopping a word from being a hashtag - if severalwordsinhashtag: - # remove ' from hashtag - tmphashtags = tmphashtags.replace("'", "") - # remove - from hashtag - tmphashtags = tmphashtags.replace("-", "") - # remove . from hashtag - tmphashtags = tmphashtags.replace(".", "") - # remove space from hashtag - finalhashtags = tmphashtags.replace(" ", "") - rss['hashtags'].append('#{}'.format(finalhashtags)) - else: - nospace = ''.join(entry['tags'][i]['term']) - # remove space from hashtag - nospace = nospace.replace(" ", "") - rss['hashtags'].append('#{}'.format(nospace)) + # populate rss with new entry to send + rss = populate_rss(entry) + rss = build_hashtags(entry, rss, options, severalwordshashtags) # parse tweetfomat to elements elements = re.findall(r"\{(.*?)\}",tweetformat) # strip : from elements to allow string formating, eg. {title:.20} @@ -172,56 +115,14 @@ class Main: fe = FilterEntry(elements, entry, options, feed['patterns'], feed['rssobject'], feed['feedname']) entrytosend = fe.finalentry if entrytosend: - tweetwithnotag = tweetformat.format(**entrytosend) - # remove duplicates from the final tweet - dedup = RemoveDuplicates(tweetwithnotag) - # only append hashtags if they exist - # remove last tags if tweet too long - if 'hashtags' in rss: - addtag = AddTags(dedup.finaltweet, rss['hashtags']) - finaltweet = addtag.finaltweet - else: - finaltweet = dedup.finaltweet - - # strip html tags - finaltweet = BeautifulSoup(finaltweet, 'html.parser').get_text() - + finaltweet = build_message(entrytosend, tweetformat, rss) if clioptions.dryrun: - if entrytosend: - logging.warning('Would toot with visibility "{visibility}": {toot}'.format( - toot=finaltweet, - visibility=config.get( - 'mastodon', 'toot_visibility', - fallback='public'))) - else: - logging.debug('This rss entry did not meet pattern criteria. Should have not been sent') + send_message_dry_run(config, entrytosend, finaltweet) else: - storeit = True - if entrytosend and not clioptions.populate: - logging.debug('Tooting with visibility "{visibility}": {toot}'.format( - toot=finaltweet, - visibility=config.get( - 'mastodon', 'toot_visibility', - fallback='public'))) - twp = TootPost(config, options, finaltweet) - storeit = twp.storeit() - else: - logging.debug('populating RSS entry {}'.format(rss['id'])) - # in both cas we store the id of the sent tweet - if storeit: - cache.append(rss['id']) + send_message(config, clioptions, options, entrytosend, finaltweet, cache, rss) # plugins if plugins and entrytosend: - for plugin in plugins: - capitalizedplugin = plugin.title() - pluginclassname = '{plugin}Plugin'.format(plugin=capitalizedplugin) - pluginmodulename = 'feed2toot.plugins.{pluginmodule}'.format(pluginmodule=pluginclassname.lower()) - try: - pluginmodule = importlib.import_module(pluginmodulename) - pluginclass = getattr(pluginmodule, pluginclassname) - pluginclass(plugins[plugin], finaltweet) - except ImportError as err: - print(err) + activate_plugins(plugins) # do not forget to close cache (shelf object) cache.close() # release the lock file diff --git a/feed2toot/message.py b/feed2toot/message.py new file mode 100644 index 0000000..8ccbad0 --- /dev/null +++ b/feed2toot/message.py @@ -0,0 +1,72 @@ +# vim:ts=4:sw=4:ft=python:fileencoding=utf-8 +# Copyright © 2015-2019 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see + +'''Build the message''' + +# standard libraires imports +import logging + +# external liraries imports +from bs4 import BeautifulSoup + +# app libraries imports +from feed2toot.addtags import AddTags +from feed2toot.removeduplicates import RemoveDuplicates +from feed2toot.tootpost import TootPost + +def build_message(entrytosend, tweetformat, rss): + '''populate the rss dict with the new entry''' + tweetwithnotag = tweetformat.format(**entrytosend) + # remove duplicates from the final tweet + dedup = RemoveDuplicates(tweetwithnotag) + # only append hashtags if they exist + # remove last tags if tweet too long + if 'hashtags' in rss: + addtag = AddTags(dedup.finaltweet, rss['hashtags']) + finaltweet = addtag.finaltweet + else: + finaltweet = dedup.finaltweet + + # strip html tags + finaltweet = BeautifulSoup(finaltweet, 'html.parser').get_text() + return finaltweet + +def send_message_dry_run(config, entrytosend, finaltweet): + '''simulate sending message using dry run mode''' + if entrytosend: + logging.warning('Would toot with visibility "{visibility}": {toot}'.format( + toot=finaltweet, + visibility=config.get( + 'mastodon', 'toot_visibility', + fallback='public'))) + else: + logging.debug('This rss entry did not meet pattern criteria. Should have not been sent') + +def send_message(config, clioptions, options, entrytosend, finaltweet, cache, rss): + '''send message''' + storeit = True + if entrytosend and not clioptions.populate: + logging.debug('Tooting with visibility "{visibility}": {toot}'.format( + toot=finaltweet, + visibility=config.get( + 'mastodon', 'toot_visibility', + fallback='public'))) + twp = TootPost(config, options, finaltweet) + storeit = twp.storeit() + else: + logging.debug('populating RSS entry {}'.format(rss['id'])) + # in both cas we store the id of the sent tweet + if storeit: + cache.append(rss['id']) diff --git a/feed2toot/plugins/__init__.py b/feed2toot/plugins/__init__.py index d19bf9f..bcccc6a 100644 --- a/feed2toot/plugins/__init__.py +++ b/feed2toot/plugins/__init__.py @@ -13,3 +13,19 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see + +# standard libraires imports +import importlib + +def activate_plugins(entrytosend): + '''activate plugins''' + for plugin in plugins: + capitalizedplugin = plugin.title() + pluginclassname = '{plugin}Plugin'.format(plugin=capitalizedplugin) + pluginmodulename = 'feed2toot.plugins.{pluginmodule}'.format(pluginmodule=pluginclassname.lower()) + try: + pluginmodule = importlib.import_module(pluginmodulename) + pluginclass = getattr(pluginmodule, pluginclassname) + pluginclass(plugins[plugin], finaltweet) + except ImportError as err: + print(err) diff --git a/feed2toot/rss.py b/feed2toot/rss.py new file mode 100644 index 0000000..3edbbd7 --- /dev/null +++ b/feed2toot/rss.py @@ -0,0 +1,42 @@ +# vim:ts=4:sw=4:ft=python:fileencoding=utf-8 +# Copyright © 2015-2019 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see + +'''Manage a lock file''' + +# standard libraires imports +import datetime +import logging +import os +import os.path +import sys + +def populate_rss(entry): + '''populate the rss dict with the new entry''' + if 'id' in entry: + logging.debug('found feed entry {entryid}'.format(entryid=entry['id'])) + rss = { + 'id': entry['id'], + } + elif 'guid' in entry: + logging.debug('found feed entry {entryid}'.format(entryid=entry['guid'])) + rss = { + 'id': entry['guid'], + } + else: + logging.debug('found feed entry {entryid}'.format(entryid=entry['link'])) + rss = { + 'id': entry['link'], + } + return rss diff --git a/feed2toot/sortentries.py b/feed2toot/sortentries.py new file mode 100644 index 0000000..09d02ba --- /dev/null +++ b/feed2toot/sortentries.py @@ -0,0 +1,42 @@ +# vim:ts=4:sw=4:ft=python:fileencoding=utf-8 +# Copyright © 2015-2019 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see + +'''Manage a lock file''' + +# standard libraires imports +import datetime +import logging +import os +import os.path +import sys + +def sort_entries(is_all, cache, entries): + '''sort entries before sending''' + totweet = [] + if not is_all: + for i in entries: + if 'id' in i: + if i['id'] not in cache.getdeque(): + totweet.append(i) + elif 'guid' in i: + if i['guid'] not in cache.getdeque(): + totweet.append(i) + else: + # if id or guid not in the entry, use link + if i['link'] not in cache.getdeque(): + totweet.append(i) + else: + totweet = entries + return totweet