massivebox/feed2toot-docker
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

push code from main to new modules

Browse source
This commit is contained in:
Carl Chenet 2019-08-26 15:34:37 +02:00
parent ec664e24d3
commit 3b5e66af40
6 changed files with 262 additions and 123 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

66
feed2toot/hashtags.py Normal file
View file

@ -0,0 +1,66 @@
# vim:ts=4:sw=4:ft=python:fileencoding=utf-8
# Copyright © 2015-2019 Carl Chenet <carl.chenet@ohmytux.com>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>
'''Manage a lock file'''
# standard libraires imports
import codecs
def extract_hashtags_from_list(options):
'''extract hashtags from the the list'''
if 'hashtaglist' in options and options['hashtaglist']:
severalwordshashtags = codecs.open(options['hashtaglist'],
encoding='utf-8').readlines()
severalwordshashtags = [i.rstrip('\n') for i in severalwordshashtags]
else:
severalwordshashtags = []
return severalwordshashtags
def build_hashtags(entry, rss, options, severalwordshashtags):
'''build hashtags'''
severalwordsinhashtag = False
# has the the rss feed hashtag
if 'tags' in entry and options['addtags']:
hastags = True
else:
hastags = False
if hastags:
rss['hashtags'] = []
for i, _ in enumerate(entry['tags']):
if 'hashtaglist' in options:
prehashtags = entry['tags'][i]['term']
tmphashtags = entry['tags'][i]['term']
for element in severalwordshashtags:
if element in prehashtags:
severalwordsinhashtag = True
tmphashtags = prehashtags.replace(element,
''.join(element.split()))
# replace characters stopping a word from being a hashtag
if severalwordsinhashtag:
# remove ' from hashtag
tmphashtags = tmphashtags.replace("'", "")
# remove - from hashtag
tmphashtags = tmphashtags.replace("-", "")
# remove . from hashtag
tmphashtags = tmphashtags.replace(".", "")
# remove space from hashtag
finalhashtags = tmphashtags.replace(" ", "")
rss['hashtags'].append('#{}'.format(finalhashtags))
else:
nospace = ''.join(entry['tags'][i]['term'])
# remove space from hashtag
nospace = nospace.replace(" ", "")
rss['hashtags'].append('#{}'.format(nospace))
return rss

147
feed2toot/main.py
View file

@ -24,16 +24,25 @@ import logging.handlers
import sys
import re
# external liraries imports
from bs4 import BeautifulSoup
# app libraries imports
from feed2toot.addtags import AddTags
from feed2toot.cliparse import CliParse
from feed2toot.confparse import ConfParse
from feed2toot.filterentry import FilterEntry
from feed2toot.removeduplicates import RemoveDuplicates
from feed2toot.tootpost import TootPost
from feed2toot.feedcache import FeedCache
from feed2toot.filterentry import FilterEntry
from feed2toot.hashtags import build_hashtags
from feed2toot.hashtags import extract_hashtags_from_list
from feed2toot.lock import LockFile
from bs4 import BeautifulSoup
from feed2toot.message import build_message
from feed2toot.message import send_message_dry_run
from feed2toot.message import send_message
from feed2toot.removeduplicates import RemoveDuplicates
from feed2toot.rss import populate_rss
from feed2toot.sortentries import sort_entries
from feed2toot.tootpost import TootPost
class Main:
'''Main class of Feed2toot'''
@ -59,7 +68,7 @@ class Main:
logging.debug('configured stdout level %s' % sh.level)
def main(self):
"""The main function."""
'''The main function'''
clip = CliParse()
clioptions = clip.options
self.setup_logging(clioptions)
@ -76,12 +85,7 @@ class Main:
lockfile = LockFile(options['lockfile'], options['locktimeout'])
# create link to the persistent list
cache = FeedCache(options)
if 'hashtaglist' in options and options['hashtaglist']:
severalwordshashtags = codecs.open(options['hashtaglist'],
encoding='utf-8').readlines()
severalwordshashtags = [i.rstrip('\n') for i in severalwordshashtags]
else:
severalwordshashtags = []
severalwordshashtags = extract_hashtags_from_list(options)
# reverse feed entries because most recent one should be sent as the last one in Mastodon
for feed in feeds:
# store the patterns by rss
@ -96,73 +100,12 @@ class Main:
sys.exit(0)
else:
sys.exit('Could not parse the section of the rss feed')
totweet = []
# cache the ids of last rss feeds
if not clioptions.all:
for i in entries:
if 'id' in i:
if i['id'] not in cache.getdeque():
totweet.append(i)
elif 'guid' in i:
if i['guid'] not in cache.getdeque():
totweet.append(i)
else:
# if id or guid not in the entry, use link
if i['link'] not in cache.getdeque():
totweet.append(i)
else:
totweet = entries
# sort entries and check if they were not previously sent
totweet = sort_entries(clioptions.all, cache, entries)
for entry in totweet:
if 'id' in entry:
logging.debug('found feed entry {entryid}'.format(entryid=entry['id']))
rss = {
'id': entry['id'],
}
elif 'guid' in entry:
logging.debug('found feed entry {entryid}'.format(entryid=entry['guid']))
rss = {
'id': entry['guid'],
}
else:
logging.debug('found feed entry {entryid}'.format(entryid=entry['link']))
rss = {
'id': entry['link'],
}
severalwordsinhashtag = False
# lets see if the rss feed has hashtag
if 'tags' in entry and options['addtags']:
hastags = True
else:
hastags = False
if hastags:
rss['hashtags'] = []
for i, _ in enumerate(entry['tags']):
if 'hashtaglist' in options:
prehashtags = entry['tags'][i]['term']
tmphashtags = entry['tags'][i]['term']
for element in severalwordshashtags:
if element in prehashtags:
severalwordsinhashtag = True
tmphashtags = prehashtags.replace(element,
''.join(element.split()))
# replace characters stopping a word from being a hashtag
if severalwordsinhashtag:
# remove ' from hashtag
tmphashtags = tmphashtags.replace("'", "")
# remove - from hashtag
tmphashtags = tmphashtags.replace("-", "")
# remove . from hashtag
tmphashtags = tmphashtags.replace(".", "")
# remove space from hashtag
finalhashtags = tmphashtags.replace(" ", "")
rss['hashtags'].append('#{}'.format(finalhashtags))
else:
nospace = ''.join(entry['tags'][i]['term'])
# remove space from hashtag
nospace = nospace.replace(" ", "")
rss['hashtags'].append('#{}'.format(nospace))
# populate rss with new entry to send
rss = populate_rss(entry)
rss = build_hashtags(entry, rss, options, severalwordshashtags)
# parse tweetfomat to elements
elements = re.findall(r"\{(.*?)\}",tweetformat)
# strip : from elements to allow string formating, eg. {title:.20}
@ -172,56 +115,14 @@ class Main:
fe = FilterEntry(elements, entry, options, feed['patterns'], feed['rssobject'], feed['feedname'])
entrytosend = fe.finalentry
if entrytosend:
tweetwithnotag = tweetformat.format(**entrytosend)
# remove duplicates from the final tweet
dedup = RemoveDuplicates(tweetwithnotag)
# only append hashtags if they exist
# remove last tags if tweet too long
if 'hashtags' in rss:
addtag = AddTags(dedup.finaltweet, rss['hashtags'])
finaltweet = addtag.finaltweet
else:
finaltweet = dedup.finaltweet
# strip html tags
finaltweet = BeautifulSoup(finaltweet, 'html.parser').get_text()
finaltweet = build_message(entrytosend, tweetformat, rss)
if clioptions.dryrun:
if entrytosend:
logging.warning('Would toot with visibility "{visibility}": {toot}'.format(
toot=finaltweet,
visibility=config.get(
'mastodon', 'toot_visibility',
fallback='public')))
else:
logging.debug('This rss entry did not meet pattern criteria. Should have not been sent')
send_message_dry_run(config, entrytosend, finaltweet)
else:
storeit = True
if entrytosend and not clioptions.populate:
logging.debug('Tooting with visibility "{visibility}": {toot}'.format(
toot=finaltweet,
visibility=config.get(
'mastodon', 'toot_visibility',
fallback='public')))
twp = TootPost(config, options, finaltweet)
storeit = twp.storeit()
else:
logging.debug('populating RSS entry {}'.format(rss['id']))
# in both cas we store the id of the sent tweet
if storeit:
cache.append(rss['id'])
send_message(config, clioptions, options, entrytosend, finaltweet, cache, rss)
# plugins
if plugins and entrytosend:
for plugin in plugins:
capitalizedplugin = plugin.title()
pluginclassname = '{plugin}Plugin'.format(plugin=capitalizedplugin)
pluginmodulename = 'feed2toot.plugins.{pluginmodule}'.format(pluginmodule=pluginclassname.lower())
try:
pluginmodule = importlib.import_module(pluginmodulename)
pluginclass = getattr(pluginmodule, pluginclassname)
pluginclass(plugins[plugin], finaltweet)
except ImportError as err:
print(err)
activate_plugins(plugins)
# do not forget to close cache (shelf object)
cache.close()
# release the lock file

72
feed2toot/message.py Normal file
View file

@ -0,0 +1,72 @@
# vim:ts=4:sw=4:ft=python:fileencoding=utf-8
# Copyright © 2015-2019 Carl Chenet <carl.chenet@ohmytux.com>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>
'''Build the message'''
# standard libraires imports
import logging
# external liraries imports
from bs4 import BeautifulSoup
# app libraries imports
from feed2toot.addtags import AddTags
from feed2toot.removeduplicates import RemoveDuplicates
from feed2toot.tootpost import TootPost
def build_message(entrytosend, tweetformat, rss):
'''populate the rss dict with the new entry'''
tweetwithnotag = tweetformat.format(**entrytosend)
# remove duplicates from the final tweet
dedup = RemoveDuplicates(tweetwithnotag)
# only append hashtags if they exist
# remove last tags if tweet too long
if 'hashtags' in rss:
addtag = AddTags(dedup.finaltweet, rss['hashtags'])
finaltweet = addtag.finaltweet
else:
finaltweet = dedup.finaltweet
# strip html tags
finaltweet = BeautifulSoup(finaltweet, 'html.parser').get_text()
return finaltweet
def send_message_dry_run(config, entrytosend, finaltweet):
'''simulate sending message using dry run mode'''
if entrytosend:
logging.warning('Would toot with visibility "{visibility}": {toot}'.format(
toot=finaltweet,
visibility=config.get(
'mastodon', 'toot_visibility',
fallback='public')))
else:
logging.debug('This rss entry did not meet pattern criteria. Should have not been sent')
def send_message(config, clioptions, options, entrytosend, finaltweet, cache, rss):
'''send message'''
storeit = True
if entrytosend and not clioptions.populate:
logging.debug('Tooting with visibility "{visibility}": {toot}'.format(
toot=finaltweet,
visibility=config.get(
'mastodon', 'toot_visibility',
fallback='public')))
twp = TootPost(config, options, finaltweet)
storeit = twp.storeit()
else:
logging.debug('populating RSS entry {}'.format(rss['id']))
# in both cas we store the id of the sent tweet
if storeit:
cache.append(rss['id'])

16
feed2toot/plugins/__init__.py
View file

@ -13,3 +13,19 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>
# standard libraires imports
import importlib
def activate_plugins(entrytosend):
'''activate plugins'''
for plugin in plugins:
capitalizedplugin = plugin.title()
pluginclassname = '{plugin}Plugin'.format(plugin=capitalizedplugin)
pluginmodulename = 'feed2toot.plugins.{pluginmodule}'.format(pluginmodule=pluginclassname.lower())
try:
pluginmodule = importlib.import_module(pluginmodulename)
pluginclass = getattr(pluginmodule, pluginclassname)
pluginclass(plugins[plugin], finaltweet)
except ImportError as err:
print(err)

42
feed2toot/rss.py Normal file
View file

@ -0,0 +1,42 @@
# vim:ts=4:sw=4:ft=python:fileencoding=utf-8
# Copyright © 2015-2019 Carl Chenet <carl.chenet@ohmytux.com>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>
'''Manage a lock file'''
# standard libraires imports
import datetime
import logging
import os
import os.path
import sys
def populate_rss(entry):
'''populate the rss dict with the new entry'''
if 'id' in entry:
logging.debug('found feed entry {entryid}'.format(entryid=entry['id']))
rss = {
'id': entry['id'],
}
elif 'guid' in entry:
logging.debug('found feed entry {entryid}'.format(entryid=entry['guid']))
rss = {
'id': entry['guid'],
}
else:
logging.debug('found feed entry {entryid}'.format(entryid=entry['link']))
rss = {
'id': entry['link'],
}
return rss

42
feed2toot/sortentries.py Normal file
View file

@ -0,0 +1,42 @@
# vim:ts=4:sw=4:ft=python:fileencoding=utf-8
# Copyright © 2015-2019 Carl Chenet <carl.chenet@ohmytux.com>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>
'''Manage a lock file'''
# standard libraires imports
import datetime
import logging
import os
import os.path
import sys
def sort_entries(is_all, cache, entries):
'''sort entries before sending'''
totweet = []
if not is_all:
for i in entries:
if 'id' in i:
if i['id'] not in cache.getdeque():
totweet.append(i)
elif 'guid' in i:
if i['guid'] not in cache.getdeque():
totweet.append(i)
else:
# if id or guid not in the entry, use link
if i['link'] not in cache.getdeque():
totweet.append(i)
else:
totweet = entries
return totweet