more refactoring and cleanup - linuxgaming - Unnamed repository; edit this file… | |
Log | |
Files | |
Refs | |
README | |
--- | |
commit 5917aef9ef70f99064dd0db3711c4dff95a73db4 | |
parent f1830964e2f2a24fab826cbdc8bac5ce3f3548a7 | |
Author: Jay Scott <[email protected]> | |
Date: Wed, 18 Jul 2018 19:00:19 +0100 | |
more refactoring and cleanup | |
Diffstat: | |
M TODO.md | 8 -------- | |
M linuxgaming/__init__.py | 20 ++++++++++---------- | |
M linuxgaming/database.py | 21 +++++++++++++++------ | |
M linuxgaming/details.py | 34 ++++++++---------------------… | |
M linuxgaming/search.py | 23 ++++++++--------------- | |
M linuxgaming/update.py | 169 ++++++++++++-----------------… | |
A linuxgaming/util.py | 13 +++++++++++++ | |
7 files changed, 118 insertions(+), 170 deletions(-) | |
--- | |
diff --git a/TODO.md b/TODO.md | |
@@ -1,14 +1,6 @@ | |
# TODO | |
-## pre 1.0 | |
- | |
- | |
- - Refactor code | |
- | |
- | |
-## other | |
- | |
- Add menu with source info / about etc | |
- Move updates to AWS Lambda function | |
- Add API | |
diff --git a/linuxgaming/__init__.py b/linuxgaming/__init__.py | |
@@ -35,30 +35,30 @@ def create_app(): | |
@app.route("/") | |
def home(): | |
- result = database.db_search( | |
- {"date": {'$gte': datetime.now() - timedelta(hours=24)}}) | |
+ result = database.find_all({ | |
+ "date": { | |
+ '$gte': datetime.now() - timedelta(hours=24) | |
+ } | |
+ }) | |
return render_template('pages/home.html', entries=result) | |
@app.errorhandler(500) | |
def internal_error(error): | |
app.logger.error('internal error %s', error) | |
return render_template( | |
- "message.html", | |
- icon="frown", | |
- msg="Something went wrong!"), 500 | |
+ "message.html", icon="frown", msg="Something went wrong!"), 500 | |
@app.errorhandler(404) | |
- def page_not_found(e): | |
+ def page_not_found(): | |
app.logger.info('page not found') | |
return render_template( | |
- "message.html", | |
- icon="frown", | |
- msg="I think you are lost!"), 404 | |
+ "message.html", icon="frown", msg="I think you are lost!"), 404 | |
@app.template_filter('strftime') | |
- def _jinja2_filter_datetime(date, fmt=None): | |
+ def _jinja2_filter_datetime(date): | |
date = dateutil.parser.parse(str(date)) | |
native = date.replace(tzinfo=None) | |
format = '%a %d %b %X %Y' | |
return native.strftime(format) | |
+ | |
return app | |
diff --git a/linuxgaming/database.py b/linuxgaming/database.py | |
@@ -1,14 +1,23 @@ | |
from flask import current_app | |
-def db_search(query={}): | |
+def find_all(query={}): | |
try: | |
- d = current_app.mongo.db.items.find( | |
- query | |
- ).sort('date', -1) | |
- except pymongo.errors.OperationFailure: | |
- print("DB Error") | |
+ d = current_app.mongo.db.items.find(query).sort('date', -1) | |
+ except Exception as e: | |
+ current_app.logger.error('DB replace error %s', e) | |
return False | |
return d | |
+ | |
+ | |
+def replace_one(query, data, upsert=True): | |
+ | |
+ try: | |
+ current_app.mongo.db.items.replace_one(query, data, upsert) | |
+ except Exception as e: | |
+ current_app.logger.error('DB replace error %s', e) | |
+ return False | |
+ | |
+ return True | |
diff --git a/linuxgaming/details.py b/linuxgaming/details.py | |
@@ -1,30 +1,15 @@ | |
-from flask import ( | |
- Blueprint, | |
- flash, | |
- redirect, | |
- render_template, | |
- url_for, | |
- current_app) | |
-import yaml | |
+from flask import (Blueprint, flash, redirect, render_template, url_for, | |
+ current_app) | |
+from . import database | |
+from . import util | |
bp = Blueprint('details', __name__, url_prefix='/details') | |
-def load(): | |
- """Return the YAML parsed config file.""" | |
- try: | |
- with open('config/feed_config.yaml', 'r') as ymlfile: | |
- cfg = yaml.load(ymlfile) | |
- except yaml.YAMLError as exc: | |
- current_app.logger.error('YAML read error %s', exc) | |
- | |
- return cfg | |
- | |
- | |
[email protected]("/<path:path>", methods=('GET', 'POST')) | |
[email protected]("/<path:path>", methods=["GET"]) | |
def details(path): | |
"""Source details page""" | |
- feed_config = load() | |
+ feed_config = util.load_yaml() | |
if path in feed_config: | |
source_data = feed_config[path] | |
@@ -33,10 +18,7 @@ def details(path): | |
current_app.logger.info('Manual details probe %s', path) | |
return redirect(url_for('home')) | |
- source_items = current_app.mongo.db.items.find( | |
- {"name": path}).sort('date', -1) | |
+ source_items = database.find_all({"name": path}) | |
return render_template( | |
- 'pages/details.html', | |
- data=source_data, | |
- items=source_items) | |
+ 'pages/details.html', data=source_data, items=source_items) | |
diff --git a/linuxgaming/search.py b/linuxgaming/search.py | |
@@ -1,28 +1,21 @@ | |
-from flask import ( | |
- Blueprint, | |
- flash, | |
- redirect, | |
- url_for, | |
- render_template, | |
- current_app) | |
+from flask import (Blueprint, flash, redirect, url_for, render_template, | |
+ current_app) | |
+from . import database | |
bp = Blueprint('search', __name__, url_prefix='/search') | |
[email protected]("/<path:path>", methods=('GET', 'POST')) | |
-def test(path): | |
[email protected]("/<path:path>", methods=["GET"]) | |
+def search(path): | |
pages = ['game', 'twitch', 'youtube', 'article', 'podcast'] | |
if any(x in path for x in pages): | |
- result = current_app.mongo.db.items.find( | |
- {"type": path}).sort('date', -1) | |
+ result = database.find_all({"type": path}) | |
return render_template( | |
- 'pages/search.html', | |
- entries=result, | |
- count=result.count()) | |
+ 'pages/search.html', entries=result, count=result.count()) | |
else: | |
flash('1337 Hacks in progress...') | |
current_app.logger.info('Manual search probe %s', path) | |
- | |
+ | |
return redirect(url_for('home')) | |
diff --git a/linuxgaming/update.py b/linuxgaming/update.py | |
@@ -1,29 +1,24 @@ | |
from googleapiclient.discovery import build | |
from twitch import TwitchClient | |
from flask import Blueprint, render_template, current_app | |
-import yaml | |
+from . import database | |
+import json | |
+import requests | |
import feedparser | |
import dateutil.parser | |
+from . import util | |
bp = Blueprint('update', __name__, url_prefix='/update') | |
-def load(): | |
- | |
- with open('config/feed_config.yaml', 'r') as ymlfile: | |
- cfg = yaml.load(ymlfile) | |
- | |
- return cfg | |
- | |
- | |
def parse(url): | |
return feedparser.parse(url).entries | |
[email protected]('/rss', methods=('GET', 'POST')) | |
[email protected]('/rss', methods=["GET"]) | |
def rss_update(): | |
- feed_config = load() | |
+ feed_config = util.load_yaml() | |
for section in feed_config: | |
if 'rss' not in feed_config[section]: | |
@@ -40,52 +35,45 @@ def rss_update(): | |
description = "" | |
else: | |
description = feed.description | |
- data = {"name": section, | |
- "icon": feed_config[section]['icon'], | |
- "title": trimtitle, | |
- "description": description, | |
- "url": feed.link, | |
- "type": feed_config[section]['rss']['type'], | |
- "date": dateutil.parser.parse(feed.updated)} | |
- | |
- try: | |
- current_app.mongo.db.items.replace_one( | |
- {'title': trimtitle}, data, True) | |
- except Exception as e: | |
- current_app.logger.error('DB replace error %s', e) | |
- return render_template( | |
- "message.html", icon="frown", msg=str(e)) | |
- return render_template( | |
- "message.html", | |
- icon="smile", | |
- msg="RSS feeds updated!") | |
+ data = { | |
+ "name": section, | |
+ "icon": feed_config[section]['icon'], | |
+ "title": trimtitle, | |
+ "description": description, | |
+ "url": feed.link, | |
+ "type": feed_config[section]['rss']['type'], | |
+ "date": dateutil.parser.parse(feed.updated) | |
+ } | |
+ | |
+ database.replace_one({'title': trimtitle}, data) | |
+ | |
+ return render_template("message.html", msg="RSS feeds updated!") | |
[email protected]('/twitch', methods=('GET', 'POST')) | |
[email protected]('/twitch', methods=["GET"]) | |
def twitch_update(): | |
- feed_config = load() | |
+ feed_config = util.load_yaml() | |
for section in feed_config: | |
if 'twitch' not in feed_config[section]: | |
continue | |
current_app.logger.info('[TWITCH] Updating %s', section) | |
- twitch_channelID = feed_config[section]['twitch']['twitch_id'] | |
+ twitch_channelid = feed_config[section]['twitch']['twitch_id'] | |
client = TwitchClient( | |
client_id=current_app.config['TWITCH_CLIENTID'], | |
- oauth_token=current_app.config['TWITCH_TOKEN'] | |
- ) | |
+ oauth_token=current_app.config['TWITCH_TOKEN']) | |
videos = client.channels.get_videos( | |
- twitch_channelID, # Channel ID | |
+ twitch_channelid, # Channel ID | |
10, # Limit | |
0, # Offset | |
'archive', # Broadcast type | |
- 'en', # Lang | |
- 'time', # Sort | |
+ 'en', # Lang | |
+ 'time', # Sort | |
) | |
for search_results in videos: | |
@@ -100,81 +88,64 @@ def twitch_update(): | |
"date": dateutil.parser.parse(search_results['recorded_at']) | |
} | |
- try: | |
- current_app.mongo.db.items.replace_one( | |
- {'url': search_results['url']}, data, True) | |
- except Exception as e: | |
- current_app.logger.error('DB replace error %s', e) | |
- return render_template( | |
- "message.html", icon="frown", msg=str(e)) | |
+ database.replace_one({'url': search_results['url']}, data) | |
- return render_template( | |
- "message.html", | |
- icon="smile", | |
- msg="Twitch API updated!") | |
+ return render_template("message.html", msg="Twitch API updated!") | |
[email protected]('/youtube', methods=('GET', 'POST')) | |
[email protected]('/youtube', methods=["GET"]) | |
def youtube_update(): | |
- DEVELOPER_KEY = current_app.config['YOUTUBE_APIKEY'] | |
- YOUTUBE_API_SERVICE_NAME = 'youtube' | |
- YOUTUBE_API_VERSION = 'v3' | |
+ key = current_app.config['YOUTUBE_APIKEY'] | |
+ youtube_api = 'youtube' | |
+ api_version = 'v3' | |
- feed_config = load() | |
+ feed_config = util.load_yaml() | |
for section in feed_config: | |
if 'youtube' not in feed_config[section]: | |
continue | |
- youtube_channelID = feed_config[section]['youtube']['channel_id'] | |
+ youtube_channel_id = feed_config[section]['youtube']['channel_id'] | |
- youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION, | |
- developerKey=DEVELOPER_KEY) | |
+ youtube = build(youtube_api, api_version, developerKey=key) | |
current_app.logger.info('[YOUTUBE] Updating %s', section) | |
search_response = youtube.search().list( | |
q="", | |
- channelId=youtube_channelID, | |
+ channelId=youtube_channel_id, | |
part='id,snippet', | |
order='date', | |
- maxResults=5 | |
- ).execute() | |
+ maxResults=5).execute() | |
for search_result in search_response.get('items', []): | |
trimtitle = search_result['snippet']['title'][0:150] | |
if search_result['id']['kind'] == 'youtube#video': | |
data = { | |
- "name": section, | |
- "icon": feed_config[section]['icon'], | |
- "title": trimtitle, | |
- "description": search_result['snippet']['description'], | |
- "type": "youtube", | |
- "url": "https://www.youtube.com/watch?v=" + | |
+ "name": | |
+ section, | |
+ "icon": | |
+ feed_config[section]['icon'], | |
+ "title": | |
+ trimtitle, | |
+ "description": | |
+ search_result['snippet']['description'], | |
+ "type": | |
+ "youtube", | |
+ "url": | |
+ "https://www.youtube.com/watch?v=" + | |
search_result['id']['videoId'], | |
- "date": dateutil.parser.parse( | |
- search_result['snippet']['publishedAt'])} | |
+ "date": | |
+ dateutil.parser.parse( | |
+ search_result['snippet']['publishedAt']) | |
+ } | |
- try: | |
- current_app.mongo.db.items.replace_one( | |
- {'title': trimtitle}, data, True) | |
- except Exception as e: | |
- current_app.logger.error('DB replace error %s', e) | |
- return render_template( | |
- "message.html", icon="frown", msg=str(e)) | |
+ database.replace_one({'title': trimtitle}, data) | |
- return render_template( | |
- "message.html", | |
- icon="smile", | |
- msg="Youtube API updated!") | |
- | |
-# | |
-# This GoG import is just so nasty and needs fixed, Will | |
-# be enabling it until it is. | |
-# | |
+ return render_template("message.html", msg="Youtube API updated!") | |
[email protected]('/gog', methods=('GET', 'POST')) | |
[email protected]('/gog', methods=["GET"]) | |
def gog_update(): | |
from datetime import datetime | |
@@ -184,10 +155,7 @@ def gog_update(): | |
query = "mediaType=game&system=Linux&limit=50&page=" + str(count) | |
game_data = get_gog_info(query) | |
if game_data is None: | |
- return render_template( | |
- "message.html", | |
- icon="frown", | |
- msg="GoG query error") | |
+ return render_template("message.html", msg="GoG query error") | |
for search_result in game_data['products']: | |
@@ -219,33 +187,24 @@ def gog_update(): | |
"publisher": search_result['publisher'], | |
"category": search_result['category'], | |
"url": "https://www.gog.com" + search_result['url'], | |
- "date": dateutil.parser.parse(release_date)} | |
+ "date": dateutil.parser.parse(release_date) | |
+ } | |
- try: | |
- current_app.mongo.db.items.replace_one( | |
- {'title': search_result['title']}, data, True) | |
- except Exception as e: | |
- return render_template( | |
- "message.html", icon="frown", msg=str(e)) | |
+ database.replace_one({'title': search_result['title']}, data) | |
count = count + 1 | |
return render_template( | |
- "message.html", | |
- icon="smile", | |
- msg="GoG games updated!") | |
+ "message.html", icon="smile", msg="GoG games updated!") | |
def get_gog_info(query): | |
- import json | |
- import requests | |
- | |
- GOG_API_URL = "https://embed.gog.com/games/ajax/filtered?" | |
+ gog_api_url = "https://embed.gog.com/games/ajax/filtered?" | |
- response = requests.get(GOG_API_URL + query) | |
+ response = requests.get(gog_api_url + query) | |
if response.status_code == 200: | |
- return (json.loads(response.content.decode('utf-8'))) | |
+ return json.loads(response.content.decode('utf-8')) | |
else: | |
return None | |
diff --git a/linuxgaming/util.py b/linuxgaming/util.py | |
@@ -0,0 +1,13 @@ | |
+import yaml | |
+from flask import (current_app) | |
+ | |
+ | |
+def load_yaml(): | |
+ """Return the YAML parsed config file.""" | |
+ try: | |
+ with open('config/feed_config.yaml', 'r') as ymlfile: | |
+ cfg = yaml.load(ymlfile) | |
+ except yaml.YAMLError as exc: | |
+ current_app.logger.error('YAML read error %s', exc) | |
+ | |
+ return cfg |