Introduction
Introduction Statistics Contact Development Disclaimer Help
more refactoring and cleanup - linuxgaming - Linux gaming aggregate tool, built…
Log
Files
Refs
README
---
commit 5917aef9ef70f99064dd0db3711c4dff95a73db4
parent f1830964e2f2a24fab826cbdc8bac5ce3f3548a7
Author: Jay Scott <[email protected]>
Date: Wed, 18 Jul 2018 19:00:19 +0100
more refactoring and cleanup
Diffstat:
M TODO.md | 8 --------
M linuxgaming/__init__.py | 20 ++++++++++----------
M linuxgaming/database.py | 21 +++++++++++++++------
M linuxgaming/details.py | 34 ++++++++---------------------…
M linuxgaming/search.py | 23 ++++++++---------------
M linuxgaming/update.py | 169 ++++++++++++-----------------…
A linuxgaming/util.py | 13 +++++++++++++
7 files changed, 118 insertions(+), 170 deletions(-)
---
diff --git a/TODO.md b/TODO.md
@@ -1,14 +1,6 @@
# TODO
-## pre 1.0
-
-
- - Refactor code
-
-
-## other
-
- Add menu with source info / about etc
- Move updates to AWS Lambda function
- Add API
diff --git a/linuxgaming/__init__.py b/linuxgaming/__init__.py
@@ -35,30 +35,30 @@ def create_app():
@app.route("/")
def home():
- result = database.db_search(
- {"date": {'$gte': datetime.now() - timedelta(hours=24)}})
+ result = database.find_all({
+ "date": {
+ '$gte': datetime.now() - timedelta(hours=24)
+ }
+ })
return render_template('pages/home.html', entries=result)
@app.errorhandler(500)
def internal_error(error):
app.logger.error('internal error %s', error)
return render_template(
- "message.html",
- icon="frown",
- msg="Something went wrong!"), 500
+ "message.html", icon="frown", msg="Something went wrong!"), 500
@app.errorhandler(404)
- def page_not_found(e):
+ def page_not_found():
app.logger.info('page not found')
return render_template(
- "message.html",
- icon="frown",
- msg="I think you are lost!"), 404
+ "message.html", icon="frown", msg="I think you are lost!"), 404
@app.template_filter('strftime')
- def _jinja2_filter_datetime(date, fmt=None):
+ def _jinja2_filter_datetime(date):
date = dateutil.parser.parse(str(date))
native = date.replace(tzinfo=None)
format = '%a %d %b %X %Y'
return native.strftime(format)
+
return app
diff --git a/linuxgaming/database.py b/linuxgaming/database.py
@@ -1,14 +1,23 @@
from flask import current_app
-def db_search(query={}):
+def find_all(query={}):
try:
- d = current_app.mongo.db.items.find(
- query
- ).sort('date', -1)
- except pymongo.errors.OperationFailure:
- print("DB Error")
+ d = current_app.mongo.db.items.find(query).sort('date', -1)
+ except Exception as e:
+ current_app.logger.error('DB replace error %s', e)
return False
return d
+
+
+def replace_one(query, data, upsert=True):
+
+ try:
+ current_app.mongo.db.items.replace_one(query, data, upsert)
+ except Exception as e:
+ current_app.logger.error('DB replace error %s', e)
+ return False
+
+ return True
diff --git a/linuxgaming/details.py b/linuxgaming/details.py
@@ -1,30 +1,15 @@
-from flask import (
- Blueprint,
- flash,
- redirect,
- render_template,
- url_for,
- current_app)
-import yaml
+from flask import (Blueprint, flash, redirect, render_template, url_for,
+ current_app)
+from . import database
+from . import util
bp = Blueprint('details', __name__, url_prefix='/details')
-def load():
- """Return the YAML parsed config file."""
- try:
- with open('config/feed_config.yaml', 'r') as ymlfile:
- cfg = yaml.load(ymlfile)
- except yaml.YAMLError as exc:
- current_app.logger.error('YAML read error %s', exc)
-
- return cfg
-
-
[email protected]("/<path:path>", methods=('GET', 'POST'))
[email protected]("/<path:path>", methods=["GET"])
def details(path):
"""Source details page"""
- feed_config = load()
+ feed_config = util.load_yaml()
if path in feed_config:
source_data = feed_config[path]
@@ -33,10 +18,7 @@ def details(path):
current_app.logger.info('Manual details probe %s', path)
return redirect(url_for('home'))
- source_items = current_app.mongo.db.items.find(
- {"name": path}).sort('date', -1)
+ source_items = database.find_all({"name": path})
return render_template(
- 'pages/details.html',
- data=source_data,
- items=source_items)
+ 'pages/details.html', data=source_data, items=source_items)
diff --git a/linuxgaming/search.py b/linuxgaming/search.py
@@ -1,28 +1,21 @@
-from flask import (
- Blueprint,
- flash,
- redirect,
- url_for,
- render_template,
- current_app)
+from flask import (Blueprint, flash, redirect, url_for, render_template,
+ current_app)
+from . import database
bp = Blueprint('search', __name__, url_prefix='/search')
[email protected]("/<path:path>", methods=('GET', 'POST'))
-def test(path):
[email protected]("/<path:path>", methods=["GET"])
+def search(path):
pages = ['game', 'twitch', 'youtube', 'article', 'podcast']
if any(x in path for x in pages):
- result = current_app.mongo.db.items.find(
- {"type": path}).sort('date', -1)
+ result = database.find_all({"type": path})
return render_template(
- 'pages/search.html',
- entries=result,
- count=result.count())
+ 'pages/search.html', entries=result, count=result.count())
else:
flash('1337 Hacks in progress...')
current_app.logger.info('Manual search probe %s', path)
-
+
return redirect(url_for('home'))
diff --git a/linuxgaming/update.py b/linuxgaming/update.py
@@ -1,29 +1,24 @@
from googleapiclient.discovery import build
from twitch import TwitchClient
from flask import Blueprint, render_template, current_app
-import yaml
+from . import database
+import json
+import requests
import feedparser
import dateutil.parser
+from . import util
bp = Blueprint('update', __name__, url_prefix='/update')
-def load():
-
- with open('config/feed_config.yaml', 'r') as ymlfile:
- cfg = yaml.load(ymlfile)
-
- return cfg
-
-
def parse(url):
return feedparser.parse(url).entries
[email protected]('/rss', methods=('GET', 'POST'))
[email protected]('/rss', methods=["GET"])
def rss_update():
- feed_config = load()
+ feed_config = util.load_yaml()
for section in feed_config:
if 'rss' not in feed_config[section]:
@@ -40,52 +35,45 @@ def rss_update():
description = ""
else:
description = feed.description
- data = {"name": section,
- "icon": feed_config[section]['icon'],
- "title": trimtitle,
- "description": description,
- "url": feed.link,
- "type": feed_config[section]['rss']['type'],
- "date": dateutil.parser.parse(feed.updated)}
-
- try:
- current_app.mongo.db.items.replace_one(
- {'title': trimtitle}, data, True)
- except Exception as e:
- current_app.logger.error('DB replace error %s', e)
- return render_template(
- "message.html", icon="frown", msg=str(e))
- return render_template(
- "message.html",
- icon="smile",
- msg="RSS feeds updated!")
+ data = {
+ "name": section,
+ "icon": feed_config[section]['icon'],
+ "title": trimtitle,
+ "description": description,
+ "url": feed.link,
+ "type": feed_config[section]['rss']['type'],
+ "date": dateutil.parser.parse(feed.updated)
+ }
+
+ database.replace_one({'title': trimtitle}, data)
+
+ return render_template("message.html", msg="RSS feeds updated!")
[email protected]('/twitch', methods=('GET', 'POST'))
[email protected]('/twitch', methods=["GET"])
def twitch_update():
- feed_config = load()
+ feed_config = util.load_yaml()
for section in feed_config:
if 'twitch' not in feed_config[section]:
continue
current_app.logger.info('[TWITCH] Updating %s', section)
- twitch_channelID = feed_config[section]['twitch']['twitch_id']
+ twitch_channelid = feed_config[section]['twitch']['twitch_id']
client = TwitchClient(
client_id=current_app.config['TWITCH_CLIENTID'],
- oauth_token=current_app.config['TWITCH_TOKEN']
- )
+ oauth_token=current_app.config['TWITCH_TOKEN'])
videos = client.channels.get_videos(
- twitch_channelID, # Channel ID
+ twitch_channelid, # Channel ID
10, # Limit
0, # Offset
'archive', # Broadcast type
- 'en', # Lang
- 'time', # Sort
+ 'en', # Lang
+ 'time', # Sort
)
for search_results in videos:
@@ -100,81 +88,64 @@ def twitch_update():
"date": dateutil.parser.parse(search_results['recorded_at'])
}
- try:
- current_app.mongo.db.items.replace_one(
- {'url': search_results['url']}, data, True)
- except Exception as e:
- current_app.logger.error('DB replace error %s', e)
- return render_template(
- "message.html", icon="frown", msg=str(e))
+ database.replace_one({'url': search_results['url']}, data)
- return render_template(
- "message.html",
- icon="smile",
- msg="Twitch API updated!")
+ return render_template("message.html", msg="Twitch API updated!")
[email protected]('/youtube', methods=('GET', 'POST'))
[email protected]('/youtube', methods=["GET"])
def youtube_update():
- DEVELOPER_KEY = current_app.config['YOUTUBE_APIKEY']
- YOUTUBE_API_SERVICE_NAME = 'youtube'
- YOUTUBE_API_VERSION = 'v3'
+ key = current_app.config['YOUTUBE_APIKEY']
+ youtube_api = 'youtube'
+ api_version = 'v3'
- feed_config = load()
+ feed_config = util.load_yaml()
for section in feed_config:
if 'youtube' not in feed_config[section]:
continue
- youtube_channelID = feed_config[section]['youtube']['channel_id']
+ youtube_channel_id = feed_config[section]['youtube']['channel_id']
- youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION,
- developerKey=DEVELOPER_KEY)
+ youtube = build(youtube_api, api_version, developerKey=key)
current_app.logger.info('[YOUTUBE] Updating %s', section)
search_response = youtube.search().list(
q="",
- channelId=youtube_channelID,
+ channelId=youtube_channel_id,
part='id,snippet',
order='date',
- maxResults=5
- ).execute()
+ maxResults=5).execute()
for search_result in search_response.get('items', []):
trimtitle = search_result['snippet']['title'][0:150]
if search_result['id']['kind'] == 'youtube#video':
data = {
- "name": section,
- "icon": feed_config[section]['icon'],
- "title": trimtitle,
- "description": search_result['snippet']['description'],
- "type": "youtube",
- "url": "https://www.youtube.com/watch?v=" +
+ "name":
+ section,
+ "icon":
+ feed_config[section]['icon'],
+ "title":
+ trimtitle,
+ "description":
+ search_result['snippet']['description'],
+ "type":
+ "youtube",
+ "url":
+ "https://www.youtube.com/watch?v=" +
search_result['id']['videoId'],
- "date": dateutil.parser.parse(
- search_result['snippet']['publishedAt'])}
+ "date":
+ dateutil.parser.parse(
+ search_result['snippet']['publishedAt'])
+ }
- try:
- current_app.mongo.db.items.replace_one(
- {'title': trimtitle}, data, True)
- except Exception as e:
- current_app.logger.error('DB replace error %s', e)
- return render_template(
- "message.html", icon="frown", msg=str(e))
+ database.replace_one({'title': trimtitle}, data)
- return render_template(
- "message.html",
- icon="smile",
- msg="Youtube API updated!")
-
-#
-# This GoG import is just so nasty and needs fixed, Will
-# be enabling it until it is.
-#
+ return render_template("message.html", msg="Youtube API updated!")
[email protected]('/gog', methods=('GET', 'POST'))
[email protected]('/gog', methods=["GET"])
def gog_update():
from datetime import datetime
@@ -184,10 +155,7 @@ def gog_update():
query = "mediaType=game&system=Linux&limit=50&page=" + str(count)
game_data = get_gog_info(query)
if game_data is None:
- return render_template(
- "message.html",
- icon="frown",
- msg="GoG query error")
+ return render_template("message.html", msg="GoG query error")
for search_result in game_data['products']:
@@ -219,33 +187,24 @@ def gog_update():
"publisher": search_result['publisher'],
"category": search_result['category'],
"url": "https://www.gog.com" + search_result['url'],
- "date": dateutil.parser.parse(release_date)}
+ "date": dateutil.parser.parse(release_date)
+ }
- try:
- current_app.mongo.db.items.replace_one(
- {'title': search_result['title']}, data, True)
- except Exception as e:
- return render_template(
- "message.html", icon="frown", msg=str(e))
+ database.replace_one({'title': search_result['title']}, data)
count = count + 1
return render_template(
- "message.html",
- icon="smile",
- msg="GoG games updated!")
+ "message.html", icon="smile", msg="GoG games updated!")
def get_gog_info(query):
- import json
- import requests
-
- GOG_API_URL = "https://embed.gog.com/games/ajax/filtered?"
+ gog_api_url = "https://embed.gog.com/games/ajax/filtered?"
- response = requests.get(GOG_API_URL + query)
+ response = requests.get(gog_api_url + query)
if response.status_code == 200:
- return (json.loads(response.content.decode('utf-8')))
+ return json.loads(response.content.decode('utf-8'))
else:
return None
diff --git a/linuxgaming/util.py b/linuxgaming/util.py
@@ -0,0 +1,13 @@
+import yaml
+from flask import (current_app)
+
+
+def load_yaml():
+ """Return the YAML parsed config file."""
+ try:
+ with open('config/feed_config.yaml', 'r') as ymlfile:
+ cfg = yaml.load(ymlfile)
+ except yaml.YAMLError as exc:
+ current_app.logger.error('YAML read error %s', exc)
+
+ return cfg
You are viewing proxied material from jay.scot. The copyright of proxied material belongs to its original authors. Any comments or complaints in relation to proxied material should be directed to the original authors of the content concerned. Please see the disclaimer for more details.