Introduction
Introduction Statistics Contact Development Disclaimer Help
more refactoring - linuxgaming - Linux gaming aggregate tool, built to test out…
Log
Files
Refs
README
---
commit b11ff41ea03339b4f740e2bc49fe3a718623321d
parent 5917aef9ef70f99064dd0db3711c4dff95a73db4
Author: Jay Scott <[email protected]>
Date: Wed, 18 Jul 2018 20:18:33 +0100
more refactoring
Diffstat:
M config/feed_config.yaml | 11 +++++------
M linuxgaming/__init__.py | 6 +++---
M linuxgaming/database.py | 6 +++---
M linuxgaming/update.py | 103 ++++++++++++++---------------…
M linuxgaming/util.py | 36 ++++++++++++++++++++++++++++-…
5 files changed, 91 insertions(+), 71 deletions(-)
---
diff --git a/config/feed_config.yaml b/config/feed_config.yaml
@@ -18,14 +18,14 @@ NuSuey:
twitch:
website: "https://www.twitch.tv/nusuey/"
twitch_id: "7863182"
-
+
HexDSL:
icon: "hexdsl.png"
about: "Linux Gaming Rambles, shows and streams presented by HexDSL. Apparen…
youtube:
website: "https://www.youtube.com/user/hexdsl/"
channel_id: "UCRE3NFNtdjR96-H4QG4U1Fg"
- twitch:
+ twitch:
website: "https://www.twitch.tv/hexdsl/"
twitch_id: "40594186"
@@ -35,7 +35,7 @@ TheLinuxGamer:
youtube:
website: "https://www.youtube.com/user/tuxreviews/"
channel_id: "UCv1Kcz-CuGM6mxzL3B1_Eiw"
-
+
GamingOnLinux:
icon: "gol.png"
about: "A little bit of information on who GamingOnLinux are, we are just a …
@@ -46,7 +46,7 @@ GamingOnLinux:
twitch:
website: "https://www.twitch.tv/gamingonlinux/"
twitch_id: "50905707"
-
+
BLGP:
icon: "blgp.png"
about: "The Best Linux Games Podcast features only the best news, reviews, a…
@@ -80,4 +80,4 @@ XPenguin:
rss:
website: "http://xpenguin.club/"
type: "podcast"
- url: "http://xpenguin.club/rss.xml"
-\ No newline at end of file
+ url: "http://xpenguin.club/rss.xml"
diff --git a/linuxgaming/__init__.py b/linuxgaming/__init__.py
@@ -46,13 +46,13 @@ def create_app():
def internal_error(error):
app.logger.error('internal error %s', error)
return render_template(
- "message.html", icon="frown", msg="Something went wrong!"), 500
+ "message.html", msg="Something went wrong!"), 500
@app.errorhandler(404)
- def page_not_found():
+ def page_not_found(page):
app.logger.info('page not found')
return render_template(
- "message.html", icon="frown", msg="I think you are lost!"), 404
+ "message.html", msg="I think you are lost!"), 404
@app.template_filter('strftime')
def _jinja2_filter_datetime(date):
diff --git a/linuxgaming/database.py b/linuxgaming/database.py
@@ -1,4 +1,4 @@
-from flask import current_app
+from flask import current_app, abort
def find_all(query={}):
@@ -7,7 +7,7 @@ def find_all(query={}):
d = current_app.mongo.db.items.find(query).sort('date', -1)
except Exception as e:
current_app.logger.error('DB replace error %s', e)
- return False
+ abort(500)
return d
@@ -18,6 +18,6 @@ def replace_one(query, data, upsert=True):
current_app.mongo.db.items.replace_one(query, data, upsert)
except Exception as e:
current_app.logger.error('DB replace error %s', e)
- return False
+ abort(500)
return True
diff --git a/linuxgaming/update.py b/linuxgaming/update.py
@@ -1,52 +1,60 @@
from googleapiclient.discovery import build
from twitch import TwitchClient
from flask import Blueprint, render_template, current_app
-from . import database
-import json
-import requests
-import feedparser
import dateutil.parser
+from . import database
from . import util
bp = Blueprint('update', __name__, url_prefix='/update')
-def parse(url):
- return feedparser.parse(url).entries
-
-
@bp.route('/rss', methods=["GET"])
def rss_update():
+ # load sources config
feed_config = util.load_yaml()
for section in feed_config:
+ # if it does not have an rss section skip
if 'rss' not in feed_config[section]:
continue
current_app.logger.info('[RSS] Updating %s', section)
- feeds = parse(feed_config[section]['rss']['url'])
+
+ # parse the source url rss feed
+ feeds = util.feed_parse(feed_config[section]['rss']['url'])
+
+ # check for errors
+ if feeds is None:
+ continue
for feed in feeds:
- trimtitle = feed.title[0:150]
+ # if not title, then just skip
+ if hasattr(feed, 'title'):
+ trimmed_title = feed.title[0:150]
+ else:
+ continue
+ # some feeds dont have a description, RSS 2.0
if not hasattr(feed, 'description'):
description = ""
else:
description = feed.description
+ # construct db item
data = {
"name": section,
"icon": feed_config[section]['icon'],
- "title": trimtitle,
+ "title": trimmed_title,
"description": description,
"url": feed.link,
"type": feed_config[section]['rss']['type'],
"date": dateutil.parser.parse(feed.updated)
}
- database.replace_one({'title': trimtitle}, data)
+ # insert based on title
+ database.replace_one({'title': trimmed_title}, data)
return render_template("message.html", msg="RSS feeds updated!")
@@ -61,6 +69,7 @@ def twitch_update():
continue
current_app.logger.info('[TWITCH] Updating %s', section)
+
twitch_channelid = feed_config[section]['twitch']['twitch_id']
client = TwitchClient(
@@ -77,11 +86,11 @@ def twitch_update():
)
for search_results in videos:
- trimtitle = search_results['title'][0:150]
+ trimmed_title = search_results['title'][0:150]
data = {
"name": section,
"icon": feed_config[section]['icon'],
- "title": trimtitle,
+ "title": trimmed_title,
"description": search_results['description'],
"url": search_results['url'],
"type": "twitch",
@@ -96,10 +105,6 @@ def twitch_update():
@bp.route('/youtube', methods=["GET"])
def youtube_update():
- key = current_app.config['YOUTUBE_APIKEY']
- youtube_api = 'youtube'
- api_version = 'v3'
-
feed_config = util.load_yaml()
for section in feed_config:
@@ -108,9 +113,11 @@ def youtube_update():
youtube_channel_id = feed_config[section]['youtube']['channel_id']
- youtube = build(youtube_api, api_version, developerKey=key)
+ youtube = build(
+ 'youtube', 'v3', developerKey=current_app.config['YOUTUBE_APIKEY'])
current_app.logger.info('[YOUTUBE] Updating %s', section)
+
search_response = youtube.search().list(
q="",
channelId=youtube_channel_id,
@@ -119,28 +126,26 @@ def youtube_update():
maxResults=5).execute()
for search_result in search_response.get('items', []):
- trimtitle = search_result['snippet']['title'][0:150]
- if search_result['id']['kind'] == 'youtube#video':
- data = {
- "name":
- section,
- "icon":
- feed_config[section]['icon'],
- "title":
- trimtitle,
- "description":
- search_result['snippet']['description'],
- "type":
- "youtube",
- "url":
- "https://www.youtube.com/watch?v=" +
- search_result['id']['videoId'],
- "date":
- dateutil.parser.parse(
- search_result['snippet']['publishedAt'])
- }
-
- database.replace_one({'title': trimtitle}, data)
+ trimmed_title = search_result['snippet']['title'][0:150]
+ data = {
+ "name":
+ section,
+ "icon":
+ feed_config[section]['icon'],
+ "title":
+ trimmed_title,
+ "description":
+ search_result['snippet']['description'],
+ "type":
+ "youtube",
+ "url":
+ "https://www.youtube.com/watch?v=" +
+ search_result['id']['videoId'],
+ "date":
+ dateutil.parser.parse(search_result['snippet']['publishedAt'])
+ }
+
+ database.replace_one({'title': trimmed_title}, data)
return render_template("message.html", msg="Youtube API updated!")
@@ -153,9 +158,7 @@ def gog_update():
count = 1
while count < 51:
query = "mediaType=game&system=Linux&limit=50&page=" + str(count)
- game_data = get_gog_info(query)
- if game_data is None:
- return render_template("message.html", msg="GoG query error")
+ game_data = util.get_gog_info(query)
for search_result in game_data['products']:
@@ -196,15 +199,3 @@ def gog_update():
return render_template(
"message.html", icon="smile", msg="GoG games updated!")
-
-
-def get_gog_info(query):
-
- gog_api_url = "https://embed.gog.com/games/ajax/filtered?"
-
- response = requests.get(gog_api_url + query)
-
- if response.status_code == 200:
- return json.loads(response.content.decode('utf-8'))
- else:
- return None
diff --git a/linuxgaming/util.py b/linuxgaming/util.py
@@ -1,13 +1,43 @@
import yaml
-from flask import (current_app)
+import feedparser
+import json
+import requests
+from flask import (current_app, abort)
def load_yaml():
"""Return the YAML parsed config file."""
+
try:
with open('config/feed_config.yaml', 'r') as ymlfile:
cfg = yaml.load(ymlfile)
- except yaml.YAMLError as exc:
- current_app.logger.error('YAML read error %s', exc)
+ except (yaml.YAMLError, FileNotFoundError) as e:
+ current_app.logger.error('YAML read error %s', e)
+ abort(500)
return cfg
+
+
+def feed_parse(url):
+
+ # parse the feed and get the results
+ res = feedparser.parse(url)
+
+ if res.entries:
+ return res.entries
+ else:
+ current_app.logger.error('FEED parse error %s', url)
+
+ return None
+
+
+def get_gog_info(query):
+
+ gog_api_url = "https://embed.gog.com/games/ajax/filtered?"
+
+ response = requests.get(gog_api_url + query)
+
+ if response.status_code == 200:
+ return json.loads(response.content.decode('utf-8'))
+ else:
+ abort(500)
You are viewing proxied material from jay.scot. The copyright of proxied material belongs to its original authors. Any comments or complaints in relation to proxied material should be directed to the original authors of the content concerned. Please see the disclaimer for more details.