Introduction
Introduction Statistics Contact Development Disclaimer Help
documentation improvements - sfeed - RSS and Atom parser
git clone git://git.codemadness.org/sfeed
Log
Files
Refs
README
LICENSE
---
commit ec7abe8b5ea4c56bd442f536544c4dde4b38a847
parent 4b99778344bca3a1606046b5bf1087160ea1661c
Author: Hiltjo Posthuma <[email protected]>
Date: Sun, 27 Oct 2024 09:23:53 +0100
documentation improvements
- Add ETag example to sfeedrc.5 man page.
This was in the README but now also added to the man page for visibility.
- Indent code examples to separate leading text from code examples.
- Some textual improvements.
Diffstat:
M README | 4 ++--
M sfeed.1 | 32 ++++++++++++++++-------------…
M sfeed_content.1 | 9 +++++----
M sfeed_curses.1 | 9 +++++----
M sfeed_markread.1 | 11 +++++++----
M sfeed_mbox.1 | 9 +++++----
M sfeed_update.1 | 8 ++++----
M sfeed_web.1 | 6 +++---
M sfeedrc.5 | 102 +++++++++++++++++++++++++----…
9 files changed, 129 insertions(+), 61 deletions(-)
---
diff --git a/README b/README
@@ -639,9 +639,9 @@ RSS/Atom data or change the default curl options:
- - -
-Caching, incremental data updates and bandwidth-saving
+Caching, incremental data updates and bandwidth saving
-For servers that support it some incremental updates and bandwidth-saving can
+For servers that support it some incremental updates and bandwidth saving can
be done by using the "ETag" HTTP header.
Create a directory for storing the ETags per feed:
diff --git a/sfeed.1 b/sfeed.1
@@ -1,4 +1,4 @@
-.Dd January 7, 2023
+.Dd October 27, 2024
.Dt SFEED 1
.Os
.Sh NAME
@@ -52,51 +52,51 @@ Item, categories, multiple values are separated by the '|' …
.Sh EXIT STATUS
.Ex -std
.Sh EXAMPLES
-.Bd -literal
+.Bd -literal -offset 4n
curl -s 'https://codemadness.org/atom.xml' | sfeed
.Ed
.Pp
To convert the character set from a feed that is not UTF-8 encoded the
.Xr iconv 1
tool can be used:
-.Bd -literal
+.Bd -literal -offset 4n
curl -s 'https://codemadness.org/some_iso-8859-1_feed.xml' | \e
iconv -f iso-8859-1 -t utf-8 | \e
sfeed
.Ed
.Sh EXAMPLE SETUP
1. Create a directory for the sfeedrc configuration and the feeds:
-.Bd -literal
- mkdir -p ~/.sfeed/feeds
+.Bd -literal -offset 4n
+mkdir -p ~/.sfeed/feeds
.Ed
.Pp
2. Copy the example
.Xr sfeedrc 5
configuration:
-.Bd -literal
- cp sfeedrc.example ~/.sfeed/sfeedrc
- $EDITOR ~/.sfeed/sfeedrc
+.Bd -literal -offset 4n
+cp sfeedrc.example ~/.sfeed/sfeedrc
+$EDITOR ~/.sfeed/sfeedrc
.Ed
.Pp
Or import existing OPML subscriptions using
.Xr sfeed_opml_import 1 :
-.Bd -literal
- sfeed_opml_import < file.opml > ~/.sfeed/sfeedrc
+.Bd -literal -offset 4n
+sfeed_opml_import < file.opml > ~/.sfeed/sfeedrc
.Ed
.Pp
3. To update feeds and merge the new items with existing items:
-.Bd -literal
- sfeed_update
+.Bd -literal -offset 4n
+sfeed_update
.Ed
.Pp
4. Format feeds to a plain-text list:
-.Bd -literal
- sfeed_plain ~/.sfeed/feeds/*
+.Bd -literal -offset 4n
+sfeed_plain ~/.sfeed/feeds/*
.Ed
.Pp
Or format feeds to a curses interface:
-.Bd -literal
- sfeed_curses ~/.sfeed/feeds/*
+.Bd -literal -offset 4n
+sfeed_curses ~/.sfeed/feeds/*
.Ed
.Pp
There are also other formatting programs included.
diff --git a/sfeed_content.1 b/sfeed_content.1
@@ -1,4 +1,4 @@
-.Dd December 22, 2021
+.Dd October 27, 2024
.Dt SFEED_CONTENT 1
.Os
.Sh NAME
@@ -35,12 +35,13 @@ If it is not set it will use lynx by default.
.Sh EXIT STATUS
.Ex -std
.Sh EXAMPLES
-.Bd -literal
+Example:
+.Bd -literal -offset 4n
curl -s 'https://codemadness.org/atom_content.xml' | sfeed | sfeed_content
.Ed
.Pp
-The output format looks like this:
-.Bd -literal
+The output format will look like this:
+.Bd -literal -offset 4n
Title: The title.
Author: The line with the author if it is set.
Category: The line with the categories if it is set.
diff --git a/sfeed_curses.1 b/sfeed_curses.1
@@ -1,4 +1,4 @@
-.Dd August 1, 2023
+.Dd October 27, 2024
.Dt SFEED_CURSES 1
.Os
.Sh NAME
@@ -299,14 +299,15 @@ When plumbing an URL then stdin is closed also.
.Ex -std
The exit status is 130 on SIGINT and 143 on SIGTERM.
.Sh EXAMPLES
-.Bd -literal
+Example:
+.Bd -literal -offset 4n
sfeed_curses ~/.sfeed/feeds/*
.Ed
.Pp
Another example which shows some of the features
.Nm
has:
-.Bd -literal
+.Bd -literal -offset 4n
export SFEED_AUTOCMD="2tgo"
export SFEED_URL_FILE="$HOME/.sfeed/urls"
[ -f "$SFEED_URL_FILE" ] || touch "$SFEED_URL_FILE"
@@ -316,7 +317,7 @@ sfeed_curses ~/.sfeed/feeds/*
Which does the following:
.Bl -enum
.It
-Set commands to execute automatically:
+Set commands to execute automatically on startup:
.Pp
Set the current layout to a horizontal mode ('2' keybind).
Showing a feeds sidebar on the top and the feed items on the bottom.
diff --git a/sfeed_markread.1 b/sfeed_markread.1
@@ -1,4 +1,4 @@
-.Dd July 25, 2021
+.Dd October 27, 2024
.Dt SFEED_MARKREAD 1
.Os
.Sh NAME
@@ -31,13 +31,16 @@ plain-text list of read URLs.
.Sh EXIT STATUS
.Ex -std
.Sh EXAMPLES
-.Bd -literal
+Example:
+.Bd -literal -offset 4n
export SFEED_URL_FILE="$HOME/.sfeed/urls"
echo 'https://codemadness.org/sfeed.html' | sfeed_markread read
.Ed
.Pp
-or
-.Bd -literal
+or pass the
+.Ar urlfile
+as a parameter:
+.Bd -literal -offset 4n
echo 'https://codemadness.org/sfeed.html' | sfeed_markread read ~/.sfeed/urls
.Ed
.Sh SEE ALSO
diff --git a/sfeed_mbox.1 b/sfeed_mbox.1
@@ -1,4 +1,4 @@
-.Dd August 4, 2021
+.Dd October 27, 2024
.Dt SFEED_MBOX 1
.Os
.Sh NAME
@@ -47,14 +47,15 @@ By default this is set to "0".
.Sh EXIT STATUS
.Ex -std
.Sh EXAMPLES
-.Bd -literal
+Example:
+.Bd -literal -offset 4n
sfeed_mbox ~/.sfeed/feeds/*
.Ed
.Pp
-To include the content.
+Below is an example to include the content.
This can be insecure for some of the mail clients that interpret HTML code in
an unsafe way:
-.Bd -literal
+.Bd -literal -offset 4n
SFEED_MBOX_CONTENT=1 sfeed_mbox ~/.sfeed/feeds/*
.Ed
.Sh SEE ALSO
diff --git a/sfeed_update.1 b/sfeed_update.1
@@ -1,4 +1,4 @@
-.Dd August 1, 2023
+.Dd October 27, 2024
.Dt SFEED_UPDATE 1
.Os
.Sh NAME
@@ -35,7 +35,7 @@ man page for a detailed description of the format and an exam…
.El
.Sh FILES WRITTEN
.Bl -tag -width Ds
-.It feedname
+.It Pa feedname
TAB-separated
.Xr sfeed 5
format containing all items per feed.
@@ -55,7 +55,7 @@ entry-point.
.Sh LOGGING
When processing a feed it will log failures to stderr and non-failures to
stdout in the format:
-.Bd -literal
+.Bd -literal -offset 4n
[HH:MM:SS] feedname message
.Ed
.Sh EXIT STATUS
@@ -63,7 +63,7 @@ stdout in the format:
If any of the feeds failed to update then the exit status is non-zero.
.Sh EXAMPLES
To update your feeds and format them in various formats:
-.Bd -literal
+.Bd -literal -offset 4n
# Update feeds
sfeed_update "configfile"
# Format to a plain-text list
diff --git a/sfeed_web.1 b/sfeed_web.1
@@ -13,8 +13,8 @@ reads the HTML data of the webpage from stdin and writes the …
stdout.
.Pp
Such a link reference in HTML code looks like:
-.Bd -literal
- <link rel="alternate" href="atom.xml" type="application/atom+xml" />
+.Bd -literal -offset 4n
+<link rel="alternate" href="atom.xml" type="application/atom+xml" />
.Ed
.Sh OPTIONS
.Bl -tag -width 8n
@@ -39,7 +39,7 @@ Usually application/atom+xml or application/rss+xml.
.Ex -std
.Sh EXAMPLES
Get URLs from a website:
-.Bd -literal
+.Bd -literal -offset 4n
curl -s -L 'https://codemadness.org/' | sfeed_web 'https://codemadness.org/'
.Ed
.Sh SEE ALSO
diff --git a/sfeedrc.5 b/sfeedrc.5
@@ -1,4 +1,4 @@
-.Dd December 26, 2023
+.Dd October 30, 2024
.Dt SFEEDRC 5
.Os
.Sh NAME
@@ -7,7 +7,8 @@
.Sh DESCRIPTION
.Nm
is the configuration file for
-.Xr sfeed_update 1 .
+.Xr sfeed_update 1
+and is evaluated as a shellscript.
.Sh VARIABLES
.Bl -tag -width Ds
.It Va sfeedpath
@@ -30,7 +31,8 @@ Inside the
.Fn feeds
function feeds can be defined by calling the
.Fn feed
-function, its arguments are:
+function.
+Its arguments are:
.Bl -tag -width Ds
.It Fa name
Name of the feed, this is also used as the filename for the TAB-separated
@@ -41,16 +43,22 @@ Each
.Fa name
should be unique.
.It Fa feedurl
-URL to fetch the RSS/Atom data from, usually a HTTP or HTTPS URL.
+URL to fetch the RSS/Atom data from.
+This is usually a HTTP or HTTPS URL.
.It Op Fa basesiteurl
Base URL of the feed links.
-This argument allows to fix relative item links.
+This argument allows fixing relative item links.
.Pp
-According to the RSS and Atom specification feeds should always have absolute
-URLs, however this is not always the case in practise.
+According to the RSS and Atom specification, feeds should always have absolute
+URLs, but this is not always the case in practise.
.It Op Fa encoding
-Feeds are decoded from this name to UTF-8, the name should be a usable
-character-set for the
+Feeds are converted from this
+.Ar encoding
+to UTF-8.
+The
+.Ar encoding
+should be a usable
+character-set name for the
.Xr iconv 1
tool.
.El
@@ -58,11 +66,12 @@ tool.
.Sh OVERRIDE FUNCTIONS
Because
.Xr sfeed_update 1
-is a shellscript each function can be overridden to change its behaviour,
-notable functions are:
+is a shellscript each function can be overridden to change its behaviour.
+Notable functions are:
.Bl -tag -width Ds
.It Fn fetch "name" "url" "feedfile"
-Fetch feed from URL and write the data to stdout, its arguments are:
+Fetch feed from URL and write the data to stdout.
+Its arguments are:
.Bl -tag -width Ds
.It Fa name
Specified name in configuration file (useful for logging).
@@ -77,8 +86,8 @@ By default the tool
is used.
.It Fn convertencoding "name" "from" "to"
Convert data from stdin from one text-encoding to another and write it to
-stdout,
-its arguments are:
+stdout.
+Its arguments are:
.Bl -tag -width Ds
.It Fa name
Feed name.
@@ -95,6 +104,7 @@ is used.
Read RSS/Atom XML data from stdin, convert and write it as
.Xr sfeed 5
data to stdout.
+Its arguments are:
.Bl -tag -width Ds
.It Fa name
Name of the feed.
@@ -107,7 +117,8 @@ This argument allows to fix relative item links.
.It Fn filter "name" "url"
Filter
.Xr sfeed 5
-data from stdin and write it to stdout, its arguments are:
+data from stdin and write it to stdout.
+Its arguments are:
.Bl -tag -width Ds
.It Fa name
Feed name.
@@ -117,7 +128,8 @@ URL of the feed.
.It Fn merge "name" "oldfile" "newfile"
Merge
.Xr sfeed 5
-data of oldfile with newfile and write it to stdout, its arguments are:
+data of oldfile with newfile and write it to stdout.
+Its arguments are:
.Bl -tag -width Ds
.It Fa name
Feed name.
@@ -129,7 +141,8 @@ New file.
.It Fn order "name" "url"
Sort
.Xr sfeed 5
-data from stdin and write it to stdout, its arguments are:
+data from stdin and write it to stdout.
+Its arguments are:
.Bl -tag -width Ds
.It Fa name
Feed name.
@@ -140,7 +153,7 @@ URL of the feed.
.Sh EXAMPLES
An example configuration file is included named sfeedrc.example and also
shown below:
-.Bd -literal
+.Bd -literal -offset 4n
#sfeedpath="$HOME/.sfeed/feeds"
# list of feeds to fetch:
@@ -165,8 +178,8 @@ options for fetching the data, the
.Fn fetch
function can be overridden and added at the top of the
.Nm
-file:
-.Bd -literal
+file, for example:
+.Bd -literal -offset 4n
# fetch(name, url, feedfile)
fetch() {
# allow for 1 redirect, set User-Agent, timeout is 15 seconds.
@@ -174,6 +187,55 @@ fetch() {
"$2" 2>/dev/null
}
.Ed
+.Pp
+Caching, incremental data updates and bandwidth saving
+.Pp
+For HTTP servers that support it some bandwidth saving can be done by changing
+some of the default curl options.
+These options can come at a cost of some privacy, because it exposes additional
+metadata from the previous request.
+.Pp
+.Bl -bullet -compact
+.It
+The curl ETag options (--etag-save and --etag-compare) can be used to store and
+send the previous ETag header value.
+curl version 7.73+ is recommended for it to work properly.
+.It
+The curl -z option can be used to send the modification date of a local file as
+a HTTP If-Modified-Since request header.
+The server can then respond if the data is modified or not or respond with only
+the incremental data.
+.It
+The curl --compressed option can be used to indicate the client supports
+decompression.
+Because RSS/Atom feeds are textual XML data this generally compresses very
+well.
+.It
+The example below also sets the User-Agent to sfeed, because some CDNs block
+HTTP clients based on the User-Agent request header.
+.El
+.Pp
+Example:
+.Bd -literal -offset 4n
+etagpath="$HOME/.sfeed/etags"
+mkdir -p "${etagpath}"
+
+# fetch(name, url, feedfile)
+fetch() {
+ etag="${etagpath}/$(basename "$3")"
+
+ curl \e
+ -L --max-redirs 0 \e
+ -H "User-Agent: sfeed" \e
+ -f -s -m 15 \e
+ --compressed \e
+ --etag-save "${etag}" --etag-compare "${etag}" \e
+ -z "${etag}" \e
+ "$2" 2>/dev/null
+}
+.Ed
+.Pp
+The README file has more examples.
.Sh SEE ALSO
.Xr curl 1 ,
.Xr iconv 1 ,
You are viewing proxied material from codemadness.org. The copyright of proxied material belongs to its original authors. Any comments or complaints in relation to proxied material should be directed to the original authors of the content concerned. Please see the disclaimer for more details.