From 896c86d9d2095fa44e88ab12cd464b44740ffc50 Mon Sep 17 00:00:00 2001 From: Devin Haska <2636402+wonderfulfrog@users.noreply.github.com> Date: Fri, 24 May 2024 12:55:31 -0700 Subject: [PATCH] feat: add error handling for data cascade --- src/_data/lastfm.js | 39 ++++++++++++--- src/_data/letterboxd.js | 19 ++++++-- src/_data/robots.js | 103 +++++++++++++++++++++++++++++++++++----- src/pages/changelog.md | 4 ++ src/pages/now.html | 76 +++++++++++++++-------------- 5 files changed, 185 insertions(+), 56 deletions(-) diff --git a/src/_data/lastfm.js b/src/_data/lastfm.js index f4c7e03..bde62a4 100644 --- a/src/_data/lastfm.js +++ b/src/_data/lastfm.js @@ -13,11 +13,35 @@ const lastFmApiKey = process.env.LAST_FM_API_KEY; const baseUrl = "http://ws.audioscrobbler.com"; const username = "wonderfulfrog"; -const fetchRecentAlbums = async (period = "7day") => { - const path = `/2.0/?method=user.gettopalbums&user=${username}&api_key=${lastFmApiKey}&format=json`; - const url = `${baseUrl}${path}&period=${period}`; +const fetchLastFm = async (method, duration, extraArgs) => { + try { + const path = `/2.0/?method=${method}&user=${username}&api_key=${lastFmApiKey}&format=json`; + let url = `${baseUrl}${path}`; + + if (extraArgs) { + url = `${url}&${extraArgs}`; + } + + const response = await EleventyFetch(url, { duration, type: "json" }); + + return response; + } catch (e) { + console.error(`Error fetching last.fm data for method=${method}`, e); + return undefined; + } +}; + +const fetchRecentAlbums = async (period = "7day") => { + const response = await fetchLastFm( + "user.gettopalbums", + "7d", + `period=${period}`, + ); + + if (!response) { + return []; + } - const response = await EleventyFetch(url, { duration: "7d", type: "json" }); const albums = response.topalbums.album.slice(0, 8); const recentAlbums = albums.map((album) => { @@ -40,9 +64,12 @@ const fetchRecentAlbums = async (period = "7day") => { }; const fetchRecentTracks = async () => { - const url = `http://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=wonderfulfrog&api_key=${lastFmApiKey}&format=json`; + const response = await fetchLastFm("user.getrecenttracks", "5m"); + + if (!response) { + return []; + } - const response = await EleventyFetch(url, { duration: "5m", type: "json" }); const tracks = response.recenttracks.track.slice(0, 5); const recentTracks = tracks.map((track) => { diff --git a/src/_data/letterboxd.js b/src/_data/letterboxd.js index 2ee1edc..e05731e 100644 --- a/src/_data/letterboxd.js +++ b/src/_data/letterboxd.js @@ -9,10 +9,23 @@ const relativeTime = require("dayjs/plugin/relativeTime"); dayjs.extend(utc); dayjs.extend(relativeTime); -const fetchRecentMovies = async () => { - const url = `https://letterboxd.com/wonderfulfrog/rss/`; +const fetchLetterboxd = async (duration) => { + try { + const url = `https://letterboxd.com/wonderfulfrog/rss/`; + const response = await EleventyFetch(url, { duration, type: "text" }); + return response; + } catch (e) { + console.error("Error fetching data from Letterboxd", e); + return undefined; + } +}; - const response = await EleventyFetch(url, { duration: "1d", type: "text" }); +const fetchRecentMovies = async () => { + const response = await fetchLetterboxd("1d"); + + if (!response) { + return []; + } const $ = cheerio.load(response, { xml: true }); diff --git a/src/_data/robots.js b/src/_data/robots.js index 9ac4e4c..0b134b5 100644 --- a/src/_data/robots.js +++ b/src/_data/robots.js @@ -4,30 +4,109 @@ const EleventyFetch = require("@11ty/eleventy-fetch"); const accessToken = process.env.DARK_VISITORS_ACCESS_TOKEN; +const STATIC = `# AI Data Scraper +# https://darkvisitors.com/agents/bytespider + +User-agent: Bytespider +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/ccbot + +User-agent: CCBot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/claudebot + +User-agent: ClaudeBot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/diffbot + +User-agent: Diffbot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/facebookbot + +User-agent: FacebookBot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/google-extended + +User-agent: Google-Extended +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/gptbot + +User-agent: GPTBot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/omgili + +User-agent: omgili +Disallow: / + +# Undocumented AI Agent +# https://darkvisitors.com/agents/anthropic-ai + +User-agent: anthropic-ai +Disallow: / + +# Undocumented AI Agent +# https://darkvisitors.com/agents/claude-web + +User-agent: Claude-Web +Disallow: / + +# Undocumented AI Agent +# https://darkvisitors.com/agents/cohere-ai + +User-agent: cohere-ai +Disallow: / +`; + const fetchRobotsTxt = async () => { const url = "https://api.darkvisitors.com/robots-txts"; const body = JSON.stringify({ agent_types: ["AI Assistant", "AI Data Scraper", "AI Search Crawler"], disallow: "/", }); - const response = await EleventyFetch(url, { - duration: "1d", - type: "text", - fetchOptions: { - method: "POST", - headers: { - Authorization: `Bearer ${accessToken}`, - ["Content-Type"]: "application/json", + try { + const response = await EleventyFetch(url, { + duration: "1d", + type: "text", + fetchOptions: { + method: "POST", + headers: { + Authorization: `Bearer ${accessToken}`, + ["Content-Type"]: "application/json", + }, + body, }, - body, - }, - }); + }); - return response.toString(); + return response.toString(); + } catch (e) { + console.error( + "Error fetching robots.txt from Dark Visitors API, falling back to static version", + e, + ); + return undefined; + } }; module.exports = async function () { const robotsTxt = await fetchRobotsTxt(); + if (!robotsTxt) { + return STATIC; + } + return robotsTxt; }; diff --git a/src/pages/changelog.md b/src/pages/changelog.md index a3e9c4e..ac0d210 100644 --- a/src/pages/changelog.md +++ b/src/pages/changelog.md @@ -10,6 +10,10 @@ All the changes that are fit to read! If preferred, the [commit log is available here][commits]. +## May 24th 2024 + +- Improve error handling when env vars are missing, or fetch requests fail. + ## April 7th 2024 - Updated `robots.txt` to use [Dark Visitors][darkvisitors]' API. diff --git a/src/pages/now.html b/src/pages/now.html index 30b41db..272f5c5 100644 --- a/src/pages/now.html +++ b/src/pages/now.html @@ -6,43 +6,49 @@ description: What's going on now and all the latest with myself. {% set recentTrack = lastfm.recentTracks[0] %}

/now

What am I doing right now? Everything on here is automatically generated from various data sources.

-

- 🎶 + 🎶 {{ recentTrack.artist }} - {{ recentTrack.track }} -

-

💿 Albums

- -

🍿 Movies

- +

+{% endif %} +{% if lastfm.recentAlbums.length > 0 %} +

💿 Albums

+ +{% endif %} +{% if letterboxd.length > 0 %} +

🍿 Movies

+ +{% endif %}

What is a `/now` page? Check out nownownow.com.