feat: add error handling for data cascade

src/_data/lastfm.js

@@ -13,11 +13,35 @@ const lastFmApiKey = process.env.LAST_FM_API_KEY;
 const baseUrl = "http://ws.audioscrobbler.com";
 const username = "wonderfulfrog";
 
-const fetchRecentAlbums = async (period = "7day") => {
-  const path = `/2.0/?method=user.gettopalbums&user=${username}&api_key=${lastFmApiKey}&format=json`;
-  const url = `${baseUrl}${path}&period=${period}`;
+const fetchLastFm = async (method, duration, extraArgs) => {
+  try {
+    const path = `/2.0/?method=${method}&user=${username}&api_key=${lastFmApiKey}&format=json`;
+    let url = `${baseUrl}${path}`;
+
+    if (extraArgs) {
+      url = `${url}&${extraArgs}`;
+    }
+
+    const response = await EleventyFetch(url, { duration, type: "json" });
+
+    return response;
+  } catch (e) {
+    console.error(`Error fetching last.fm data for method=${method}`, e);
+    return undefined;
+  }
+};
+
+const fetchRecentAlbums = async (period = "7day") => {
+  const response = await fetchLastFm(
+    "user.gettopalbums",
+    "7d",
+    `period=${period}`,
+  );
+
+  if (!response) {
+    return [];
+  }
 
-  const response = await EleventyFetch(url, { duration: "7d", type: "json" });
   const albums = response.topalbums.album.slice(0, 8);
 
   const recentAlbums = albums.map((album) => {
@@ -40,9 +64,12 @@ const fetchRecentAlbums = async (period = "7day") => {
 };
 
 const fetchRecentTracks = async () => {
-  const url = `http://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=wonderfulfrog&api_key=${lastFmApiKey}&format=json`;
+  const response = await fetchLastFm("user.getrecenttracks", "5m");
+
+  if (!response) {
+    return [];
+  }
 
-  const response = await EleventyFetch(url, { duration: "5m", type: "json" });
   const tracks = response.recenttracks.track.slice(0, 5);
 
   const recentTracks = tracks.map((track) => {

src/_data/letterboxd.js

@@ -9,10 +9,23 @@ const relativeTime = require("dayjs/plugin/relativeTime");
 dayjs.extend(utc);
 dayjs.extend(relativeTime);
 
-const fetchRecentMovies = async () => {
+const fetchLetterboxd = async (duration) => {
+  try {
     const url = `https://letterboxd.com/wonderfulfrog/rss/`;
+    const response = await EleventyFetch(url, { duration, type: "text" });
+    return response;
+  } catch (e) {
+    console.error("Error fetching data from Letterboxd", e);
+    return undefined;
+  }
+};
 
-  const response = await EleventyFetch(url, { duration: "1d", type: "text" });
+const fetchRecentMovies = async () => {
+  const response = await fetchLetterboxd("1d");
+
+  if (!response) {
+    return [];
+  }
 
   const $ = cheerio.load(response, { xml: true });
 

src/_data/robots.js

@@ -4,12 +4,80 @@ const EleventyFetch = require("@11ty/eleventy-fetch");
 
 const accessToken = process.env.DARK_VISITORS_ACCESS_TOKEN;
 
+const STATIC = `# AI Data Scraper
+# https://darkvisitors.com/agents/bytespider
+
+User-agent: Bytespider
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/ccbot
+
+User-agent: CCBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/claudebot
+
+User-agent: ClaudeBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/diffbot
+
+User-agent: Diffbot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/facebookbot
+
+User-agent: FacebookBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/google-extended
+
+User-agent: Google-Extended
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/gptbot
+
+User-agent: GPTBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/omgili
+
+User-agent: omgili
+Disallow: /
+
+# Undocumented AI Agent
+# https://darkvisitors.com/agents/anthropic-ai
+
+User-agent: anthropic-ai
+Disallow: /
+
+# Undocumented AI Agent
+# https://darkvisitors.com/agents/claude-web
+
+User-agent: Claude-Web
+Disallow: /
+
+# Undocumented AI Agent
+# https://darkvisitors.com/agents/cohere-ai
+
+User-agent: cohere-ai
+Disallow: /
+`;
+
 const fetchRobotsTxt = async () => {
   const url = "https://api.darkvisitors.com/robots-txts";
   const body = JSON.stringify({
     agent_types: ["AI Assistant", "AI Data Scraper", "AI Search Crawler"],
     disallow: "/",
   });
+  try {
     const response = await EleventyFetch(url, {
       duration: "1d",
       type: "text",
@@ -24,10 +92,21 @@ const fetchRobotsTxt = async () => {
     });
 
     return response.toString();
+  } catch (e) {
+    console.error(
+      "Error fetching robots.txt from Dark Visitors API, falling back to static version",
+      e,
+    );
+    return undefined;
+  }
 };
 
 module.exports = async function () {
   const robotsTxt = await fetchRobotsTxt();
 
+  if (!robotsTxt) {
+    return STATIC;
+  }
+
   return robotsTxt;
 };

src/pages/changelog.md

@@ -10,6 +10,10 @@ All the changes that are fit to read!
 
 If preferred, the [commit log is available here][commits].
 
+## May 24th 2024
+
+- Improve error handling when env vars are missing, or fetch requests fail.
+
 ## April 7th 2024
 
 - Updated `robots.txt` to use [Dark Visitors][darkvisitors]' API.

src/pages/now.html

@@ -6,11 +6,14 @@ description: What's going on now and all the latest with myself.
 {% set recentTrack = lastfm.recentTracks[0] %}
 <h1>/now</h1>
 <p>What am I doing right now? Everything on here is automatically generated from various data sources.</p>
+{% if recentTrack %}
     <p>
         🎶 <a href="{{ recentTrack.url }}"
     target="_blank"
     rel="external noreferrer noopener">{{ recentTrack.artist }} - {{ recentTrack.track }}</a>
     </p>
+{% endif %}
+{% if lastfm.recentAlbums.length > 0 %}
     <h2>💿 Albums</h2>
     <ul class="[ media-grid square ][ p-0 ]" role="list">
         {% for album in lastfm.recentAlbums %}
@@ -27,6 +30,8 @@ description: What's going on now and all the latest with myself.
             </li>
         {% endfor %}
     </ul>
+{% endif %}
+{% if letterboxd.length > 0 %}
     <h2>🍿 Movies</h2>
     <ul class="[ media-grid poster ][ p-0 ]" role="list">
         {% for movie in letterboxd %}
@@ -43,6 +48,7 @@ description: What's going on now and all the latest with myself.
             </li>
         {% endfor %}
     </ul>
+{% endif %}
 <p class="[ font-size-s mt-2 ]">
     What is a `/now` page? Check out <a href="https://nownownow.com/about">nownownow.com</a>.
 </p>