feat: add error handling for data cascade

src/_data/lastfm.js

@@ -13,11 +13,35 @@ const lastFmApiKey = process.env.LAST_FM_API_KEY;
 const baseUrl = "http://ws.audioscrobbler.com";
 const username = "wonderfulfrog";
 
-const fetchRecentAlbums = async (period = "7day") => {
+const fetchLastFm = async (method, duration, extraArgs) => {
-  const path = `/2.0/?method=user.gettopalbums&user=${username}&api_key=${lastFmApiKey}&format=json`;
+  try {
-  const url = `${baseUrl}${path}&period=${period}`;
+    const path = `/2.0/?method=${method}&user=${username}&api_key=${lastFmApiKey}&format=json`;
+    let url = `${baseUrl}${path}`;
+
+    if (extraArgs) {
+      url = `${url}&${extraArgs}`;
+    }
+
+    const response = await EleventyFetch(url, { duration, type: "json" });
+
+    return response;
+  } catch (e) {
+    console.error(`Error fetching last.fm data for method=${method}`, e);
+    return undefined;
+  }
+};
+
+const fetchRecentAlbums = async (period = "7day") => {
+  const response = await fetchLastFm(
+    "user.gettopalbums",
+    "7d",
+    `period=${period}`,
+  );
+
+  if (!response) {
+    return [];
+  }
 
-  const response = await EleventyFetch(url, { duration: "7d", type: "json" });
   const albums = response.topalbums.album.slice(0, 8);
 
   const recentAlbums = albums.map((album) => {
@@ -40,9 +64,12 @@ const fetchRecentAlbums = async (period = "7day") => {
 };
 
 const fetchRecentTracks = async () => {
-  const url = `http://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=wonderfulfrog&api_key=${lastFmApiKey}&format=json`;
+  const response = await fetchLastFm("user.getrecenttracks", "5m");
+
+  if (!response) {
+    return [];
+  }
 
-  const response = await EleventyFetch(url, { duration: "5m", type: "json" });
   const tracks = response.recenttracks.track.slice(0, 5);
 
   const recentTracks = tracks.map((track) => {

src/_data/letterboxd.js

@@ -9,10 +9,23 @@ const relativeTime = require("dayjs/plugin/relativeTime");
 dayjs.extend(utc);
 dayjs.extend(relativeTime);
 
-const fetchRecentMovies = async () => {
+const fetchLetterboxd = async (duration) => {
-  const url = `https://letterboxd.com/wonderfulfrog/rss/`;
+  try {
+    const url = `https://letterboxd.com/wonderfulfrog/rss/`;
+    const response = await EleventyFetch(url, { duration, type: "text" });
+    return response;
+  } catch (e) {
+    console.error("Error fetching data from Letterboxd", e);
+    return undefined;
+  }
+};
 
-  const response = await EleventyFetch(url, { duration: "1d", type: "text" });
+const fetchRecentMovies = async () => {
+  const response = await fetchLetterboxd("1d");
+
+  if (!response) {
+    return [];
+  }
 
   const $ = cheerio.load(response, { xml: true });
 

src/_data/robots.js

@@ -4,30 +4,109 @@ const EleventyFetch = require("@11ty/eleventy-fetch");
 
 const accessToken = process.env.DARK_VISITORS_ACCESS_TOKEN;
 
+const STATIC = `# AI Data Scraper
+# https://darkvisitors.com/agents/bytespider
+
+User-agent: Bytespider
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/ccbot
+
+User-agent: CCBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/claudebot
+
+User-agent: ClaudeBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/diffbot
+
+User-agent: Diffbot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/facebookbot
+
+User-agent: FacebookBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/google-extended
+
+User-agent: Google-Extended
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/gptbot
+
+User-agent: GPTBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/omgili
+
+User-agent: omgili
+Disallow: /
+
+# Undocumented AI Agent
+# https://darkvisitors.com/agents/anthropic-ai
+
+User-agent: anthropic-ai
+Disallow: /
+
+# Undocumented AI Agent
+# https://darkvisitors.com/agents/claude-web
+
+User-agent: Claude-Web
+Disallow: /
+
+# Undocumented AI Agent
+# https://darkvisitors.com/agents/cohere-ai
+
+User-agent: cohere-ai
+Disallow: /
+`;
+
 const fetchRobotsTxt = async () => {
   const url = "https://api.darkvisitors.com/robots-txts";
   const body = JSON.stringify({
     agent_types: ["AI Assistant", "AI Data Scraper", "AI Search Crawler"],
     disallow: "/",
   });
-  const response = await EleventyFetch(url, {
+  try {
-    duration: "1d",
+    const response = await EleventyFetch(url, {
-    type: "text",
+      duration: "1d",
-    fetchOptions: {
+      type: "text",
-      method: "POST",
+      fetchOptions: {
-      headers: {
+        method: "POST",
-        Authorization: `Bearer ${accessToken}`,
+        headers: {
-        ["Content-Type"]: "application/json",
+          Authorization: `Bearer ${accessToken}`,
+          ["Content-Type"]: "application/json",
+        },
+        body,
       },
-      body,
+    });
-    },
-  });
 
-  return response.toString();
+    return response.toString();
+  } catch (e) {
+    console.error(
+      "Error fetching robots.txt from Dark Visitors API, falling back to static version",
+      e,
+    );
+    return undefined;
+  }
 };
 
 module.exports = async function () {
   const robotsTxt = await fetchRobotsTxt();
 
+  if (!robotsTxt) {
+    return STATIC;
+  }
+
   return robotsTxt;
 };

src/pages/changelog.md

@@ -10,6 +10,10 @@ All the changes that are fit to read!
 
 If preferred, the [commit log is available here][commits].
 
+## May 24th 2024
+
+- Improve error handling when env vars are missing, or fetch requests fail.
+
 ## April 7th 2024
 
 - Updated `robots.txt` to use [Dark Visitors][darkvisitors]' API.

src/pages/now.html

@@ -6,43 +6,49 @@ description: What's going on now and all the latest with myself.
 {% set recentTrack = lastfm.recentTracks[0] %}
 <h1>/now</h1>
 <p>What am I doing right now? Everything on here is automatically generated from various data sources.</p>
-<p>
+{% if recentTrack %}
-    🎶 <a href="{{ recentTrack.url }}"
+    <p>
+        🎶 <a href="{{ recentTrack.url }}"
     target="_blank"
     rel="external noreferrer noopener">{{ recentTrack.artist }} - {{ recentTrack.track }}</a>
-</p>
+    </p>
-<h2>💿 Albums</h2>
+{% endif %}
-<ul class="[ media-grid square ][ p-0 ]" role="list">
+{% if lastfm.recentAlbums.length > 0 %}
-    {% for album in lastfm.recentAlbums %}
+    <h2>💿 Albums</h2>
-        <li>
+    <ul class="[ media-grid square ][ p-0 ]" role="list">
-            <a href="{{ album.url }}"
+        {% for album in lastfm.recentAlbums %}
-               target="_blank"
+            <li>
-               rel="external noreferrer noopener">
+                <a href="{{ album.url }}"
-                {% set textContent %}
+                   target="_blank"
-                    {{ album.artist }} - {{ album.album }}
+                   rel="external noreferrer noopener">
-                {% endset %}
+                    {% set textContent %}
-                <p class="[ visually-hidden ]">{{ textContent }}</p>
+                        {{ album.artist }} - {{ album.album }}
-                {% image album.imageUrl, "", "", "", textContent %}
+                    {% endset %}
-            </a>
+                    <p class="[ visually-hidden ]">{{ textContent }}</p>
-        </li>
+                    {% image album.imageUrl, "", "", "", textContent %}
-    {% endfor %}
+                </a>
-</ul>
+            </li>
-<h2>🍿 Movies</h2>
+        {% endfor %}
-<ul class="[ media-grid poster ][ p-0 ]" role="list">
+    </ul>
-    {% for movie in letterboxd %}
+{% endif %}
-        <li>
+{% if letterboxd.length > 0 %}
-            <a href="{{ movie.url }}"
+    <h2>🍿 Movies</h2>
-               target="_blank"
+    <ul class="[ media-grid poster ][ p-0 ]" role="list">
-               rel="external noreferrer noopener">
+        {% for movie in letterboxd %}
-                {% set textContent %}
+            <li>
-                    {{ movie.title }}
+                <a href="{{ movie.url }}"
-                {% endset %}
+                   target="_blank"
-                <p class="[ visually-hidden ]">{{ movie.title }}</p>
+                   rel="external noreferrer noopener">
-                {% image movie.imgSrc, "", "", "" , textContent %}
+                    {% set textContent %}
-            </a>
+                        {{ movie.title }}
-        </li>
+                    {% endset %}
-    {% endfor %}
+                    <p class="[ visually-hidden ]">{{ movie.title }}</p>
-</ul>
+                    {% image movie.imgSrc, "", "", "" , textContent %}
+                </a>
+            </li>
+        {% endfor %}
+    </ul>
+{% endif %}
 <p class="[ font-size-s mt-2 ]">
     What is a `/now` page? Check out <a href="https://nownownow.com/about">nownownow.com</a>.
 </p>