feat: add darkvisitors API to robots.txt

src/_data/robots.js

@@ -0,0 +1,33 @@
+require("dotenv").config();
+
+const EleventyFetch = require("@11ty/eleventy-fetch");
+
+const accessToken = process.env.DARK_VISITORS_ACCESS_TOKEN;
+
+const fetchRobotsTxt = async () => {
+  const url = "https://api.darkvisitors.com/robots-txts";
+  const body = JSON.stringify({
+    agent_types: ["AI Assistant", "AI Data Scraper", "AI Search Crawler"],
+    disallow: "/",
+  });
+  const response = await EleventyFetch(url, {
+    duration: "1d",
+    type: "text",
+    fetchOptions: {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${accessToken}`,
+        ["Content-Type"]: "application/json",
+      },
+      body,
+    },
+  });
+
+  return response.toString();
+};
+
+module.exports = async function () {
+  const robotsTxt = await fetchRobotsTxt();
+
+  return robotsTxt;
+};

src/pages/changelog.md

@@ -10,6 +10,10 @@ All the changes that are fit to read!
 
 If preferred, the [commit log is available here][commits].
 
+## April 7th 2024
+
+- Updated `robots.txt` to use [Dark Visitors][darkvisitors]' API.
+
 ## March 23rd 2024
 
 - Fixed "Posts" page showing up under new posts
@@ -64,3 +68,4 @@ If preferred, the [commit log is available here][commits].
 [11ty]: https://www.11ty.dev/
 [v3]: /posts/version-3/
 [blogroll]: /blogroll/
+[darkvisitors]: https://darkvisitors.com

src/robots.njk

@@ -10,3 +10,5 @@ User-agent: *
 Disallow: /404.html
 Disallow: /admin/index.html
 Disallow: /admin/config.yml
+
+{{ robots }}