diff --git a/src/_data/robots.js b/src/_data/robots.js new file mode 100644 index 0000000..9ac4e4c --- /dev/null +++ b/src/_data/robots.js @@ -0,0 +1,33 @@ +require("dotenv").config(); + +const EleventyFetch = require("@11ty/eleventy-fetch"); + +const accessToken = process.env.DARK_VISITORS_ACCESS_TOKEN; + +const fetchRobotsTxt = async () => { + const url = "https://api.darkvisitors.com/robots-txts"; + const body = JSON.stringify({ + agent_types: ["AI Assistant", "AI Data Scraper", "AI Search Crawler"], + disallow: "/", + }); + const response = await EleventyFetch(url, { + duration: "1d", + type: "text", + fetchOptions: { + method: "POST", + headers: { + Authorization: `Bearer ${accessToken}`, + ["Content-Type"]: "application/json", + }, + body, + }, + }); + + return response.toString(); +}; + +module.exports = async function () { + const robotsTxt = await fetchRobotsTxt(); + + return robotsTxt; +}; diff --git a/src/pages/changelog.md b/src/pages/changelog.md index ef758fb..a3e9c4e 100644 --- a/src/pages/changelog.md +++ b/src/pages/changelog.md @@ -10,6 +10,10 @@ All the changes that are fit to read! If preferred, the [commit log is available here][commits]. +## April 7th 2024 + +- Updated `robots.txt` to use [Dark Visitors][darkvisitors]' API. + ## March 23rd 2024 - Fixed "Posts" page showing up under new posts @@ -64,3 +68,4 @@ If preferred, the [commit log is available here][commits]. [11ty]: https://www.11ty.dev/ [v3]: /posts/version-3/ [blogroll]: /blogroll/ +[darkvisitors]: https://darkvisitors.com diff --git a/src/robots.njk b/src/robots.njk index 3243f9a..3ea1995 100644 --- a/src/robots.njk +++ b/src/robots.njk @@ -10,3 +10,5 @@ User-agent: * Disallow: /404.html Disallow: /admin/index.html Disallow: /admin/config.yml + +{{ robots }}