From fbb70a3e902c5f795b139658e28b3c9f02d7fc47 Mon Sep 17 00:00:00 2001
From: KazooTTT <work@kazoottt.top>
Date: Sat, 23 Nov 2024 23:24:44 +0800
Subject: [PATCH] feat: add robots.txt

---
 public/robots.txt | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 public/robots.txt

diff --git a/public/robots.txt b/public/robots.txt
new file mode 100644
index 0000000..b18adbf
--- /dev/null
+++ b/public/robots.txt
@@ -0,0 +1,17 @@
+User-agent: *
+Allow: /
+
+# Sitemaps
+Sitemap: https://blog.kazoottt.top/sitemap-index.xml
+
+# Common crawl optimizations
+Disallow: /api/
+Disallow: /_astro/
+Disallow: /404
+Disallow: /500
+
+# Allow crawling of content
+Allow: /posts/
+Allow: /categories/
+Allow: /tags/
+Allow: /blog/