fix: use page to provide robot txt

public/robots.txt

@@ -1,22 +0,0 @@
-User-agent: *
-Allow: /
-
-
-HOST: https://blog.kazoottt.top
-# Sitemaps
-Sitemap: https://blog.kazoottt.top/sitemap-index.xml
-
-# Common crawl optimizations
-Disallow: /api/
-Disallow: /_astro/
-Disallow: /404
-Disallow: /500
-
-# Allow crawling of content
-Allow: /posts/
-Allow: /categories/
-Allow: /tags/
-Allow: /blog/
-Allow: /friends/
-Allow: /blog/
-Allow: /tools/

src/pages/robots.txt.ts

@@ -0,0 +1,26 @@
+import type { APIRoute } from 'astro'
+import { siteConfig } from '../site.config'
+
+const robotsTxt = `
+User-agent: *
+Allow: /
+
+HOST: ${siteConfig.site}
+
+# Sitemaps
+Sitemap: ${siteConfig.site}/sitemap.xml
+
+# Disallow system and error pages
+Disallow: /api/
+Disallow: /_astro/
+Disallow: /404
+Disallow: /500
+`
+
+export const GET: APIRoute = () => {
+  return new Response(robotsTxt.trim(), {
+    headers: {
+      'Content-Type': 'text/plain; charset=utf-8',
+    },
+  })
+}

src/site.config.ts

@@ -20,7 +20,8 @@ export const siteConfig: SiteConfig = {
 			month: 'short',
 			year: 'numeric'
 		}
-	}
+	},
+	site: 'https://blog.kazoottt.top',
 }
 
 export const menuLinks: Array<{ title: string; path: string }> = [

src/types.ts

@@ -8,6 +8,7 @@ export type SiteConfig = {
 		locale: string | string[] | undefined
 		options: Intl.DateTimeFormatOptions
 	}
+	site: string
 }
 
 export type PaginationLink = {