diff --git a/public/robots.txt b/public/robots.txt deleted file mode 100644 index cd76b06..0000000 --- a/public/robots.txt +++ /dev/null @@ -1,22 +0,0 @@ -User-agent: * -Allow: / - - -HOST: https://blog.kazoottt.top -# Sitemaps -Sitemap: https://blog.kazoottt.top/sitemap-index.xml - -# Common crawl optimizations -Disallow: /api/ -Disallow: /_astro/ -Disallow: /404 -Disallow: /500 - -# Allow crawling of content -Allow: /posts/ -Allow: /categories/ -Allow: /tags/ -Allow: /blog/ -Allow: /friends/ -Allow: /blog/ -Allow: /tools/ diff --git a/src/pages/robots.txt.ts b/src/pages/robots.txt.ts new file mode 100644 index 0000000..68eb08d --- /dev/null +++ b/src/pages/robots.txt.ts @@ -0,0 +1,26 @@ +import type { APIRoute } from 'astro' +import { siteConfig } from '../site.config' + +const robotsTxt = ` +User-agent: * +Allow: / + +HOST: ${siteConfig.site} + +# Sitemaps +Sitemap: ${siteConfig.site}/sitemap.xml + +# Disallow system and error pages +Disallow: /api/ +Disallow: /_astro/ +Disallow: /404 +Disallow: /500 +` + +export const GET: APIRoute = () => { + return new Response(robotsTxt.trim(), { + headers: { + 'Content-Type': 'text/plain; charset=utf-8', + }, + }) +} diff --git a/src/site.config.ts b/src/site.config.ts index 67a4e01..5ed35b0 100644 --- a/src/site.config.ts +++ b/src/site.config.ts @@ -20,7 +20,8 @@ export const siteConfig: SiteConfig = { month: 'short', year: 'numeric' } - } + }, + site: 'https://blog.kazoottt.top', } export const menuLinks: Array<{ title: string; path: string }> = [ diff --git a/src/types.ts b/src/types.ts index d48cc51..29eed6d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -8,6 +8,7 @@ export type SiteConfig = { locale: string | string[] | undefined options: Intl.DateTimeFormatOptions } + site: string } export type PaginationLink = {