fix(docs): address GSC crawl and metadata issues

Co-authored-by: Junior Garcia <jrgarciadev@gmail.com>

apps/docs/next-redirects.ts

@@ -142,6 +142,13 @@ export async function getRedirects(): Promise<Redirect[]> {
     source: "/:path*",
   });
 
+  // The default-locale homepage is canonical at the apex, without `/en`.
+  redirects.push({
+    destination: "/",
+    permanent: true,
+    source: "/en",
+  });
+
   // Theme builder redirect - redirect /theme to /themes
   redirects.push({
     destination: "/themes",
@@ -266,6 +273,14 @@ export async function getRedirects(): Promise<Redirect[]> {
         destination: "/docs/react/getting-started/design-principles",
         source: "/docs/design-principles",
       },
+      {
+        destination: "/docs/react/getting-started/colors",
+        source: "/docs/customization/colors",
+      },
+      {
+        destination: "/docs/react/getting-started/colors",
+        source: "/docs/react/customization/colors",
+      },
     ]),
   );
 
@@ -320,6 +335,14 @@ export async function getRedirects(): Promise<Redirect[]> {
         destination: "/docs/react/components/number-field",
         source: "/docs/react/components/numberfield",
       },
+      {
+        destination: "/docs/react/components/tag-group",
+        source: "/docs/react/components/taggroup",
+      },
+      {
+        destination: "/docs/react/components/tag-group",
+        source: "/docs/components/taggroup",
+      },
     ]),
   );
 

apps/docs/next.config.ts

@@ -16,6 +16,17 @@ const withMDX = createMDX();
 // silently de-index the production site.
 const appEnv = process.env["NEXT_PUBLIC_APP_ENV"];
 const isIndexable = appEnv !== "preview" && appEnv !== "development";
+const llmsFileNames = ["llms.txt", "llms-full.txt", "llms-components.txt", "llms-patterns.txt"];
+const machineOnlyPaths = [
+  "/:path*.mdx",
+  "/llms.mdx/:path*",
+  "/llms-raw.mdx/:path*",
+  ...llmsFileNames.map((fileName) => `/${fileName}`),
+  ...["react", "native"].flatMap((platform) =>
+    llmsFileNames.map((fileName) => `/${platform}/${fileName}`),
+  ),
+];
+const noindexHeaders = [{key: "X-Robots-Tag", value: "noindex, nofollow"}];
 
 const config: NextConfig = {
   compress: true,
@@ -33,6 +44,10 @@ const config: NextConfig = {
   },
   async headers() {
     return [
+      ...machineOnlyPaths.map((source) => ({
+        headers: noindexHeaders,
+        source,
+      })),
       ...(!isIndexable
         ? [
             {

apps/docs/src/app/[lang]/docs/[[...slug]]/page.tsx

@@ -38,6 +38,28 @@ import {
 // import { getGithubLastEdit } from "fumadocs-core/server";
 
 const componentStatusIcons = ["preview", "new", "updated"];
+const ENGLISH_SEO_OVERRIDES: Record<string, {description: string; title: string}> = {
+  "/docs/react/components": {
+    description:
+      "Browse accessible HeroUI React components for forms, overlays, navigation, data display, and more, built with React Aria and Tailwind CSS v4.",
+    title: "HeroUI React Components – Accessible UI Library",
+  },
+  "/docs/react/components/button": {
+    description:
+      "Build accessible React buttons with HeroUI. Explore variants, sizes, loading and disabled states, icon buttons, events, and ButtonGroup patterns.",
+    title: "HeroUI Button – Accessible React Button Component",
+  },
+  "/docs/react/components/select": {
+    description:
+      "Build accessible React select menus with HeroUI. Learn options, sections, validation, disabled states, controlled values, and customizable listboxes.",
+    title: "HeroUI Select – Accessible React Select Component",
+  },
+  "/docs/react/getting-started": {
+    description:
+      "Get started with HeroUI v3, an accessible React UI library built on React Aria and Tailwind CSS v4. Learn installation, principles, and customization.",
+    title: "HeroUI v3 – React UI Library Getting Started",
+  },
+};
 
 const getRawMDXContent = cache(async (pagePath: string): Promise<string> => {
   try {
@@ -195,23 +217,29 @@ export async function generateMetadata(props: {
   // `page.url` already carries the locale prefix (`/en/docs/...`), which is the
   // URL actually served — the unprefixed `/docs/...` form permanently redirects.
   const url = page.url;
-  const alternates = getLocalizedAlternates({locale: params.lang, path: stripLocale(url)});
+  const unlocalizedPath = stripLocale(url);
+  const alternates = getLocalizedAlternates({locale: params.lang, path: unlocalizedPath});
+  const seoOverride = params.lang === "en" ? ENGLISH_SEO_OVERRIDES[unlocalizedPath] : undefined;
+  const description = seoOverride?.description ?? page.data.description;
+  const title = seoOverride?.title ?? page.data.title;
 
   return {
     alternates,
-    description: page.data.description,
+    description,
     openGraph: {
-      description: page.data.description,
+      description,
       images: imageUrl,
       siteName: siteConfig.name,
-      title: page.data.title,
+      title,
       url,
     },
-    title: page.data.title,
+    title: seoOverride ? {absolute: title} : title,
     twitter: {
       card: "summary_large_image",
+      description,
       images: imageUrl,
       site: "@hero_ui",
+      title,
     },
   };
 }

apps/docs/src/app/[lang]/themes/page.tsx

@@ -44,7 +44,14 @@ export async function generateMetadata({
       type: "website",
       url: absoluteUrl(alternates.canonical),
     },
-    title,
+    title: {absolute: title},
+    twitter: {
+      card: "summary_large_image",
+      description,
+      images: siteConfig.ogImage,
+      site: "@hero_ui",
+      title,
+    },
   };
 }
 

apps/docs/src/app/robots.txt/route.ts

@@ -9,16 +9,26 @@ export const revalidate = false;
  * never drift from the deployed canonical host.
  */
 const CONTENT_SIGNAL = "Content-Signal: ai-train=yes, search=yes, ai-input=yes";
+const GOOGLEBOT_DISALLOWS = [
+  "/*.mdx$",
+  "/llms*.txt$",
+  "/react/llms*.txt$",
+  "/native/llms*.txt$",
+  "/*/node_modules/*",
+  "/*/src/*.d.ts$",
+];
 
 export const GET = () => {
   const body = [
-    "# Every crawlable route is open to every crawler. Non-indexable endpoints",
-    "# (preview deployments, machine-only routes) opt out via X-Robots-Tag or",
-    "# page metadata instead, so nothing indexable is ever blocked here.",
+    "# Public HTML pages are crawlable. Machine-readable docs stay available to",
+    "# humans and agents, but are excluded from Google Search.",
     "User-agent: *",
     "Allow: /",
     CONTENT_SIGNAL,
     "",
+    "User-agent: Googlebot",
+    ...GOOGLEBOT_DISALLOWS.map((path) => `Disallow: ${path}`),
+    "",
     `Sitemap: ${absoluteUrl("/sitemap.xml")}`,
     "",
   ].join("\n");

apps/docs/src/app/sitemap.ts

@@ -73,13 +73,9 @@ export default function sitemap(): MetadataRoute.Sitemap {
     }
   }
 
-  // Served outside `app/[lang]`, so it has no locale variants.
-  const unlocalized: SitemapEntry[] = [
-    {
-      lastModified: generatedAt,
-      url: absoluteUrl("/docs/native-showcase/privacy-policy"),
-    },
-  ];
+  const localizedEntries = toEntries(paths, generatedAt);
 
-  return [...toEntries(paths, generatedAt), ...unlocalized];
+  // Keep one canonical entry per URL. Machine-readable `.mdx`/`llms` routes
+  // and the unprefixed native-showcase fallback are intentionally excluded.
+  return [...new Map(localizedEntries.map((entry) => [entry.url, entry])).values()];
 }

apps/docs/src/lib/dictionaries/cn.json

@@ -117,8 +117,8 @@
     "androidComingSoon": "Android · 即将推出"
   },
   "themes": {
-    "metaTitle": "主题",
-    "metaDescription": "通过可视化控件自定义 HeroUI 的颜色、字体与圆角，并复制可直接用于生产环境的 CSS 变量。",
+    "metaTitle": "HeroUI 主题编辑器 – 自定义 React UI",
+    "metaDescription": "通过可视化控件自定义 HeroUI React 主题，调整颜色、字体与圆角，实时预览组件并复制可用于生产环境的 Tailwind CSS v4 变量。",
     "theme": "主题",
     "custom": "自定义",
     "designTheme": "设计主题",

apps/docs/src/lib/dictionaries/en.json

@@ -117,8 +117,8 @@
     "androidComingSoon": "Android · Coming soon"
   },
   "themes": {
-    "metaTitle": "Themes",
-    "metaDescription": "Design and customize a HeroUI theme with visual controls for colors, typography, and radius, then copy production-ready CSS variables.",
+    "metaTitle": "HeroUI Theme Builder – Customize React UI",
+    "metaDescription": "Customize HeroUI React themes visually. Tune colors, typography, and radius, preview components live, and copy production-ready Tailwind CSS v4 variables.",
     "theme": "Theme",
     "custom": "Custom",
     "designTheme": "Design theme",

apps/docs/src/lib/llms-utils.ts

@@ -9,6 +9,7 @@ export type ContentType = "all" | "components" | "patterns";
 export const LLMS_TEXT_HEADERS = {
   "Cache-Control": "public, s-maxage=3600, stale-while-revalidate=86400",
   "Content-Type": "text/plain; charset=utf-8",
+  "X-Robots-Tag": "noindex, nofollow",
 } as const;
 
 export function getPlatformFromPage(page: Page): Platform {

apps/docs/src/proxy.ts

@@ -29,6 +29,8 @@ const MARKDOWN_EXCLUDED_PATHS = new Set([
 ]);
 
 const PUBLIC_FILE_PATTERN = /\.[a-z0-9]+$/i;
+const UNRESOLVED_TEMPLATE_PATTERN = /(?:\{[^/{}]+\}|%7B[^/]+%7D)/i;
+const PERMANENT_LOCALIZED_PREFIXES = ["/blog", "/docs", "/showcase", "/themes"];
 
 // Routes that live outside `app/[lang]` and must never receive a locale prefix.
 const LOCALE_REDIRECT_EXCLUDED_PREFIXES = [
@@ -71,6 +73,12 @@ function shouldSkipMarkdownRewrite(pathname: string): boolean {
   return PUBLIC_FILE_PATTERN.test(pathname) && !pathname.startsWith("/docs/");
 }
 
+function shouldPermanentlyRedirect(pathname: string): boolean {
+  return PERMANENT_LOCALIZED_PREFIXES.some(
+    (prefix) => pathname === prefix || pathname.startsWith(`${prefix}/`),
+  );
+}
+
 function addHomepageDiscoveryHeaders(response: NextResponse, pathname: string): NextResponse {
   if (isHomepage(pathname)) {
     response.headers.set("Link", getHomepageLinkHeader());
@@ -82,6 +90,17 @@ function addHomepageDiscoveryHeaders(response: NextResponse, pathname: string):
 export function proxy(request: NextRequest) {
   const {pathname} = request.nextUrl;
 
+  if (UNRESOLVED_TEMPLATE_PATTERN.test(pathname)) {
+    return new NextResponse("Not Found", {
+      headers: {
+        "Cache-Control": "public, max-age=0, s-maxage=3600",
+        "Content-Type": "text/plain; charset=utf-8",
+        "X-Robots-Tag": "noindex, nofollow",
+      },
+      status: 404,
+    });
+  }
+
   // Markdown handler runs first so agent requests are served regardless of locale.
   if (
     (request.method === "GET" || request.method === "HEAD") &&
@@ -124,7 +143,7 @@ export function proxy(request: NextRequest) {
 
     url.pathname = `/${DEFAULT_LOCALE}${pathname}`;
 
-    return NextResponse.redirect(url, pathname.startsWith("/docs/") ? 308 : 307);
+    return NextResponse.redirect(url, shouldPermanentlyRedirect(pathname) ? 308 : 307);
   }
 
   return addHomepageDiscoveryHeaders(NextResponse.next(), pathname);

skills/heroui-native/SKILL.md

@@ -76,7 +76,8 @@ node scripts/get_docs.mjs /docs/native/getting-started/theming
 
 ### Direct MDX URLs
 
-Component docs: `https://heroui.com/docs/native/components/{component-name}.mdx`
+Use a concrete kebab-case component slug in every MDX request. Never request an unresolved
+template URL.
 
 Examples:
 
@@ -84,7 +85,8 @@ Examples:
 - Dialog: `https://heroui.com/docs/native/components/dialog.mdx`
 - TextField: `https://heroui.com/docs/native/components/text-field.mdx`
 
-Getting started guides: `https://heroui.com/docs/native/getting-started/{topic}.mdx`
+For getting-started guides, use a concrete topic URL such as
+`https://heroui.com/docs/native/getting-started/quick-start.mdx`.
 
 **Important:** Always fetch component docs before implementing. The MDX docs include complete examples, props, anatomy, and API references.
 

skills/heroui-react/SKILL.md

@@ -98,7 +98,8 @@ node scripts/get_docs.mjs /docs/react/getting-started/theming
 
 ### Direct MDX URLs
 
-Component docs: `https://heroui.com/docs/react/components/{component-name}.mdx`
+Use a concrete kebab-case component slug in every MDX request. Never request an unresolved
+template URL.
 
 Examples:
 
@@ -106,7 +107,8 @@ Examples:
 - Modal: `https://heroui.com/docs/react/components/modal.mdx`
 - Form: `https://heroui.com/docs/react/components/form.mdx`
 
-Getting started guides: `https://heroui.com/docs/react/getting-started/{topic}.mdx`
+For getting-started guides, use a concrete topic URL such as
+`https://heroui.com/docs/react/getting-started/quick-start.mdx`.
 
 **Important:** Always fetch component docs before implementing. The MDX docs include complete examples, props, anatomy, and API references.