lobe-chat

1
import { MetadataRoute } from 'next';
2

3
import { sitemapModule } from '@/server/sitemap';
4
import { getCanonicalUrl } from '@/server/utils/url';
5

6
const robots = (): MetadataRoute.Robots => {
7
  return {
8
    host: getCanonicalUrl(),
9
    rules: [
10
      {
11
        allow: ['/discover/*'],
12
        disallow: ['/discover/search/*'],
13
        userAgent: ['Facebot', 'facebookexternalhit'],
14
      },
15
      {
16
        allow: ['/discover/*'],
17
        disallow: ['/discover/search/*'],
18
        userAgent: 'LinkedInBot',
19
      },
20
      {
21
        allow: ['/discover/*'],
22
        disallow: ['/discover/search/*'],
23
        userAgent: 'Twitterbot',
24
      },
25
      {
26
        allow: ['/'],
27
        disallow: ['/api/*', '/login', '/signup', '/files', '/repos/*', '/discover/search/*'],
28
        userAgent: '*',
29
      },
30
    ],
31
    sitemap: sitemapModule.getRobots(),
32
  };
33
};
34

35
export default robots;
36