Free tools

Robots.txt Examples

Robots.txt file content for bbcgoodfood.com.

Robot.txt file for: bbcgoodfood.com

      User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /4817/
Disallow: /176986657/
Disallow: /styleguide/
Disallow: /author/mrben/
Disallow: /members/
Disallow: /account/
Disallow: /signin/
Disallow: /account/
Disallow: /auth/
Disallow: /search/recipes/?q=
Disallow: /search/recipes/?sort=
Disallow: /search/recipes/page/
Disallow: /search/node
Disallow: /api/
Disallow: /article/welcome-email
Disallow: /article/thank-you-registering
Disallow: /article/thank-you-signing-health-edit
Disallow: *null?
Disallow: *obOrigUrl=true
Disallow: /seasonal-calendar/all
Disallow: /Search?searchstring_input=
Disallow: /v1/
Disallow: /v5/
Disallow: *wp-sitemap-taxonomies-post_tag
Disallow: /user/login
Disallow: /search
Disallow: /search-results/
Disallow: /user/*/collections/
Disallow: /user/*/collection/
Disallow: /user/register
Disallow: /wp-sitemap-posts-

Disallow: /list-v2/
Disallow: /article-v2/
Disallow: /search-results/

Disallow: /registration
Disallow: /registration/
Disallow: /forgotten-password
Disallow: /forgotten-password/
Disallow: /change-password/
Disallow: /reset-password
Disallow: /reset-password/
Disallow: /login
Disallow: /login/
Disallow: /log-out
Disallow: /log-out/

# GPTBot
User-agent: GPTBot
Disallow: /

#Member Sitemap
Sitemap: https://www.bbcgoodfood.com/membersitemap.xml

# News Sitemap
Sitemap: https://www.bbcgoodfood.com/google-news.xml/


# Sitemap archive
Sitemap: https://www.bbcgoodfood.com/sitemap.xml