Free tools

Robots.txt Examples

Robots.txt file content for fiverr.com.

Robot.txt file for: fiverr.com

      # See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
#
User-Agent: *
Disallow: /orders/timeline/*
Disallow: /s/
Disallow: */pinned_flashes/*
Disallow: /gigs/*/share/
Disallow: /gigs/*/share?*
Disallow: /specials/*
Disallow: /packages/*
Disallow: /categories/silly
Disallow: /categories/fifa
Disallow: /categories/Halloween
Disallow: /categories/Postcards
Disallow: /purchases
Disallow: /user_sessions
Disallow: /users/
Disallow: /counter/*?
Disallow: /collaborate/*
Disallow: /search/
Disallow: /search_results/gigs/*
Disallow: /match/website/*
Disallow: /pages/website-developer-match
Disallow: /v4/*
Disallow: /pro/*
Disallow: /about-pro
Disallow: /pro-resources
Disallow: /pro-solutions
Disallow: /gigs/search
Disallow: /recommendations/
Disallow: /contact_me/
Disallow: /conversations/
Disallow: /bookmarks/
Disallow: /inbox/
Disallow: /inbox$
Disallow: /seller_onboarding/
Disallow: /checkout/package/
Disallow: /match/
Disallow: /pages/website-developer-match
Disallow: /cdn-proxy/px/*
Disallow: /cdn-proxy/pim/*
Disallow: /search_results/
Disallow: /custom_orders/
Disallow: /studios/
Disallow: /v1/stats
Disallow: /v1/browser-performance
Disallow: /assets/shared/*
Allow: /pro/about
# Blocking Pagination - tag pages only
Disallow: /gigs/*page=
Allow: /gigs/*page=2$
# Blocking Gig 2 Gig duplications
Disallow: /*context_referrer=gig_page
# Blocking Seller 2 Gig duplications
Disallow: /*?*source=user_page*
Disallow: /*context_referrer=user_page
# Blocking activity api
Disallow: /api/v1/activities

Disallow: /logo-maker/brief/
Disallow: /logo-maker/choose-variation/
Disallow: /logo-maker/wordpress
Disallow: /logo-maker/woo
Sitemap: https://www.fiverr.com/sitemap_gigs.xml.gz
Sitemap: https://www.fiverr.com/sitemap_users.xml.gz
Sitemap: https://www.fiverr.com/sitemap_categories.xml.gz
Sitemap: https://www.fiverr.com/sitemap_tags.xml.gz
Sitemap: https://www.fiverr.com/sitemap_logomaker_tags.xml.gz
# Blocking AdsBots
User-agent: AdsBot-Google
Disallow: /api/v1/activities
Disallow: /recommendations/