# SerraFi Robots.txt # Last updated: 2024 User-agent: * # AI crawlers share the same rules as everyone else User-agent: GPTBot User-agent: ChatGPT-User User-agent: OAI-SearchBot User-agent: ClaudeBot User-agent: Claude-User User-agent: Claude-SearchBot User-agent: anthropic-ai User-agent: Google-Extended User-agent: PerplexityBot User-agent: Perplexity-User User-agent: CCBot User-agent: Applebot-Extended User-agent: Bytespider User-agent: Meta-ExternalAgent User-agent: Amazonbot User-agent: cohere-ai User-agent: Diffbot # Allow indexing of main pages Allow: / Allow: /index.html Allow: /about-us-new.html Allow: /contact-us.html Allow: /blog.html # Allow product pages Allow: /product/cards.html Allow: /product/cashback.html Allow: /product/payments.html Allow: /product/accounts.html # Allow solution pages Allow: /solutions/healthcare-medical.html Allow: /solutions/logistics-freight.html Allow: /solutions/wholesale-distribution.html Allow: /solutions/auto-sales-service.html Allow: /solutions/nonprofits.html Allow: /solutions/contractors-builders.html # Disallow legal/policy pages (marked with noindex) Disallow: /privacy.html Disallow: /terms.html Disallow: /disclaimers.html # Disallow pages not meant for search (marked with noindex) Disallow: /changelog.html # Disallow partner-only pages Disallow: /partners/ # Disallow template and utility pages Disallow: /template/ Disallow: /components/ Disallow: /test-* Disallow: /validate_optimizations.html ## removed: css-test-homepage.html no longer exists # Disallow backup files Disallow: *.backup Disallow: *.bak # Disallow error pages Disallow: /404.html Disallow: /401.html # Disallow development files Disallow: /*.md Disallow: /.* Disallow: /.git/ Disallow: /.github/ # Allow access to assets Allow: /css/ Allow: /js/ Allow: /images/ Allow: /fonts/ Allow: /animations/ # Block sensitive files Disallow: /.htaccess Disallow: /.gitignore Disallow: /package.json Disallow: /package-lock.json # Crawl delay (optional) Crawl-delay: 1 # Future sitemap reference (uncomment when sitemap is created) # Sitemap: https://serrafi.com/sitemap.xml