# robots.txt for TakeMarket Ltd (https://takemarket.co.uk/)
# Last updated: October 11, 2025
# Allows search engines to crawl all site pages, including career, TM University, and TM People portals
# Focuses on finance & B2B tech sales/lead gen for Nordics (Sweden, Denmark, Norway, Finland), UK, Western Europe, with international reach

# Notice: Unauthorized crawling or scraping of TakeMarket Ltd's network is strictly prohibited unless expressed written permission is obtained from TakeMarket Ltd. Violations may result in legal action. For detailed terms and conditions, refer to our Automated Data Collection and Bot Scraping Policy at: https://takemarket.co.uk/automated-data-collection-bot-scraping-policy/

# Default rule: Allow all bots to crawl unless specified
User-agent: *

# WP Defender - Begin
Disallow: /a8572c7463506576
# WP Defender - End

Disallow: /wp-admin/
Disallow: /?s=  # Block search result pages
Disallow: /feed/  # Block feed pages
Disallow: /comments/feed/  # Block comment feeds
Disallow: /trackback/  # Block trackbacks
Disallow: /xmlrpc.php  # Block XML-RPC for security
Allow: /  # Allow all other pages, including /en/, /sv/, /da/, /career/, /tmuniversity/, /tmpeople/

# Specific rules for major search engine bots
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Mobile
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Mediapartners-Google
Allow: /

User-agent: Adsbot-Google
Allow: /

User-agent: YandexBot
Allow: /  # Allow Yandex to crawl for international finance/B2B tech visibility

User-agent: Bingbot
Allow: /  # Allow Bing for UK, Sweden, Denmark, and global SEO

User-agent: Bravebot
Allow: /  # Allow Brave Search for privacy-focused users

User-agent: Baiduspider
Allow: /  # Allow Baidu for potential Asia-Pacific expansion (optional, monitor usage)

User-agent: DuckDuckBot
Allow: /  # Allow DuckDuckGo for privacy-focused international users

User-agent: Slurp
Allow: /

User-agent: msnbot
Allow: /

# AI-powered search bots - ALLOW for visibility in AI search results
User-agent: ChatGPT-User
Allow: /  # Allow ChatGPT search to surface TM when users ask about B2B services

User-agent: PerplexityBot
Allow: /  # Allow Perplexity AI search for B2B lead gen queries in Nordics/Europe

User-agent: YouBot
Allow: /  # Allow You.com AI search

# AI training/scraping bots - BLOCK to prevent unauthorized bulk data collection
User-agent: GPTBot
Disallow: /  # Block OpenAI training crawler

User-agent: Google-Extended
Disallow: /  # Block Google AI training (separate from standard Googlebot)

User-agent: CCBot
Disallow: /  # Block Common Crawl for AI training datasets

User-agent: anthropic-ai
Disallow: /  # Block Anthropic training crawler

User-agent: ClaudeBot
Disallow: /  # Block Claude training crawler

User-agent: FacebookBot
Disallow: /  # Block Meta AI training

User-agent: cohere-ai
Disallow: /  # Block Cohere AI training

User-agent: Omgilibot
Disallow: /  # Block Omgili scraper

User-agent: Bytespider
Disallow: /  # Block TikTok/ByteDance crawler

User-agent: Diffbot
Disallow: /  # Block Diffbot scraper

User-agent: ImagesiftBot
Disallow: /  # Block ImageSift scraper

# Sitemap location for efficient crawling of all pages
Sitemap: https://takemarket.co.uk/sitemap.xml

# Additional notes for crawlers
# Language-specific paths (/en/, /sv/, /da/) are allowed for geotargeting UK, Sweden, Denmark
# Career, TM University, and TM People portals are crawlable for potential future SEO
# Yandex, Bing, Baidu, and DuckDuckGo included for international reach in finance/B2B tech
# Bravebot added for Brave Search visibility
# AI search bots (ChatGPT-User, PerplexityBot, YouBot) allowed for visibility when users search for B2B services
# AI training bots (GPTBot, CCBot, ClaudeBot, etc.) blocked to prevent unauthorized bulk scraping per our data collection policy
# Bingbot powers both Bing search and Microsoft Copilot

Sitemap: https://takemarket.co.uk/sitemap.xml

Sitemap: https://takemarket.co.uk/sitemap.xml