User-agent: *

# Prevent crawling of marketing campaign parameter URLs to reduce duplicate indexing
Disallow: /*utm_

# Prevent duplicate crawling/indexing caused by Google Ads click IDs
Disallow: /*gclid=

# Prevent duplicate crawling/indexing caused by Facebook click tracking parameters
Disallow: /*fbclid=

# Block internal/private content paths from being crawled
Disallow: /private/


# Prevent crawling of internal repository/resources area
Disallow: /repository/

# Prevent crawling of internal framework or development-related resources
Disallow: /mast-framework/

# Qualified conversational marketing crawler
User-agent: QualifiedBot
Allow: /

# OpenAI crawler used for model training/content discovery
User-agent: GPTBot
Allow: /

# OpenAI retrieval crawler used when users interact with ChatGPT
User-agent: ChatGPT-User
Allow: /

# Anthropic AI crawler
User-agent: ClaudeBot
Allow: /

# Perplexity AI crawler
User-agent: PerplexityBot
Allow: /

# Google AI systems control (Gemini/Vertex AI related usage)
User-agent: Google-Extended
Allow: /

User-agent: rogerbot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: BadBot
Disallow: /

User-agent: EvilScraper
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: WebBandit
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: Web Downloader
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Offline Explorer Pro
Disallow: /

User-agent: HTTrack Website Copier
Disallow: /

User-agent: Offline Commander
Disallow: /

User-agent: Leech
Disallow: /

User-agent: WebSnake
Disallow: /

User-agent: BlackWidow
Disallow: /

User-agent: HTTP Weazel
Disallow: /

Sitemap: https://www.netradyne.com/sitemap.xml