User-Agent: *
Disallow: /go
Disallow: /admin
Disallow: /test
Disallow: /loadMore
Disallow: /blackhole
Disallow: /*page=
Disallow: /*dynamic=
Disallow: /*/random
Disallow: /*/pickone
Disallow: /*/privates
Disallow: /*/tiktok
Disallow: /*/live
Disallow: /*/watch-history
Disallow: /*/favorites
# Search engines are not rate-limited: Googlebot, Yandex, Baiduspider, bingbot,
# DuckDuckBot, Applebot and any other crawler that uses this group.
Host: https://dadcams.com

# Non-search crawlers may fetch every path. Path rules live only in User-agent: *.
# Training crawlers that ignore Crawl-delay are closed. The rest are slowed.

# GPTBot does not document Crawl-delay, and a delay of 86400 did not slow the live crawl.
User-agent: GPTBot
Disallow: /

# Training opt-out. Does not affect Googlebot or Applebot search.
User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

# Amazonbot ignores Crawl-delay. https://developer.amazon.com/amazonbot
User-agent: Amazonbot
Disallow: /

User-agent: PetalBot
Disallow: /

# ClaudeBot honors Crawl-delay: 5 seconds between requests. Search agents are not in this group.
User-agent: ClaudeBot
User-agent: anthropic-ai
Crawl-delay: 5

# Every path, at most 1 request/sec.
# PerplexityBot does not document Crawl-delay. They say robots.txt changes apply within about 24h.
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: PerplexityBot
Crawl-delay: 1

# Semrush honors Crawl-delay up to 10 seconds. https://www.semrush.com/bot/
User-agent: SemrushBot
User-agent: SiteAuditBot
User-agent: SemrushBot-BA
User-agent: SemrushBot-SI
User-agent: SemrushBot-SWA
Crawl-delay: 10

# Every path, at most 1 request/sec. Ahrefs honors Crawl-delay.
User-agent: AhrefsBot
User-agent: AhrefsSiteAudit
User-agent: DotBot
User-agent: BLEXBot
User-agent: serpstatbot
User-agent: MegaIndex
User-agent: DataForSeoBot
User-agent: Bytespider
User-agent: facebookexternalhit
User-agent: meta-externalagent
User-agent: FacebookBot
Crawl-delay: 1

User-Agent: MJ12bot
Crawl-Delay: 100

User-agent: barkrowler
Crawl-Delay: 100

# CCBot obeys Crawl-delay (seconds between requests). https://commoncrawl.org/faq
User-agent: CCBot
Crawl-delay: 1

# AIWebIndex honors Crawl-delay. Their own floor is already 2 seconds.
# https://lyrenth.com/bot
User-agent: AIWebIndex
Crawl-delay: 1