Sitemap: https://ny1.com/sitemap.xml

User-agent: *

Crawl-delay: 1




# Allowed Paths
Allow: /$ #
Allow: /nyc/* #
Allow: /sitemap.xml #
Allow: /services/* #
Allow: /etc/* #
Allow: /content/* #
Allow: /.well-known/assetlinks.json #



# Excluded Pages
Disallow: /nyc/all-boroughs/app-headlines.html #
Disallow: queens.ny1.com.html #




# Excluded Paths
Disallow: /* #
Disallow: /ny/nyc #
Disallow: /ny/nyc/* #
Disallow: /nyc/all-boroughs/app-headlines #
Disallow: /nyc/all-boroughs/app-headlines* #
Disallow: /*/*/partner-content/* #
Disallow: /911-videos/* #
Disallow: /content/news/stories/* #


# Additional Config
User-Agent: Twitterbot
Disallow: /.well-known/

User-agent: GPTBot 
Disallow: / 

User-agent: anthropic-ai 
Disallow: / 

User-agent: PetalBot 
Disallow: / 

User-agent: Amazonbot 
Disallow: / 

User-agent: ChatGPT 
Disallow: / 

User-agent: Bytespider 
Disallow: / 

User-agent: ClaudeBot 
Disallow: / 

User-agent: PerplexityBot 
Disallow: / 

User-agent: meta-externalagent 
Disallow: / 

User-agent: chatgpt
Disallow: / 

User-agent: CCBot
Disallow: / 

User-agent: Yeti
Disallow: / 

User-agent: Diffbot
Disallow: / 

User-agent: Timpibot
Disallow: / 

User-agent: Ai2Bot
Disallow: / 

User-agent: MistralAI-User
Disallow: / 

User-agent: AnthropicBot
Disallow: / 

User-agent: claude-web
Disallow: / 

User-agent: cohere-ai
Disallow: / 

User-agent: omgili 
Disallow: / 

User-agent: Google-Extended
Disallow: / 

User-agent: Applebot-Extended
Disallow: /