# Robots.txt for visartiden.com - MAXIMUM SEO & CRAWLING OPTIMIZATION
# Last updated: 2025-06-01
# Website: https://visartiden.com - Exakt Tid Världen Över
# POLICY: FULL OPEN ACCESS - ALL CONTENT CRAWLABLE

# =================================================================
# UNIVERSAL ACCESS - ALL BOTS WELCOME
# =================================================================

# Allow ALL bots to crawl EVERYTHING
User-agent: *
Allow: /
Crawl-delay: 0

# =================================================================
# PREMIUM SEARCH ENGINE BOTS - OPTIMIZED ACCESS
# =================================================================

# Google Bot (Primary search engine) - Maximum priority
User-agent: Googlebot
Allow: /
Crawl-delay: 0
Request-rate: 1/1s

# Google Mobile Bot (Critical for mobile-first indexing)
User-agent: Googlebot-Mobile
Allow: /
Crawl-delay: 0
Request-rate: 1/1s

# Google Image Bot (For enhanced image SEO)
User-agent: Googlebot-Image
Allow: /
Allow: /images/
Allow: /assets/
Allow: /media/
Crawl-delay: 0

# Google News Bot (For time-related news content)
User-agent: Googlebot-News
Allow: /
Crawl-delay: 0

# Google Video Bot
User-agent: Googlebot-Video
Allow: /
Crawl-delay: 0

# Bing Bot (Microsoft search) - High priority
User-agent: Bingbot
Allow: /
Crawl-delay: 0
Request-rate: 1/1s

# Yahoo Bot - Full access
User-agent: Slurp
Allow: /
Crawl-delay: 0

# Yandex Bot (Popular in Europe/Russia)
User-agent: YandexBot
Allow: /
Crawl-delay: 0

# DuckDuckGo Bot
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 0

# Baidu Bot (Chinese search engine)
User-agent: Baiduspider
Allow: /
Crawl-delay: 0

# =================================================================
# AI & LANGUAGE MODEL BOTS - FULL ACCESS FOR MAXIMUM VISIBILITY
# =================================================================

# OpenAI GPT Bots
User-agent: GPTBot
Allow: /
Crawl-delay: 0

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 0

# Google AI Extended
User-agent: Google-Extended
Allow: /
Crawl-delay: 0

# Anthropic Claude
User-agent: anthropic-ai
Allow: /
Crawl-delay: 0

User-agent: Claude-Web
Allow: /
Crawl-delay: 0

# Common Crawl (Used by many AI systems)
User-agent: CCBot
Allow: /
Crawl-delay: 0

# =================================================================
# SEO & ANALYSIS TOOLS - MAXIMUM ACCESS
# =================================================================

# Ahrefs Bot (SEO analysis)
User-agent: AhrefsBot
Allow: /
Crawl-delay: 0

# SEMrush Bot
User-agent: SemrushBot
Allow: /
Crawl-delay: 0

User-agent: SemrushBot-SA
Allow: /
Crawl-delay: 0

# Majestic SEO
User-agent: MJ12bot
Allow: /
Crawl-delay: 0

# Moz Bot
User-agent: dotbot
Allow: /
Crawl-delay: 0

User-agent: rogerbot
Allow: /
Crawl-delay: 0

# Screaming Frog SEO Spider
User-agent: Screaming Frog SEO Spider
Allow: /
Crawl-delay: 0

# =================================================================
# SOCIAL MEDIA BOTS - ENHANCED SHARING & VISIBILITY
# =================================================================

# Facebook
User-agent: facebookexternalhit
Allow: /
Crawl-delay: 0

User-agent: facebookexternalhit/1.1
Allow: /
Crawl-delay: 0

# Twitter/X
User-agent: Twitterbot
Allow: /
Crawl-delay: 0

# LinkedIn
User-agent: LinkedInBot
Allow: /
Crawl-delay: 0

# WhatsApp
User-agent: WhatsApp
Allow: /
Crawl-delay: 0

# Telegram
User-agent: TelegramBot
Allow: /
Crawl-delay: 0

# Pinterest
User-agent: Pinterest
Allow: /
Crawl-delay: 0

# =================================================================
# ARCHIVE & RESEARCH BOTS
# =================================================================

# Internet Archive (Wayback Machine)
User-agent: archive.org_bot
Allow: /
Crawl-delay: 0

User-agent: ia_archiver
Allow: /
Crawl-delay: 0

# =================================================================
# E-COMMERCE & COMPARISON BOTS
# =================================================================

# Price comparison and shopping bots
User-agent: Shopzilla
Allow: /
Crawl-delay: 0

User-agent: PriceGrabber
Allow: /
Crawl-delay: 0

# =================================================================
# INTERNATIONAL & REGIONAL SEARCH ENGINES
# =================================================================

# Naver (Korean search engine)
User-agent: Naverbot
Allow: /
Crawl-delay: 0

# Sogou (Chinese search engine)
User-agent: Sogou
Allow: /
Crawl-delay: 0

# Seznam (Czech search engine)
User-agent: SeznamBot
Allow: /
Crawl-delay: 0

# =================================================================
# ACCESSIBILITY & COMPLIANCE BOTS
# =================================================================

# Accessibility checking bots
User-agent: WAVE
Allow: /
Crawl-delay: 0

# =================================================================
# SITEMAP DECLARATIONS - MAXIMUM DISCOVERABILITY
# =================================================================

# Primary XML sitemap
Sitemap: https://visartiden.com/sitemap.xml

# Additional sitemaps for future expansion
Sitemap: https://visartiden.com/sitemap-index.xml
Sitemap: https://visartiden.com/sitemap-images.xml
Sitemap: https://visartiden.com/sitemap-mobile.xml
Sitemap: https://visartiden.com/sitemap-news.xml

# =================================================================
# HOST & CANONICAL DECLARATION
# =================================================================

# Specify canonical host for maximum SEO benefit
Host: visartiden.com

# =================================================================
# SPECIAL DIRECTIVES FOR MAXIMUM CRAWLING
# =================================================================

# Allow all file types
Allow: /*.css
Allow: /*.js
Allow: /*.json
Allow: /*.xml
Allow: /*.txt
Allow: /*.ico
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.gif
Allow: /*.svg
Allow: /*.webp
Allow: /*.pdf
Allow: /*.doc
Allow: /*.docx

# Allow all directories
Allow: /images/
Allow: /css/
Allow: /js/
Allow: /assets/
Allow: /media/
Allow: /fonts/
Allow: /api/
Allow: /admin/
Allow: /backend/
Allow: /uploads/
Allow: /downloads/
Allow: /files/

# =================================================================
# CRAWLING OPTIMIZATION NOTES:
# =================================================================
# ✅ Zero crawl delays for maximum speed
# ✅ All bots and AI systems welcome
# ✅ All file types and directories accessible
# ✅ Multiple sitemap references
# ✅ Enhanced for international SEO
# ✅ Optimized for social media sharing
# ✅ Future-proofed for expansion