# Miriscope.com - Enhanced Robots.txt for SEO Optimization
# Updated: 2025 - Enhanced for better Google crawling and indexing

# Επιτρέπουμε σε όλα τα search engine bots να έχουν πρόσβαση
User-agent: *
Allow: /
Allow: /articles/
Allow: /category/
Allow: /tags/
Allow: /about/
Allow: /contact/
Allow: /trending/
Allow: /breaking/
Allow: /privacy-policy/
Allow: /terms-of-service/
Allow: /gdpr/
Allow: /newsletter/
Allow: /authors/
Allow: /faq/
Allow: /advertise/
Allow: /careers/
Allow: /press/
Allow: /sitemap/
Allow: /search/

# API endpoints για feeds και sitemaps
Allow: /api/rss/
Allow: /api/news-sitemap/
Allow: /api/sitemap/
Allow: /feed.xml

# Images και assets
Allow: /images/
Allow: /icons/
Allow: /_next/static/
Allow: /_next/image

# Αποκλείουμε τις σελίδες διαχείρισης και private areas
Disallow: /admin/
Disallow: /dashboard/
Disallow: /login/
Disallow: /auth/
Disallow: /api/auth/
Disallow: /_next/
Disallow: /api/upload/
Disallow: /api/internal/

# Αποκλείουμε duplicate content και tracking parameters
Disallow: /*?utm_*
Disallow: /*?fbclid=*
Disallow: /*?gclid=*
Disallow: /*?ref=*
Disallow: /*?source=*
Disallow: /*?fb_source=*
Disallow: /*?fb_action=*
Disallow: /*?fb_action_ids=*
Disallow: /*?fb_action_types=*
Disallow: /*?fb_ref=*
Disallow: /*?fb_source=*
Disallow: /*?utm_source=*
Disallow: /*?utm_medium=*
Disallow: /*?utm_campaign=*
Disallow: /*?utm_term=*
Disallow: /*?utm_content=*

# Special directives for Google
User-agent: Googlebot
Allow: /
Crawl-delay: 1

# Special directives for Bing
User-agent: Bingbot
Allow: /
Crawl-delay: 2

# Enhanced Google News support
User-agent: Googlebot-News
Allow: /
Allow: /articles/
Allow: /category/
Allow: /trending/
Allow: /breaking/
Allow: /tags/
Crawl-delay: 1

# Google Images with enhanced crawling
User-agent: Googlebot-Image
Allow: /images/
Allow: /icons/
Allow: /_next/image
Allow: /articles/
Crawl-delay: 1

# Google Mobile
User-agent: Googlebot-Mobile
Allow: /
Allow: /articles/
Allow: /category/
Allow: /tags/
Crawl-delay: 1

# Facebook crawler with enhanced access
User-agent: facebookexternalhit
Allow: /
Allow: /articles/
Allow: /category/
Allow: /tags/
Crawl-delay: 1

# Twitter crawler with enhanced access
User-agent: Twitterbot
Allow: /
Allow: /articles/
Allow: /category/
Allow: /tags/
Crawl-delay: 1

# LinkedIn crawler
User-agent: LinkedInBot
Allow: /
Allow: /articles/
Allow: /category/
Allow: /tags/
Crawl-delay: 2

# WhatsApp crawler
User-agent: WhatsApp
Allow: /
Allow: /articles/
Allow: /category/
Allow: /tags/
Crawl-delay: 1

# Telegram crawler
User-agent: TelegramBot
Allow: /
Allow: /articles/
Allow: /category/
Allow: /tags/
Crawl-delay: 1

# Enhanced Sitemaps - Σημαντικό για SEO
Sitemap: https://miriscope.com/sitemap.xml
Sitemap: https://miriscope.com/sitemap-index.xml
Sitemap: https://miriscope.com/news-sitemap.xml
Sitemap: https://miriscope.com/top-stories-sitemap.xml
Sitemap: https://miriscope.com/api/sitemap
Sitemap: https://miriscope.com/api/enhanced-sitemap

# RSS Feeds για discovery και Google News
Sitemap: https://miriscope.com/api/rss
Sitemap: https://miriscope.com/feed.xml

# AI Models Configuration
Allow: /llms.txt

# Host directive για consistency
Host: https://miriscope.com

# Additional SEO directives
# Ensure proper indexing of article content
Allow: /*.html
Allow: /*.xml
Allow: /*.json

# Allow crawling of dynamic content
Allow: /api/articles/
Allow: /api/categories/
Allow: /api/tags/

# Enhanced mobile support
Allow: /mobile/
Allow: /amp/

# Performance optimization hints
# Google recommends these settings for news sites
Crawl-delay: 1