<?php

/**
 * Site Audit Engine - Real Site Scanner
 *
 * @package ProRank\SEO\Core\Audits
 */

declare(strict_types=1);

namespace ProRank\SEO\Core\Audits;

use WP_Error;
use Exception;

if ( ! defined( 'ABSPATH' ) ) {
    exit;
}


/**
 * Main audit engine for crawling and analyzing sites
 */


class SiteAuditEngine {
    
    /**
     * Audit ID
     */
    private string $audit_id;
    
    /**
     * Database tables
     */
    private string $audits_table;
    private string $audit_urls_table;
    private string $audit_issues_table;
    
    /**
     * Constructor
     */
    public function __construct() {
        global $wpdb;
        $this->audits_table = $wpdb->prefix . 'prorank_audits';
        $this->audit_urls_table = $wpdb->prefix . 'prorank_audit_urls';
        $this->audit_issues_table = $wpdb->prefix . 'prorank_audit_issues';
        
        $this->maybe_create_tables();
        $this->register_cron_hooks();
    }

    /**
     * Register cron callbacks used by the local crawler pipeline.
     */
    private function register_cron_hooks(): void {
        add_action('prorank_run_site_crawler', [$this, 'run_crawler'], 10, 1);
        add_action('prorank_run_immediate_crawler', [$this, 'run_crawler'], 10, 1);
        add_action('prorank_schedule_url_checks', [$this, 'schedule_url_checks'], 10, 1);
        add_action('prorank_check_audit_url', [$this, 'check_url'], 10, 3);
    }
    
    /**
     * Start a new audit
     */
    public function start_audit(array $options = []): array {
        global $wpdb;
        
        try {
            // Generate audit ID
            $this->audit_id = 'audit_' . uniqid();
            
            prorank_log('ProRank SEO: Starting audit with ID: ' . $this->audit_id);
            
            // Insert audit record
            $result = $wpdb->insert($this->audits_table, [
                'audit_id' => $this->audit_id,
                'status' => 'crawling',
                'started_at' => current_time('mysql'),
                'options' => json_encode($options),
            ]);
            
            if ($result === false) {
                prorank_log('ProRank SEO: Failed to insert audit record. Error: ' . $wpdb->last_error);
                throw new Exception('Failed to create audit record: ' . $wpdb->last_error);
            }

            // Run site-level checks (once per audit, not per-page)
            $this->run_site_level_checks($this->audit_id, $options);

            // Schedule the crawler once and let cron/process recovery pick it up.
            wp_schedule_single_event(time(), 'prorank_run_site_crawler', [$this->audit_id]);
            
            prorank_log('ProRank SEO: Audit scheduled successfully');
            
            // Trigger cron immediately for faster response
            spawn_cron();
            
            // Never run the crawler inline during REST/AJAX requests.
            // Inline execution can hold /audit/start open for a long time and freeze the UI.
            // The crawler is already scheduled above and cron is triggered via spawn_cron().
            
            return [
                'audit_id' => $this->audit_id,
                'status' => 'crawling',
                'message' => __('Site audit started. Crawling your site...', 'prorank-seo'),
            ];
        } catch (Exception $e) {
            prorank_log('ProRank SEO: Exception in start_audit: ' . $e->getMessage());
            throw $e;
        }
    }
    
    /**
     * Run site-level checks (once per audit, not per-page)
     */
    private function run_site_level_checks(string $audit_id, array $options): void {
        $check_types = $options['check_types'] ?? [];
        $default_checks = [
            'xml_sitemap' => true,
            'wordpress_health' => true,
            'plugin_conflicts' => true,
            'database_optimization' => true,
        ];
        $enabled_checks = array_merge($default_checks, $check_types);

        global $wpdb;
        $site_url = $this->get_primary_site_url();

        // Insert site-level URL entry
        $wpdb->insert($this->audit_urls_table, [
            'audit_id' => $audit_id,
            'url' => $site_url . '?_site_level_checks',
            'status' => 'pending',
            'depth' => 0,
            'created_at' => current_time('mysql'),
        ]);
        $site_url_id = $wpdb->insert_id;

        $issues = [];
        $passed_checks = [];
        $warnings = [];

        if (!empty($enabled_checks['xml_sitemap'])) {
            $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_xml_sitemap());
        }
        if (!empty($enabled_checks['wordpress_health'])) {
            $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_wordpress_health());
        }
        if (!empty($enabled_checks['plugin_conflicts'])) {
            $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_plugin_conflicts());
        }
        if (!empty($enabled_checks['database_optimization'])) {
            $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_database_optimization());
        }

        $current_time = current_time('mysql');
        foreach (array_merge($issues, $warnings) as $finding) {
            $wpdb->insert($this->audit_issues_table, [
                'audit_id' => $audit_id,
                'url_id' => $site_url_id,
                'type' => $finding['type'],
                'severity' => $finding['severity'],
                'message' => $finding['message'],
                'data' => json_encode($finding['data'] ?? []),
                'created_at' => $current_time,
            ]);
        }
        foreach ($passed_checks as $check) {
            $wpdb->insert($this->audit_issues_table, [
                'audit_id' => $audit_id,
                'url_id' => $site_url_id,
                'type' => $check,
                'severity' => 'passed',
                'message' => 'Check passed',
                'data' => '{}',
                'created_at' => $current_time,
            ]);
        }

        $wpdb->update($this->audit_urls_table, [
            'status' => 'checked',
            'issues_count' => count($issues),
            'warnings_count' => count($warnings),
            'passed_count' => count($passed_checks),
            'checked_at' => current_time('mysql'),
        ], ['id' => $site_url_id]);
    }

    /**
     * Get configured site URLs without relying on request context.
     */
    private function get_configured_site_urls(): array {
        $candidates = [];
        $seen = [];
        $values = [
            get_option('home'),
            get_option('siteurl'),
            home_url('/', 'https'),
            site_url('/', 'https'),
            home_url('/'),
            site_url('/'),
        ];

        foreach ($values as $value) {
            $url = trim((string) $value);
            if ($url === '') {
                continue;
            }

            $parts = wp_parse_url($url);
            if (!is_array($parts)) {
                continue;
            }

            $scheme = strtolower((string) ($parts['scheme'] ?? ''));
            $host = strtolower((string) ($parts['host'] ?? ''));
            if ($host === '' || !in_array($scheme, ['http', 'https'], true)) {
                continue;
            }

            $normalized = $scheme . '://' . $host;
            if (!empty($parts['port'])) {
                $normalized .= ':' . (int) $parts['port'];
            }

            $path = trim((string) ($parts['path'] ?? ''), '/');
            if ($path !== '') {
                $normalized .= '/' . $path;
            }

            if (isset($seen[$normalized])) {
                continue;
            }

            $seen[$normalized] = true;
            $candidates[] = $normalized;
        }

        if (empty($candidates)) {
            $fallback = trim((string) home_url('/'));
            if ($fallback !== '') {
                $candidates[] = rtrim($fallback, '/');
            }
        }

        return $candidates;
    }

    /**
     * Get the primary configured site URL for site-level findings.
     */
    private function get_primary_site_url(): string {
        $urls = $this->get_configured_site_urls();
        if (!empty($urls)) {
            return $urls[0];
        }

        return rtrim((string) home_url('/'), '/');
    }

    /**
     * Run the site crawler
     */
    public function run_crawler(string $audit_id): void {
        global $wpdb;

        $this->audit_id = $audit_id;
        $seed_lock_key = 'prorank_audit_seed_lock_' . md5($audit_id);

        try {
            // Get audit options
            $audit = $wpdb->get_row($wpdb->prepare(
                "SELECT * FROM {$this->audits_table} WHERE audit_id = %s",
                $audit_id
            ));

            $options = [];
            if ($audit && !empty($audit->options)) {
                $options = json_decode($audit->options, true) ?: [];
            }

            // Get settings from options or defaults
            $settings = $options['settings'] ?? [];
            $max_urls = $settings['max_pages'] ?? 100;

            // Prevent duplicate crawler runs from seeding the same audit more than once.
            $existing_seed_count = $this->count_distinct_audit_urls($audit_id, false);
            if ($existing_seed_count > 0) {
                $wpdb->update($this->audits_table, [
                    'total_urls' => $existing_seed_count,
                    'status' => 'checking',
                ], ['audit_id' => $audit_id]);

                $this->schedule_url_checks($audit_id);

                if (!wp_next_scheduled('prorank_schedule_url_checks', [$audit_id])) {
                    wp_schedule_single_event(time() + 5, 'prorank_schedule_url_checks', [$audit_id]);
                }

                spawn_cron();
                return;
            }

            if (get_transient($seed_lock_key)) {
                return;
            }

            set_transient($seed_lock_key, 1, MINUTE_IN_SECONDS);
            
            // Update status
            $wpdb->update($this->audits_table, 
                ['status' => 'crawling'],
                ['audit_id' => $audit_id]
            );
            
            try {
                // Get all URLs to crawl with limit
                $urls = $this->get_site_urls($max_urls, $settings);
                
                // Batch insert URLs for better performance
                $current_time = current_time('mysql');
                if (!empty($urls)) {
                    $placeholders = implode(',', array_fill(0, count($urls), '(%s, %s, %s, %s)'));
                    $values = [];
                    foreach ($urls as $url) {
                        $values[] = $audit_id;
                        $values[] = $url;
                        $values[] = 'pending';
                        $values[] = $current_time;
                    }
                    $sql = "INSERT INTO {$this->audit_urls_table} 
                        (audit_id, url, status, created_at) 
                        VALUES {$placeholders}";
                    $wpdb->query($wpdb->prepare($sql, ...$values)); // phpcs:ignore WordPress.DB.PreparedSQL.NotPrepared, PluginCheck.Security.DirectDB.UnescapedDBParameter
                }
                
                // Update audit with URL count
                $wpdb->update($this->audits_table, [
                    'total_urls' => count($urls),
                    'status' => 'checking',
                ], ['audit_id' => $audit_id]);
                
                // Schedule URL checks
                $this->schedule_url_checks($audit_id);
                
                // Also schedule continuous processing
                if (!wp_next_scheduled('prorank_schedule_url_checks', [$audit_id])) {
                    wp_schedule_single_event(time() + 5, 'prorank_schedule_url_checks', [$audit_id]);
                }
                
                // Force cron to run
                spawn_cron();
            } finally {
                delete_transient($seed_lock_key);
            }
            
        } catch (Exception $e) {
            delete_transient($seed_lock_key);
            $wpdb->update($this->audits_table, [
                'status' => 'failed',
                'error' => $e->getMessage(),
            ], ['audit_id' => $audit_id]);
        }
    }
    
    /**
     * Get all site URLs
     */
    private function get_site_urls(int $max_urls = 100, array $settings = []): array {
        $urls = [];
        
        // Get homepage
        $urls[] = home_url('/');
        prorank_log('ProRank SEO: Homepage URL: ' . home_url('/'));
        
        // Get all published posts and pages
        $post_types = get_post_types(['public' => true], 'names');
        prorank_log('ProRank SEO: Post types found: ' . implode(', ', $post_types));
        
        // Remove non-content post types
        unset($post_types['attachment']);
        unset($post_types['elementor_library']);
        unset($post_types['e-landing-page']);
        unset($post_types['revision']);
        unset($post_types['nav_menu_item']);
        unset($post_types['wp_block']);
        unset($post_types['wp_template']);
        unset($post_types['wp_template_part']);
        unset($post_types['wp_navigation']);
        unset($post_types['oembed_cache']);
        unset($post_types['custom_css']);
        unset($post_types['customize_changeset']);
        unset($post_types['wp_global_styles']);
        unset($post_types['wp_font_family']);
        unset($post_types['wp_font_face']);

        foreach (array_keys($post_types) as $post_type) {
            if (strpos($post_type, 'tcb_') === 0 || strpos($post_type, 'tve_') === 0 || strpos($post_type, 'thrive_') === 0) {
                unset($post_types[$post_type]);
            }
        }
        
        // Query posts more reliably
        $args = [
            'post_type' => array_values($post_types),
            'post_status' => 'publish',
            'posts_per_page' => max(1, $max_urls - 1), // Limit based on settings, -1 for homepage
            'orderby' => 'modified',
            'order' => 'DESC',
            'fields' => 'ids',
            'no_found_rows' => true,
            'update_post_meta_cache' => false,
            'update_post_term_cache' => false,
        ];
        
        $query = new \WP_Query($args);
        $post_ids = $query->posts;
        
        prorank_log('ProRank SEO: Found ' . count($post_ids) . ' posts/pages');
        prorank_log('ProRank SEO: Query args: ' . json_encode($args));
        
        if (empty($post_ids)) {
            // Fallback: Try querying each post type separately
            prorank_log('ProRank SEO: No posts found with combined query, trying individually...');
            foreach ($post_types as $post_type) {
                $type_query = new \WP_Query([
                    'post_type' => $post_type,
                    'post_status' => 'publish',
                    'posts_per_page' => -1,
                    'fields' => 'ids',
                ]);
                prorank_log("ProRank SEO: Found {$type_query->post_count} posts for type: $post_type");
                $post_ids = array_merge($post_ids, $type_query->posts);
            }
        }
        
        foreach ($post_ids as $post_id) {
            $permalink = get_permalink($post_id);
            if ($permalink && !in_array($permalink, $urls)) {
                $urls[] = $permalink;
            } else if (!$permalink) {
                // Log failed permalink generation
                prorank_log("ProRank SEO: Failed to get permalink for post ID: $post_id");
                
                // Try alternative method
                $post = get_post($post_id);
                if ($post) {
                    $post_type_obj = get_post_type_object($post->post_type);
                    if ($post_type_obj && $post_type_obj->public) {
                        // Try to construct URL manually as fallback
                        $url = home_url('/?p=' . $post_id);
                        if (!in_array($url, $urls)) {
                            $urls[] = $url;
                            prorank_log("ProRank SEO: Using fallback URL for post ID $post_id: $url");
                        }
                    }
                }
            }
        }
        
        // Get archive pages
        $archive = get_post_type_archive_link('post');
        if ($archive && !in_array($archive, $urls)) {
            $urls[] = $archive;
        }
        
        // Get category pages (limit to prevent too many URLs)
        $categories = get_categories(['number' => 20]);
        foreach ($categories as $category) {
            $link = get_category_link($category->term_id);
            if ($link && !in_array($link, $urls)) {
                $urls[] = $link;
            }
        }
        
        // Get tag pages (limit to prevent too many URLs)
        $tags = get_tags(['number' => 20]);
        foreach ($tags as $tag) {
            $link = get_tag_link($tag->term_id);
            if ($link && !in_array($link, $urls)) {
                $urls[] = $link;
            }
        }
        
        // Remove any false or empty values
        $urls = array_filter($urls);

        // Filter out non-content URLs (Elementor library, revisions, query-param pages, etc.)
        $urls = array_filter($urls, [$this, 'should_audit_url']);

        // Normalize all URLs consistently (strip trailing slash for deduplication)
        $normalized = [];
        foreach ($urls as $url) {
            $norm = rtrim($url, '/');
            if (!isset($normalized[$norm])) {
                $normalized[$norm] = $url; // Keep original form
            }
        }
        $urls = array_values($normalized);

        // Respect max_urls limit
        if (count($urls) > $max_urls) {
            $urls = array_slice($urls, 0, $max_urls);
        }

        prorank_log('ProRank SEO: Total URLs to audit: ' . count($urls));
        prorank_log('ProRank SEO: URLs: ' . implode(', ', array_slice($urls, 0, 5)) . '...');

        return $urls;
    }
    
    /**
     * Filter out URLs that should not be audited (non-content, internal, template URLs).
     */
    private function should_audit_url(string $url): bool {
        $parsed = wp_parse_url($url);
        $path = $parsed['path'] ?? '/';
        $query = $parsed['query'] ?? '';
        $query_args = [];

        if ($query !== '') {
            wp_parse_str($query, $query_args);
        }

        // Skip Elementor library/template URLs
        if (strpos($query, 'elementor_library=') !== false || strpos($query, 'elementor-preview=') !== false) {
            return false;
        }

        // Skip WordPress admin/login/cron URLs
        if (preg_match('#^/(wp-admin|wp-login|wp-cron|wp-json|xmlrpc)#', $path)) {
            return false;
        }

        // Skip feed URLs
        if (preg_match('#/feed/?$#', $path)) {
            return false;
        }

        if (!empty($query_args)) {
            foreach (array_keys($query_args) as $key) {
                $normalized_key = strtolower((string) $key);

                if ($normalized_key === '_site_level_checks') {
                    continue;
                }

                if (
                    strpos($normalized_key, 'utm_') === 0
                    || in_array($normalized_key, [
                        'attachment_id',
                        'draft',
                        'elementor-preview',
                        'elementor_library',
                        'fbclid',
                        'gclid',
                        'mc_cid',
                        'mc_eid',
                        'preview',
                        'replytocom',
                        'srsltid',
                    ], true)
                ) {
                    return false;
                }
            }

            // Builder and utility query URLs are not canonical site content.
            if (!empty($query_args['post_type']) || !empty($query_args['tcbf']) || !empty($query_args['tve'])) {
                return false;
            }

            // On pretty-permalink sites, query-string variants are non-canonical duplicates.
            if ((string) get_option('permalink_structure') !== '') {
                return false;
            }
        }

        // Skip preview/draft URLs
        if (strpos($query, 'preview=') !== false || strpos($query, 'draft=') !== false) {
            return false;
        }

        return true;
    }

    /**
     * Schedule URL checks
     */
    public function schedule_url_checks(string $audit_id): void {
        global $wpdb;
        
        // Check if audit is still active and get settings
        $audit = $wpdb->get_row($wpdb->prepare(
            "SELECT status, options FROM {$this->audits_table} WHERE audit_id = %s",
            $audit_id
        ));
        
        if (!$audit || !in_array($audit->status, ['checking', 'crawling'])) {
            return; // Audit was stopped or completed
        }
        
        // Get settings from audit options
        $options = [];
        $settings = [];
        if (!empty($audit->options)) {
            $options = json_decode($audit->options, true) ?: [];
            $settings = $options['settings'] ?? [];
        }
        
        // Calculate batch size based on crawl speed
        $crawl_speed = $settings['crawl_speed'] ?? 10; // URLs per minute
        $concurrent_requests = $settings['concurrent_requests'] ?? 5; // Increase default concurrent requests
        // Process more URLs per batch for better performance
        $batch_size = max(10, min($crawl_speed, 50)); // Increased batch size for faster processing
        
        $urls = $wpdb->get_results($wpdb->prepare(
            "SELECT id, url FROM {$this->audit_urls_table} 
             WHERE audit_id = %s AND status = 'pending'
             LIMIT %d",
            $audit_id,
            intval($batch_size)
        ));

        if (!empty($urls)) {
            $claimed_ids = $this->claim_audit_url_batch($audit_id, wp_list_pluck($urls, 'id'));
            if (empty($claimed_ids)) {
                return;
            }

            $urls = array_values(array_filter($urls, static function ($url_data) use ($claimed_ids) {
                return in_array((int) ($url_data->id ?? 0), $claimed_ids, true);
            }));
        }
        
        prorank_log('ProRank SEO: Processing batch of ' . count($urls) . ' URLs for audit ' . $audit_id);

        // In REST/AJAX/CLI contexts, execute checks inline to avoid WP-Cron dependency
        // (many hosts disable or delay cron, which leaves local audits stuck at "checking").
        $run_inline = (defined('REST_REQUEST') && REST_REQUEST)
            || (defined('DOING_AJAX') && DOING_AJAX)
            || (defined('WP_CLI') && WP_CLI);

        if ($run_inline) {
            $remaining_before = count($urls);
            foreach ($urls as $url_data) {
                try {
                    $this->check_url($audit_id, (int) $url_data->id, (string) $url_data->url);
                } catch (\Throwable $e) {
                    prorank_log('ProRank SEO: Inline URL check failed: ' . $e->getMessage());
                }
            }

            $remaining = (int) $wpdb->get_var($wpdb->prepare(
                "SELECT COUNT(*) FROM {$this->audit_urls_table} 
                 WHERE audit_id = %s AND status = 'pending'",
                $audit_id
            ));
            $in_progress = (int) $wpdb->get_var($wpdb->prepare(
                "SELECT COUNT(*) FROM {$this->audit_urls_table}
                 WHERE audit_id = %s AND status = 'checking'",
                $audit_id
            ));

            if ($remaining > 0) {
                if ($remaining >= $remaining_before) {
                    // No progress in this pass (likely transient fetch failures) – avoid recursive loops.
                    wp_schedule_single_event(time() + 2, 'prorank_schedule_url_checks', [$audit_id]);
                    return;
                }
                $this->schedule_url_checks($audit_id);
            } elseif ($in_progress === 0) {
                $this->complete_audit($audit_id);
            }
            return;
        }
        
        // Process more URLs immediately for faster feedback
        $immediate_checks = array_slice($urls, 0, max(5, $concurrent_requests));
        
        foreach ($immediate_checks as $index => $url_data) {
            // Schedule immediate check via WP-Cron for parallel processing
            wp_schedule_single_event(
                time() + $index, // Stagger by 1 second each
                'prorank_check_audit_url',
                [$audit_id, (int)$url_data->id, $url_data->url]
            );
        }
        
        // Schedule remaining URLs with faster timing
        $scheduled_urls = array_slice($urls, $concurrent_requests);
        if (count($scheduled_urls) > 0) {
            // Much faster URL processing - minimal delays for speed
            $delay_between_urls = max(0.2, 10 / $crawl_speed); // Much faster processing
            $delay = $concurrent_requests; // Start after immediate checks
            
            foreach ($scheduled_urls as $url_data) {
                wp_schedule_single_event(
                    time() + intval($delay),
                    'prorank_check_audit_url',
                    [$audit_id, (int)$url_data->id, $url_data->url]
                );
                $delay += $delay_between_urls;
            }
        }
        
        // Check if more URLs to process
        $remaining = $wpdb->get_var($wpdb->prepare(
            "SELECT COUNT(*) FROM {$this->audit_urls_table} 
             WHERE audit_id = %s AND status = 'pending'",
            $audit_id
        ));
        $in_progress = (int) $wpdb->get_var($wpdb->prepare(
            "SELECT COUNT(*) FROM {$this->audit_urls_table}
             WHERE audit_id = %s AND status = 'checking'",
            $audit_id
        ));
        
        if ($remaining > 0) {
            // Schedule next batch much more quickly for better performance
            // Process batches with minimal delay
            $next_batch_delay = min(2, max(1, 5 / $crawl_speed)); // Much faster batch processing
            
            wp_schedule_single_event(
                time() + intval($next_batch_delay),
                'prorank_schedule_url_checks',
                [$audit_id]
            );
        } elseif ($in_progress === 0) {
            // All URLs processed, complete audit
            $this->complete_audit($audit_id);
        }
    }
    
    /**
     * Check a single URL
     */
    public function check_url(string $audit_id, int $url_id, string $url): void {
        global $wpdb;

        $current_status = (string) $wpdb->get_var($wpdb->prepare(
            "SELECT status FROM {$this->audit_urls_table} WHERE id = %d",
            $url_id
        ));

        if ($current_status === 'checked') {
            return;
        }

        if ($current_status !== 'checking') {
            $wpdb->update($this->audit_urls_table, ['status' => 'checking'], ['id' => $url_id]);
        }
        
        // Get audit with options
        $audit = $wpdb->get_row($wpdb->prepare(
            "SELECT started_at, status, options FROM {$this->audits_table} WHERE audit_id = %s",
            $audit_id
        ));
        
        // Get settings from audit options
        $options = [];
        $settings = [];
        $check_types = [];
        if ($audit && !empty($audit->options)) {
            $options = json_decode($audit->options, true) ?: [];
            $settings = $options['settings'] ?? [];
            $check_types = $options['check_types'] ?? [];
        }

        // Build enabled checks from settings
        $default_checks = [
            'meta_tags' => true,
            'headings_structure' => true,
            'robots_txt' => true,
            'open_graph' => true,
            'twitter_cards' => true,
            'canonical_tags' => true,
            'open_graph' => true,
            'schema_validation' => true,
            'image_optimization' => true,
            'https_status' => true,
            'broken_links' => true,
        ];
        $enabled_checks = array_merge($default_checks, $check_types);

        // Get timeout and max duration from settings
        $request_timeout = $settings['request_timeout'] ?? 2; // Reduced default timeout
        $max_duration = ($settings['max_duration'] ?? 30) * 60; // Convert to seconds
        
        if ($audit && $audit->started_at) {
            $elapsed = time() - strtotime($audit->started_at);
            if ($elapsed > $max_duration) {
                prorank_log("ProRank SEO: Audit {$audit_id} exceeded maximum time ({$max_duration}s), auto-completing");
                $this->complete_audit($audit_id);
                return;
            }
        }
        
        // Check if audit was stopped
        if ($audit && $audit->status === 'stopped') {
            return;
        }
        
        $issues = [];
        $passed_checks = [];
        $warnings = [];
        
        // Respect robots.txt before fetching
        if ($this->is_disallowed_by_robots($url)) {
            $this->add_issue($audit_id, $url, 'disallowed_by_robots', 'URL blocked by robots.txt', 'warning');
            $this->mark_url_checked($url_id);
            return;
        }

        // Rate-limit per host to avoid hammering
        $this->rate_limit_host($url);

        // Enhanced fetch with more data
        $start_time = microtime(true);
        $response = wp_remote_get($url, [
            'timeout' => max(4, $request_timeout), // Ensure a reasonable timeout
            'redirection' => 5,
            'user-agent' => 'ProRank SEO Audit Bot/1.1',
            'sslverify' => false, // Avoid false failures when auditing own site (self-signed, internal certs)
            'headers' => [
                'Accept' => 'text/html,application/xhtml+xml',
                'Accept-Encoding' => 'gzip, deflate, br',
            ],
            'httpversion' => '1.1',
            'compress' => true,
            'reject_unsafe_urls' => false, // Allow auditing own site even with unusual URL patterns
        ]);
        $load_time = microtime(true) - $start_time;
        
        if (is_wp_error($response)) {
            $issues[] = [
                'type' => 'connectivity',
                'severity' => 'critical',
                'message' => 'Failed to fetch URL: ' . $response->get_error_message(),
                'data' => ['error_code' => $response->get_error_code()],
            ];
        } else {
            $status_code = wp_remote_retrieve_response_code($response);
            $body = wp_remote_retrieve_body($response);
            $headers = wp_remote_retrieve_headers($response);
            $response_size = strlen($body);
            
            // Check status code

            // Optionally fetch Core Web Vitals for a small subset of URLs
            $this->maybe_record_core_web_vitals($audit_id, $url, $headers);
            $this->check_structured_data_enhanced($audit_id, $url, $body);
            if ($status_code >= 400) {
                $issues[] = [
                    'type' => 'http_error',
                    'severity' => 'critical',
                    'message' => sprintf('HTTP %d error', $status_code),
                    'data' => ['status_code' => $status_code],
                ];
            } elseif ($status_code >= 300) {
                $warnings[] = [
                    'type' => 'redirect',
                    'severity' => 'medium',
                    'message' => sprintf('Page redirects with HTTP %d', $status_code),
                    'data' => ['status_code' => $status_code, 'location' => $headers['location'] ?? ''],
                ];
            } else {
                $passed_checks[] = 'http_status';
            }
            
            // Parse HTML
            if (!empty($body) && $status_code < 400) {
                $dom = new \DOMDocument();
                libxml_use_internal_errors(true);
                @$dom->loadHTML(mb_convert_encoding($body, 'HTML-ENTITIES', 'UTF-8'));
                libxml_clear_errors();

                // Core SEO checks - conditionally run based on enabled_checks
                if (!empty($enabled_checks['meta_tags'])) {
                    $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_title($dom));
                    $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_meta_description($dom));
                }

                if (!empty($enabled_checks['headings_structure'])) {
                    $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_headings($dom));
                }

                if (!empty($enabled_checks['image_optimization'])) {
                    $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_images($dom, $url));
                }

                // Technical SEO checks
                if (!empty($enabled_checks['canonical_tags'])) {
                    $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_canonical($dom, $url));
                }

                if (!empty($enabled_checks['robots_txt'])) {
                    $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_robots_meta($dom));
                }

                // Social and structured data checks
                if (!empty($enabled_checks['open_graph']) || !empty($enabled_checks['twitter_cards'])) {
                    $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_open_graph($dom));
                }

                if (!empty($enabled_checks['schema_validation'])) {
                    $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_schema_markup($dom));
                }

                // Security check (HTTPS)
                if (!empty($enabled_checks['https_status']) || !empty($enabled_checks['ssl_certificate'])) {
                    if (strpos($url, 'https://') === 0) {
                        $passed_checks[] = 'https_enabled';
                    } else {
                        $issues[] = [
                            'type' => 'missing_https',
                            'severity' => 'high',
                            'message' => 'Page is not using HTTPS',
                            'data' => []
                        ];
                    }
                }

                // Content quality check
                if (!empty($enabled_checks['content_quality'])) {
                    $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_content_quality($dom, $body));
                }

                // Link checks (broken links, internal/external linking)
                if (!empty($enabled_checks['broken_links']) || !empty($enabled_checks['internal_linking']) || !empty($enabled_checks['external_links'])) {
                    $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_links($dom, $url));
                }

                // Page speed check
                if (!empty($enabled_checks['page_speed'])) {
                    $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_page_speed($url, $headers, $load_time, $response_size));
                }

                // Core Web Vitals check
                if (!empty($enabled_checks['core_web_vitals'])) {
                    $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_core_web_vitals($url));
                }

                // Mobile optimization check
                if (!empty($enabled_checks['mobile_friendly'])) {
                    $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_mobile_optimization($dom));
                }

                // Accessibility check
                if (!empty($enabled_checks['accessibility'])) {
                    $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_accessibility($dom));
                }

                // Mixed content check
                if (!empty($enabled_checks['mixed_content'])) {
                    $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_mixed_content($dom, $url));
                }

                // Security headers check (uses direct insert pattern)
                if (!empty($enabled_checks['security_headers'])) {
                    $this->check_security_headers($audit_id, $url, $headers);
                }

                // Redirect chains check (uses direct insert pattern)
                if (!empty($enabled_checks['redirect_chains'])) {
                    $this->check_redirect_chain($audit_id, $url);
                }

                // Thin content check
                if (!empty($enabled_checks['thin_content'])) {
                    $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_thin_content($dom, $body));
                }

                // Duplicate content check
                if (!empty($enabled_checks['duplicate_content'])) {
                    $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_duplicate_content($audit_id, $url, $dom, $body));
                }

                // Crawl depth check
                if (!empty($enabled_checks['crawl_depth'])) {
                    $this->merge_check_results($issues, $passed_checks, $warnings, $this->check_crawl_depth($audit_id, $url_id));
                }

                // Store internal links for orphaned pages detection
                if (!empty($enabled_checks['orphaned_pages'])) {
                    $this->store_internal_links_for_orphan_detection($audit_id, $dom, $url);
                }
            }
        }

        // Batch insert all findings for better performance
        $all_findings = array_merge($issues, $warnings);
        $insert_values = [];
        $current_time = current_time('mysql');
        
        $rows = count($all_findings) + count($passed_checks);
        if ($rows > 0) {
            $placeholders = implode(',', array_fill(0, $rows, '(%s, %d, %s, %s, %s, %s, %s)'));
            $values = [];

            // Prepare all findings for batch insert
            foreach ($all_findings as $finding) {
                $values[] = $audit_id;
                $values[] = $url_id;
                $values[] = $finding['type'];
                $values[] = $finding['severity'];
                $values[] = $finding['message'];
                $values[] = wp_json_encode($finding['data'] ?? []);
                $values[] = $current_time;
            }
            
            // Add passed checks
            foreach ($passed_checks as $check) {
                $values[] = $audit_id;
                $values[] = $url_id;
                $values[] = $check;
                $values[] = 'passed';
                $values[] = 'Check passed successfully';
                $values[] = '{}';
                $values[] = $current_time;
            }
            
            // Batch insert all issues at once
            $sql = "INSERT INTO {$this->audit_issues_table} 
                (audit_id, url_id, type, severity, message, data, created_at) 
                VALUES {$placeholders}";
            $wpdb->query($wpdb->prepare($sql, ...$values)); // phpcs:ignore WordPress.DB.PreparedSQL.NotPrepared, PluginCheck.Security.DirectDB.UnescapedDBParameter
        }
        
        // Calculate page score
        $page_score = $this->calculate_page_score($issues, $warnings, $passed_checks);
        
        // Update URL status with enhanced data
        $wpdb->update($this->audit_urls_table, [
            'status' => 'checked',
            'issues_count' => count($issues),
            'warnings_count' => count($warnings),
            'passed_count' => count($passed_checks),
            'score' => $page_score,
            'checked_at' => current_time('mysql'),
        ], ['id' => $url_id]);

        // Recursive link discovery with max_depth enforcement
        if (isset($dom) && isset($body) && !empty($body)) {
            $max_depth = $settings['max_depth'] ?? 5;
            $max_urls = $settings['max_pages'] ?? ($settings['max_urls'] ?? 500);
            $this->discover_and_queue_links($audit_id, $url, $url_id, $dom, $max_depth, $max_urls);
        }
    }

    /**
     * Discover internal links and queue them for crawling (respects max_depth)
     */
    private function discover_and_queue_links(string $audit_id, string $url, int $url_id, \DOMDocument $dom, int $max_depth, int $max_urls): void {
        global $wpdb;

        // Get current URL's depth
        $current_depth = (int) $wpdb->get_var($wpdb->prepare(
            "SELECT depth FROM {$this->audit_urls_table} WHERE id = %d",
            $url_id
        ));

        // Don't discover if at max depth
        if ($current_depth >= $max_depth) {
            return;
        }

        // Check current URL count
        $current_count = $this->count_distinct_audit_urls($audit_id, true);

        if ($current_count >= $max_urls) {
            return;
        }

        // Get home URL for internal link detection
        $home_url = home_url();
        $home_host = wp_parse_url($home_url, PHP_URL_HOST);

        // Find all links
        $links = $dom->getElementsByTagName('a');
        $new_urls = [];
        $new_depth = $current_depth + 1;

        foreach ($links as $link) {
            $href = $link->getAttribute('href');

            // Skip empty, anchor-only, or javascript links
            if (empty($href) || $href[0] === '#' || strpos($href, 'javascript:') === 0 || strpos($href, 'mailto:') === 0) {
                continue;
            }

            // Make absolute URL
            $absolute_url = $this->make_absolute_url($href, $url);
            if (!$absolute_url) {
                continue;
            }

            // Check if internal link
            $link_host = wp_parse_url($absolute_url, PHP_URL_HOST);
            if ($link_host !== $home_host) {
                continue;
            }

            // Normalize URL (remove fragment, trailing slash consistency)
            $normalized = strtok($absolute_url, '#');
            $normalized = rtrim($normalized, '/');

            // Skip non-content URLs (Elementor library, admin, etc.)
            if (!$this->should_audit_url($normalized)) {
                continue;
            }

            // Skip if already in queue
            $exists = $wpdb->get_var($wpdb->prepare(
                "SELECT id FROM {$this->audit_urls_table} WHERE audit_id = %s AND url = %s",
                $audit_id,
                $normalized
            ));

            if (!$exists && !isset($new_urls[$normalized])) {
                $new_urls[$normalized] = $new_depth;
            }

            // Limit new URLs per page to avoid explosion
            if (count($new_urls) >= 20) {
                break;
            }
        }

        // Insert new URLs
        if (!empty($new_urls)) {
            $remaining_slots = $max_urls - $current_count;
            $urls_to_add = array_slice($new_urls, 0, $remaining_slots, true);

            foreach ($urls_to_add as $new_url => $depth) {
                $wpdb->insert($this->audit_urls_table, [
                    'audit_id' => $audit_id,
                    'url' => $new_url,
                    'status' => 'pending',
                    'depth' => $depth,
                    'created_at' => current_time('mysql'),
                ]);
            }
        }
    }

    /**
     * Helper function to merge check results
     */
    private function merge_check_results(&$issues, &$passed_checks, &$warnings, $results): void {
        if (isset($results['issues'])) {
            $issues = array_merge($issues, $results['issues']);
        }
        if (isset($results['warnings'])) {
            $warnings = array_merge($warnings, $results['warnings']);
        }
        if (isset($results['passed'])) {
            $passed_checks = array_merge($passed_checks, $results['passed']);
        }
    }

    /**
     * Normalize duplicate/advisory issue types for scoring.
     */
    private function get_audit_issue_score_key(string $type): string {
        $normalized = strtolower(trim($type));
        $aliases = [
            'short_title' => 'title_too_short',
            'long_title' => 'title_too_long',
            'short_meta_description' => 'meta_desc_too_short',
            'long_meta_description' => 'meta_desc_too_long',
            'multiple_h1' => 'multiple_h1',
            'duplicate_h1' => 'multiple_h1',
            'missing_h1' => 'missing_h1',
            'heading_hierarchy' => 'heading_skip_levels',
            'missing_alt_text' => 'missing_alt_text',
            'missing_lang_attribute' => 'missing_lang',
            'missing_viewport' => 'missing_viewport',
            'cwv_not_measured' => 'cwv_not_measured',
        ];

        return $aliases[$normalized] ?? $normalized;
    }

    /**
     * Determine whether an issue is a site-wide maintenance/environment check.
     */
    private function is_audit_sitewide_maintenance_issue(string $type): bool {
        $key = $this->get_audit_issue_score_key($type);

        $sitewide_types = [
            'plugins_need_updates',
            'potential_plugin_conflict',
            'wordpress_outdated',
            'php_outdated',
            'too_many_plugins',
            'large_autoload',
            'moderate_autoload',
            'many_revisions',
            'database_optimization',
        ];

        return in_array($key, $sitewide_types, true);
    }

    /**
     * Determine whether an issue is a crawler/server transport failure.
     */
    private function is_audit_crawl_server_issue(string $type): bool {
        $key = $this->get_audit_issue_score_key($type);

        if (in_array($key, ['connectivity', 'crawl_fetch_failed', 'crawl_timeout', 'crawl_error', 'server_response_error'], true)) {
            return true;
        }

        return strpos($key, 'crawl_') === 0;
    }

    /**
     * Scope multiplier so maintenance and transport failures do not score like page SEO failures.
     */
    private function get_audit_issue_scope_multiplier(string $type): float {
        if ($this->is_audit_sitewide_maintenance_issue($type)) {
            return 0.35;
        }

        if ($this->is_audit_crawl_server_issue($type)) {
            return 0.45;
        }

        return 1.0;
    }

    /**
     * Return score multiplier for an issue type.
     */
    private function get_audit_issue_score_multiplier(string $type): float {
        $key = $this->get_audit_issue_score_key($type);
        $non_scoring = [
            'cwv_not_measured',
        ];

        if (in_array($key, $non_scoring, true)) {
            return 0.0;
        }

        $multipliers = [
            'title_too_short' => 0.15,
            'title_too_long' => 0.15,
            'meta_desc_too_short' => 0.10,
            'meta_desc_too_long' => 0.10,
            'multiple_h1' => 0.20,
            'heading_skip_levels' => 0.15,
        ];

        return ($multipliers[$key] ?? 1.0) * $this->get_audit_issue_scope_multiplier($type);
    }

    /**
     * Calculate a weighted deduction for a single issue.
     */
    private function calculate_audit_issue_score_deduction(string $type, string $severity): float {
        $severity = strtolower(trim($severity));
        $base = [
            'critical' => 6.0,
            'high' => 3.0,
            'medium' => 1.2,
            'low' => 0.4,
            'warning' => 0.15,
        ];

        if (!isset($base[$severity])) {
            return 0.0;
        }

        return $base[$severity] * $this->get_audit_issue_score_multiplier($type);
    }
    
    /**
     * Calculate page score based on findings
     */
    private function calculate_page_score($issues, $warnings, $passed_checks): int {
        $total_checks = count($issues) + count($warnings) + count($passed_checks);
        if ($total_checks === 0) return 0;

        $deductions = 0.0;
        foreach ($issues as $issue) {
            $deductions += $this->calculate_audit_issue_score_deduction(
                (string) ($issue['type'] ?? ''),
                (string) ($issue['severity'] ?? 'medium')
            );
        }

        foreach ($warnings as $warning) {
            $deductions += $this->calculate_audit_issue_score_deduction(
                (string) ($warning['type'] ?? ''),
                (string) ($warning['severity'] ?? 'warning')
            );
        }

        $score = max(0, min(100, round(100 - $deductions)));
        return (int) $score;
    }
    
    private function check_title(\DOMDocument $dom): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        $titles = $dom->getElementsByTagName('title');
        
        if ($titles->length === 0) {
            $result['issues'][] = [
                'type' => 'missing_title',
                'severity' => 'critical',
                'message' => 'Missing title tag - Critical for SEO',
                'data' => ['fix' => 'Add a unique <title> tag to your page'],
            ];
        } elseif ($titles->length > 1) {
            $result['issues'][] = [
                'type' => 'duplicate_title',
                'severity' => 'high',
                'message' => 'Multiple title tags found',
                'data' => ['count' => $titles->length, 'fix' => 'Keep only one <title> tag'],
            ];
        } else {
            $title = trim($titles->item(0)->textContent);
            
            if (empty($title)) {
                $result['issues'][] = [
                    'type' => 'empty_title',
                    'severity' => 'critical',
                    'message' => 'Empty title tag',
                    'data' => ['fix' => 'Add descriptive text to your title tag'],
                ];
            } else {
                // Title exists - pass basic check
                $result['passed'][] = 'has_title';
                
                $title_length = mb_strlen($title);
                
                // Length checks
                if ($title_length < 30) {
                    $result['warnings'][] = [
                        'type' => 'short_title',
                        'severity' => 'medium',
                        'message' => sprintf('Title too short (%d chars, recommended 30-60)', $title_length),
                        'data' => ['title' => $title, 'length' => $title_length],
                    ];
                } elseif ($title_length > 60) {
                    $result['warnings'][] = [
                        'type' => 'long_title',
                        'severity' => 'medium',
                        'message' => sprintf('Title too long (%d chars, may be truncated)', $title_length),
                        'data' => ['title' => $title, 'length' => $title_length],
                    ];
                } else {
                    $result['passed'][] = 'title_length_optimal';
                }
                
                // Keyword stuffing check
                if (preg_match('/(\b\w+\b).*\b\1\b.*\b\1\b/i', $title)) {
                    $result['warnings'][] = [
                        'type' => 'title_keyword_stuffing',
                        'severity' => 'medium',
                        'message' => 'Possible keyword stuffing in title',
                        'data' => ['title' => $title],
                    ];
                } else {
                    $result['passed'][] = 'no_keyword_stuffing';
                }
            }
        }
        
        return $result;
    }
    
    /**
     * Check meta description
     */
    private function check_meta_description(\DOMDocument $dom): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        $xpath = new \DOMXPath($dom);
        $descriptions = $xpath->query('//meta[@name="description"]');
        
        if ($descriptions->length === 0) {
            $result['issues'][] = [
                'type' => 'missing_meta_description',
                'severity' => 'high',
                'message' => 'Missing meta description',
            ];
        } else {
            $description = $descriptions->item(0)->getAttribute('content');
            
            if (empty($description)) {
                $result['issues'][] = [
                    'type' => 'empty_meta_description',
                    'severity' => 'high',
                    'message' => 'Empty meta description',
                ];
            } elseif (strlen($description) < 120) {
                $result['warnings'][] = [
                    'type' => 'short_meta_description',
                    'severity' => 'medium',
                    'message' => sprintf('Meta description too short (%d chars)', strlen($description)),
                    'data' => ['description' => $description, 'length' => strlen($description)],
                ];
            } elseif (strlen($description) > 160) {
                $result['warnings'][] = [
                    'type' => 'long_meta_description',
                    'severity' => 'medium',
                    'message' => sprintf('Meta description too long (%d chars)', strlen($description)),
                    'data' => ['description' => $description, 'length' => strlen($description)],
                ];
            } else {
                $result['passed'][] = 'meta_description_optimal';
            }
        }
        
        return $result;
    }
    
    /**
     * Check headings
     */
    private function check_headings(\DOMDocument $dom): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        $h1s = $dom->getElementsByTagName('h1');
        
        if ($h1s->length === 0) {
            $result['issues'][] = [
                'type' => 'missing_h1',
                'severity' => 'high',
                'message' => 'Missing H1 heading',
            ];
        } elseif ($h1s->length > 1) {
            $result['warnings'][] = [
                'type' => 'multiple_h1',
                'severity' => 'medium',
                'message' => sprintf('Multiple H1 headings found (%d)', $h1s->length),
            ];
        } else {
            $result['passed'][] = 'has_single_h1';
        }
        
        return $result;
    }
    
    /**
     * Check images
     */
    private function check_images(\DOMDocument $dom, string $base_url): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        $images = $dom->getElementsByTagName('img');
        $missing_alt_count = 0;
        $broken_images = 0;
        
        foreach ($images as $img) {
            $alt = $img->getAttribute('alt');
            $src = $img->getAttribute('src');
            
            if (empty($alt)) {
                $missing_alt_count++;
            }
            
            // Check if image exists
            if (!empty($src)) {
                $image_url = $this->make_absolute_url($src, $base_url);
                $response = wp_remote_head($image_url, ['timeout' => 5]);
                
                if (is_wp_error($response) || wp_remote_retrieve_response_code($response) >= 400) {
                    $broken_images++;
                }
            }
        }
        
        if ($missing_alt_count > 0) {
            $result['issues'][] = [
                'type' => 'missing_alt_text',
                'severity' => 'medium',
                'message' => sprintf('%d images missing alt text', $missing_alt_count),
                'data' => ['count' => $missing_alt_count],
            ];
        }
        
        if ($broken_images > 0) {
            $result['issues'][] = [
                'type' => 'broken_image',
                'severity' => 'high',
                'message' => sprintf('%d broken images found', $broken_images),
                'data' => ['count' => $broken_images],
            ];
        }
        
        if ($images->length > 0 && $missing_alt_count === 0) {
            $result['passed'][] = 'all_images_have_alt';
        }
        
        return $result;
    }
    
    /**
     * Check links
     */
    private function check_links(\DOMDocument $dom, string $base_url): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        $links = $dom->getElementsByTagName('a');
        $checked_urls = [];
        $broken_links = 0;
        $internal_links = 0;
        $external_links = 0;
        
        foreach ($links as $link) {
            $href = $link->getAttribute('href');
            
            if (empty($href) || $href === '#') {
                continue;
            }
            
            $absolute_url = $this->make_absolute_url($href, $base_url);
            
            // Skip if already checked
            if (in_array($absolute_url, $checked_urls)) {
                continue;
            }
            
            $checked_urls[] = $absolute_url;
            
            // Check if internal or external
            if (strpos($absolute_url, home_url()) === 0) {
                $internal_links++;
                $response = wp_remote_head($absolute_url, ['timeout' => 5]);
                
                if (is_wp_error($response) || wp_remote_retrieve_response_code($response) >= 400) {
                    $broken_links++;
                }
            } else {
                $external_links++;
            }
        }
        
        if ($broken_links > 0) {
            $result['issues'][] = [
                'type' => 'broken_link',
                'severity' => 'high',
                'message' => sprintf('%d broken internal links found', $broken_links),
                'data' => ['count' => $broken_links],
            ];
        }
        
        if ($internal_links === 0) {
            $result['warnings'][] = [
                'type' => 'no_internal_links',
                'severity' => 'medium',
                'message' => 'No internal links found',
            ];
        } else {
            $result['passed'][] = 'has_internal_links';
        }
        
        return $result;
    }
    
    /**
     * Process URLs synchronously (for testing/debugging)
     */
    public function process_urls_sync(string $audit_id, int $limit = 5): int {
        global $wpdb;
        
        $urls = $wpdb->get_results($wpdb->prepare(
            "SELECT id, url FROM {$this->audit_urls_table} 
             WHERE audit_id = %s AND status = 'pending'
             LIMIT %d",
            $audit_id,
            $limit
        ));
        
        $processed = 0;
        foreach ($urls as $url_data) {
            prorank_log("ProRank SEO: Processing URL sync: {$url_data->url}");
            try {
                $this->check_url($audit_id, (int)$url_data->id, $url_data->url);
                $processed++;
            } catch (\Exception $e) {
                prorank_log("ProRank SEO: Error processing URL: " . $e->getMessage());
            }
        }
        
        return $processed;
    }
    
    /**
     * Stop audit
     */
    public function stop_audit(string $audit_id = null): bool {
        global $wpdb;
        
        // If no audit_id provided, stop the latest audit
        if (!$audit_id) {
            $audit = $wpdb->get_row("
                SELECT audit_id FROM {$this->audits_table} 
                WHERE status IN ('crawling', 'checking')
                ORDER BY started_at DESC 
                LIMIT 1
            ");
            
            if (!$audit) {
                return false;
            }
            
            $audit_id = $audit->audit_id;
        }
        
        // Update audit status to stopped
        $result = $wpdb->update(
            $this->audits_table,
            [
                'status' => 'stopped',
                'completed_at' => current_time('mysql'),
            ],
            ['audit_id' => $audit_id]
        );
        
        // Clear any scheduled events for this audit
        $timestamp = wp_next_scheduled('prorank_run_site_crawler', [$audit_id]);
        if ($timestamp) {
            wp_unschedule_event($timestamp, 'prorank_run_site_crawler', [$audit_id]);
        }
        
        $timestamp = wp_next_scheduled('prorank_schedule_url_checks', [$audit_id]);
        if ($timestamp) {
            wp_unschedule_event($timestamp, 'prorank_schedule_url_checks', [$audit_id]);
        }
        
        $timestamp = wp_next_scheduled('prorank_complete_audit', [$audit_id]);
        if ($timestamp) {
            wp_unschedule_event($timestamp, 'prorank_complete_audit', [$audit_id]);
        }
        
        // Clear all URL check events
        $crons = _get_cron_array();
        foreach ($crons as $timestamp => $cron) {
            foreach ($cron as $hook => $args) {
                if ($hook === 'prorank_check_audit_url') {
                    foreach ($args as $key => $arg) {
                        if (isset($arg['args'][0]) && $arg['args'][0] === $audit_id) {
                            wp_unschedule_event($timestamp, $hook, $arg['args']);
                        }
                    }
                }
            }
        }
        
        prorank_log("ProRank SEO: Audit {$audit_id} stopped by user");
        
        return $result !== false;
    }
    
    /**
     * Complete audit
     */
    public function complete_audit(string $audit_id): array {
        global $wpdb;

        // Run orphaned pages check now that all links are collected
        $audit = $wpdb->get_row($wpdb->prepare(
            "SELECT options FROM {$this->audits_table} WHERE audit_id = %s",
            $audit_id
        ));
        $options = $audit && !empty($audit->options) ? json_decode($audit->options, true) : [];
        $check_types = $options['check_types'] ?? [];

        if (!isset($check_types['orphaned_pages']) || $check_types['orphaned_pages']) {
            $this->check_orphaned_pages($audit_id);
        }

        // Calculate statistics - count unique issues and affected URLs separately
        $stats = $wpdb->get_row($wpdb->prepare("
            SELECT 
                COUNT(DISTINCT u.url) as total_urls,
                COUNT(DISTINCT CASE WHEN u.issues_count > 0 THEN u.url END) as urls_with_issues,
                COUNT(DISTINCT CONCAT(i.type, ':', i.message)) as unique_issues,
                COUNT(DISTINCT CASE WHEN i.severity != 'passed' THEN CONCAT(u.url, ':', i.type, ':', COALESCE(i.message, '')) END) as total_issue_instances,
                COUNT(DISTINCT CASE WHEN i.severity = 'critical' THEN CONCAT(i.type, ':', i.message) END) as critical_issues,
                COUNT(DISTINCT CASE WHEN i.severity = 'high' THEN CONCAT(i.type, ':', i.message) END) as high_issues,
                COUNT(DISTINCT CASE WHEN i.severity = 'medium' THEN CONCAT(i.type, ':', i.message) END) as medium_issues,
                COUNT(DISTINCT CASE WHEN i.severity = 'low' THEN CONCAT(i.type, ':', i.message) END) as low_issues,
                COUNT(DISTINCT CASE WHEN i.severity = 'passed' THEN CONCAT(u.url, ':', i.type) END) as passed_checks,
                COUNT(DISTINCT CASE WHEN i.severity = 'critical' THEN CONCAT(u.url, ':', i.type, ':', COALESCE(i.message, '')) END) as critical_issue_instances,
                COUNT(DISTINCT CASE WHEN i.severity = 'high' THEN CONCAT(u.url, ':', i.type, ':', COALESCE(i.message, '')) END) as high_issue_instances,
                COUNT(DISTINCT CASE WHEN i.severity = 'medium' THEN CONCAT(u.url, ':', i.type, ':', COALESCE(i.message, '')) END) as medium_issue_instances,
                COUNT(DISTINCT CASE WHEN i.severity = 'low' THEN CONCAT(u.url, ':', i.type, ':', COALESCE(i.message, '')) END) as low_issue_instances
            FROM {$this->audit_urls_table} u
            LEFT JOIN {$this->audit_issues_table} i ON u.id = i.url_id
            WHERE u.audit_id = %s
        ", $audit_id));
        
        // Calculate category stats
        $categories = $this->calculate_category_stats($audit_id);

        $score_rows = $wpdb->get_results($wpdb->prepare(
            "SELECT DISTINCT i.type, i.severity
            FROM {$this->audit_issues_table} i
            WHERE i.audit_id = %s AND i.severity != 'passed'",
            $audit_id
        ));

        $weighted_counts = [
            'critical' => 0.0,
            'high' => 0.0,
            'medium' => 0.0,
            'low' => 0.0,
            'warning' => 0.0,
        ];

        if ($score_rows && !is_wp_error($score_rows)) {
            foreach ($score_rows as $score_row) {
                $severity = strtolower((string) ($score_row->severity ?? ''));
                if (!isset($weighted_counts[$severity])) {
                    continue;
                }

                $weighted_counts[$severity] += $this->get_audit_issue_score_multiplier((string) ($score_row->type ?? ''));
            }
        }

        $score = 100;
        $score -= min(30, $weighted_counts['critical'] * 6.0);
        $score -= min(24, $weighted_counts['high'] * 3.0);
        $score -= min(14, $weighted_counts['medium'] * 1.2);
        $score -= min(8, $weighted_counts['low'] * 0.4);
        $score -= min(4, $weighted_counts['warning'] * 0.15);

        if ($stats->passed_checks > 0 && $stats->total_urls > 0) {
            $pass_rate = $stats->passed_checks / max(1, ($stats->total_urls * 10));
            $score += min(5, round($pass_rate * 5));
        }

        $score = max(0, min(100, round($score)));
        
        // Update audit
        $wpdb->update($this->audits_table, [
            'status' => 'completed',
            'completed_at' => current_time('mysql'),
            'score' => $score,
            'stats' => json_encode($stats),
        ], ['audit_id' => $audit_id]);
        
        return [
            'audit_id' => $audit_id,
            'score' => $score,
            'stats' => $stats,
            'categories' => $categories,
            'completed_at' => current_time('mysql'),
        ];
    }
    
    /**
     * Calculate category statistics
     */
    private function calculate_category_stats(string $audit_id): array {
        global $wpdb;

        $results = $this->get_empty_category_scores();
        $issues = $wpdb->get_results($wpdb->prepare(
            "SELECT i.type, i.severity, i.message, COUNT(DISTINCT CONCAT(u.url, ':', i.type, ':', COALESCE(i.message, ''))) as count
            FROM {$this->audit_issues_table} i
            INNER JOIN {$this->audit_urls_table} u ON u.id = i.url_id
            WHERE i.audit_id = %s
            GROUP BY i.type, i.severity, i.message",
            $audit_id
        ));

        if (!$issues || is_wp_error($issues)) {
            return $results;
        }

        foreach ($issues as $issue) {
            $category = $this->map_audit_issue_to_category((string) $issue->type);
            if (!isset($results[$category])) {
                continue;
            }

            $results[$category]['instances'] += (int) $issue->count;

            if (array_key_exists($issue->severity, $results[$category])) {
                $results[$category][$issue->severity] += 1;
                if ($issue->severity === 'warning') {
                    $results[$category]['warnings'] = $results[$category]['warning'];
                }
            }

            $results[$category]['penalty'] += $this->calculate_audit_issue_score_deduction(
                (string) $issue->type,
                (string) $issue->severity
            );
        }

        foreach ($results as $category_key => $stats) {
            $total = $stats['critical'] + $stats['high'] + $stats['medium'] + $stats['low'] + $stats['warning'] + $stats['passed'];
            $results[$category_key]['total'] = $total;
            $results[$category_key]['warnings'] = $stats['warning'];
            $results[$category_key]['score'] = $this->calculate_issue_category_score($results[$category_key]);
        }

        return $results;
    }

    /**
     * Fetch a single audit row by audit_id.
     */
    private function get_audit_row_by_id(string $audit_id): ?\stdClass {
        global $wpdb;
        $row = $wpdb->get_row($wpdb->prepare(
            "SELECT * FROM {$this->audits_table} WHERE audit_id = %s LIMIT 1",
            $audit_id
        ));

        return $row instanceof \stdClass ? $row : null;
    }

    /**
     * Check if a URL-check cron event exists for this audit.
     */
    private function has_pending_check_event(string $audit_id): bool {
        $crons = _get_cron_array();
        if (!is_array($crons)) {
            return false;
        }

        foreach ($crons as $cron) {
            if (!is_array($cron) || !isset($cron['prorank_check_audit_url']) || !is_array($cron['prorank_check_audit_url'])) {
                continue;
            }

            foreach ($cron['prorank_check_audit_url'] as $event) {
                $args = is_array($event['args'] ?? null) ? $event['args'] : [];
                if (($args[0] ?? null) === $audit_id) {
                    return true;
                }
            }
        }

        return false;
    }

    /**
     * Count distinct audit URLs, optionally including the synthetic site-level row.
     */
    private function count_distinct_audit_urls(string $audit_id, bool $include_site_level = true): int {
        global $wpdb;

        if ($include_site_level) {
            return (int) $wpdb->get_var($wpdb->prepare(
                "SELECT COUNT(DISTINCT url) FROM {$this->audit_urls_table} WHERE audit_id = %s",
                $audit_id
            ));
        }

        return (int) $wpdb->get_var($wpdb->prepare(
            "SELECT COUNT(DISTINCT url) FROM {$this->audit_urls_table} WHERE audit_id = %s AND url NOT LIKE %s",
            $audit_id,
            '%?_site_level_checks'
        ));
    }

    /**
     * Claim a batch of pending URLs so overlapping runners do not process them twice.
     *
     * @param array<int,mixed> $url_ids
     * @return array<int,int>
     */
    private function claim_audit_url_batch(string $audit_id, array $url_ids): array {
        global $wpdb;

        $url_ids = array_values(array_unique(array_map('intval', $url_ids)));
        if (empty($url_ids)) {
            return [];
        }

        $placeholders = implode(',', array_fill(0, count($url_ids), '%d'));
        $params = array_merge([$audit_id], $url_ids);

        $wpdb->query($wpdb->prepare(
            "UPDATE {$this->audit_urls_table}
             SET status = 'checking'
             WHERE audit_id = %s AND status = 'pending' AND id IN ({$placeholders})",
            ...$params
        )); // phpcs:ignore WordPress.DB.PreparedSQL.NotPrepared

        $claimed = $wpdb->get_col($wpdb->prepare(
            "SELECT id FROM {$this->audit_urls_table}
             WHERE audit_id = %s AND status = 'checking' AND id IN ({$placeholders})",
            ...$params
        )); // phpcs:ignore WordPress.DB.PreparedSQL.NotPrepared

        return array_values(array_map('intval', $claimed));
    }

    /**
     * Recover local audits that are stuck in running states.
     */
    private function maybe_recover_running_audit(\stdClass $audit): \stdClass {
        if (!in_array((string) ($audit->status ?? ''), ['crawling', 'checking'], true)) {
            return $audit;
        }

        global $wpdb;
        $audit_id = (string) $audit->audit_id;

        $total_urls = (int) $wpdb->get_var($wpdb->prepare(
            "SELECT COUNT(*) FROM {$this->audit_urls_table} WHERE audit_id = %s",
            $audit_id
        ));
        $checked_urls = (int) $wpdb->get_var($wpdb->prepare(
            "SELECT COUNT(*) FROM {$this->audit_urls_table} WHERE audit_id = %s AND status = 'checked'",
            $audit_id
        ));
        $pending_urls = (int) $wpdb->get_var($wpdb->prepare(
            "SELECT COUNT(*) FROM {$this->audit_urls_table} WHERE audit_id = %s AND status = 'pending'",
            $audit_id
        ));
        $checking_urls = (int) $wpdb->get_var($wpdb->prepare(
            "SELECT COUNT(*) FROM {$this->audit_urls_table} WHERE audit_id = %s AND status = 'checking'",
            $audit_id
        ));

        $started_at = strtotime((string) ($audit->started_at ?? ''));
        $elapsed = $started_at ? max(0, time() - $started_at) : 0;

        // If checking is finished, finalize immediately.
        if ($audit->status === 'checking' && $total_urls > 0 && $pending_urls === 0 && $checking_urls === 0 && $checked_urls >= $total_urls) {
            $this->complete_audit($audit_id);
            $refreshed = $this->get_audit_row_by_id($audit_id);
            return $refreshed ?: $audit;
        }

        $crawler_scheduled_at = wp_next_scheduled('prorank_run_site_crawler', [$audit_id]);
        $immediate_crawler_at = wp_next_scheduled('prorank_run_immediate_crawler', [$audit_id]);
        $crawler_scheduled = (bool) $crawler_scheduled_at || (bool) $immediate_crawler_at;
        $batch_scheduled_at = wp_next_scheduled('prorank_schedule_url_checks', [$audit_id]);
        $batch_scheduled = (bool) $batch_scheduled_at;
        $check_scheduled = $this->has_pending_check_event($audit_id);

        $next_crawler_at = null;
        foreach ([$crawler_scheduled_at, $immediate_crawler_at] as $timestamp) {
            $timestamp = is_numeric($timestamp) ? (int) $timestamp : 0;
            if ($timestamp <= 0) {
                continue;
            }
            if ($next_crawler_at === null || $timestamp < $next_crawler_at) {
                $next_crawler_at = $timestamp;
            }
        }
        $crawler_overdue = $next_crawler_at !== null && (time() - $next_crawler_at) > 10;
        $batch_overdue = is_numeric($batch_scheduled_at) && (time() - (int) $batch_scheduled_at) > 10;

        // Recover from crawling state when crawler cron is missing/overdue.
        if (
            $audit->status === 'crawling'
            && $elapsed > 10
            && ($total_urls <= 1 || $pending_urls === 0)
            && (!$crawler_scheduled || $crawler_overdue)
        ) {
            $this->run_crawler($audit_id);
            $refreshed = $this->get_audit_row_by_id($audit_id);
            return $refreshed ?: $audit;
        }

        // Recover checking state with pending URLs but no queued workers.
        if (
            $audit->status === 'checking'
            && $pending_urls > 0
            && (!$batch_scheduled || $batch_overdue)
            && !$check_scheduled
            && $elapsed > 10
        ) {
            wp_schedule_single_event(time() + 1, 'prorank_schedule_url_checks', [$audit_id]);
            spawn_cron();
        }

        if (
            $audit->status === 'checking'
            && $checking_urls > 0
            && !$check_scheduled
            && (!$batch_scheduled || $batch_overdue)
            && $elapsed > 30
        ) {
            $wpdb->update(
                $this->audit_urls_table,
                ['status' => 'pending'],
                ['audit_id' => $audit_id, 'status' => 'checking']
            );
            wp_schedule_single_event(time() + 1, 'prorank_schedule_url_checks', [$audit_id]);
            spawn_cron();
        }

        // Fail long-stalled runs so users can start fresh.
        if ($elapsed > 1800 && !$crawler_scheduled && !$batch_scheduled && !$check_scheduled) {
            $wpdb->update($this->audits_table, [
                'status' => 'failed',
                'completed_at' => current_time('mysql'),
                'error' => __('Audit stalled with no runnable tasks', 'prorank-seo'),
            ], ['audit_id' => $audit_id]);

            $refreshed = $this->get_audit_row_by_id($audit_id);
            return $refreshed ?: $audit;
        }

        return $audit;
    }
    
    /**
     * Get audit status
     */
    public function get_audit_status(string $audit_id = null): array {
        global $wpdb;
        
        // Get latest crawler audit if no ID provided
        if (!$audit_id) {
            $audit = $wpdb->get_row("
                SELECT * FROM {$this->audits_table} 
                WHERE audit_id LIKE 'audit_%'
                  AND status IN ('crawling', 'checking', 'completed')
                ORDER BY started_at DESC 
                LIMIT 1
            ");
        } else {
            $audit = $wpdb->get_row($wpdb->prepare(
                "SELECT * FROM {$this->audits_table} WHERE audit_id = %s",
                $audit_id
            ));
        }
        
        if (!$audit) {
            return [
                'state' => 'idle',
                'stats' => [
                    'critical_issues' => 0,
                    'high_issues' => 0,
                    'medium_issues' => 0,
                    'low_issues' => 0,
                    'total_issues' => 0,
                    'unique_issues' => 0,
                    'total_instances' => 0,
                    'total_urls' => 0,
                    'passed' => 0,
                ],
            ];
        }

        $audit = $this->maybe_recover_running_audit($audit);
        
        $stats = $audit->stats ? json_decode($audit->stats, true) : [];
        
        // Get issues by type (excluding passed checks)
        $issues = [];
        if ($audit->status === 'completed') {
            $issue_types = $wpdb->get_results($wpdb->prepare("
                SELECT 
                    i.type,
                    i.severity,
                    i.message,
                    COUNT(*) as count,
                    GROUP_CONCAT(DISTINCT u.url SEPARATOR '|||') as affected_urls,
                    MAX(i.data) as sample_data
                FROM {$this->audit_issues_table} i
                JOIN {$this->audit_urls_table} u ON i.url_id = u.id
                WHERE i.audit_id = %s AND i.severity != 'passed'
                GROUP BY i.type, i.severity, i.message
                ORDER BY 
                    FIELD(i.severity, 'critical', 'high', 'medium', 'low'),
                    count DESC
            ", $audit->audit_id));
            
            foreach ($issue_types as $issue_type) {
                $affected_urls = $issue_type->affected_urls ? explode('|||', $issue_type->affected_urls) : [];
                $issues[] = [
                    'id' => $issue_type->type . '_' . md5($issue_type->message),
                    'type' => $issue_type->type,
                    'message' => $issue_type->message,
                    'severity' => $issue_type->severity,
                    'count' => (int)$issue_type->count,
                    'affected_urls' => array_slice($affected_urls, 0, 5),
                    'data' => json_decode($issue_type->sample_data, true),
                ];
            }
        }
        
        $unique_issues_stats = is_array($stats['unique_issues'] ?? null) ? $stats['unique_issues'] : [];
        $critical_issues = (int) ($stats['critical_issues'] ?? ($unique_issues_stats['critical'] ?? 0));
        $high_issues = (int) ($stats['high_issues'] ?? ($unique_issues_stats['high'] ?? 0));
        $medium_issues = (int) ($stats['medium_issues'] ?? ($unique_issues_stats['medium'] ?? 0));
        $low_issues = (int) ($stats['low_issues'] ?? ($unique_issues_stats['low'] ?? 0));
        $total_issues = (int) ($stats['total_issues'] ?? ($unique_issues_stats['total'] ?? 0));
        if ($total_issues <= 0) {
            if (isset($stats['unique_issues']) && is_numeric($stats['unique_issues'])) {
                $total_issues = (int) $stats['unique_issues'];
            }
        }
        if ($total_issues <= 0) {
            $total_issues = $critical_issues + $high_issues + $medium_issues + $low_issues;
        }

        $total_instances = (int) ($stats['total_issue_instances'] ?? ($stats['total_instances'] ?? $total_issues));
        $total_urls = (int) ($stats['total_urls'] ?? 0);
        $passed = (int) ($stats['passed_checks'] ?? max(0, $total_urls - (int) ($stats['urls_with_issues'] ?? 0)));
        $total_checks = (int) ($stats['total_checks'] ?? max(0, $passed + $total_issues));

        $result = [
            'state' => $audit->status,
            'audit_id' => $audit->audit_id,
            'started_at' => $audit->started_at,
            'completed_at' => $audit->completed_at,
            'score' => (int)($audit->score ?? 0),
            'stats' => [
                'critical_issues' => $critical_issues,
                'high_issues' => $high_issues,
                'medium_issues' => $medium_issues,
                'low_issues' => $low_issues,
                'warning_issues' => $medium_issues + $low_issues,
                'total_issues' => $total_issues,
                'unique_issues' => (int) ($unique_issues_stats['total'] ?? ($stats['unique_issues'] ?? $total_issues)),
                'total_instances' => $total_instances,
                'total_urls' => $total_urls,
                'passed' => $passed,
                'total_checks' => $total_checks,
            ],
            'issues' => $issues,
            'quick_fixes' => $this->get_quick_fixes($issues),
            'categories' => $this->calculate_category_scores($audit->audit_id),
        ];
        
        // Add progress data if audit is running
        if ($audit->status === 'crawling' || $audit->status === 'checking') {
            $total_urls = $wpdb->get_var($wpdb->prepare(
                "SELECT COUNT(*) FROM {$this->audit_urls_table} WHERE audit_id = %s",
                $audit->audit_id
            ));
            
            $checked_urls = $wpdb->get_var($wpdb->prepare(
                "SELECT COUNT(*) FROM {$this->audit_urls_table} WHERE audit_id = %s AND status = 'checked'",
                $audit->audit_id
            ));
            
            // Prevent negative remaining / >100% progress if counts drift
            if ($checked_urls > $total_urls) {
                $total_urls = $checked_urls;
            }

            $result['progress'] = [
                'total_urls'   => (int)$total_urls,
                'checked_urls' => (int)$checked_urls,
                'percentage'   => $total_urls > 0 ? min(100, round(($checked_urls / $total_urls) * 100, 2)) : 0
            ];
        }
        
        return $result;
    }
    
    /**
     * Get quick fixes based on issues
     */
    private function get_quick_fixes(array $issues): array {
        $fixes = [];
        
        foreach ($issues as $issue) {
            switch ($issue['id']) {
                case 'missing_cache_headers':
                    $fixes[] = [
                        'id' => 'enable_caching',
                        'type' => 'enable_caching',
                        'title' => __('Enable browser caching', 'prorank-seo'),
                        'description' => __('Set proper cache headers for static resources', 'prorank-seo'),
                        'severity' => 'warning',
                        'impact' => __('Reduces server load and improves repeat visit speed', 'prorank-seo'),
                    ];
                    break;
                    
                case 'missing_compression':
                    $fixes[] = [
                        'id' => 'enable_compression',
                        'type' => 'enable_compression',
                        'title' => __('Enable GZIP compression', 'prorank-seo'),
                        'description' => __('Compress text-based resources to reduce file sizes', 'prorank-seo'),
                        'severity' => 'warning',
                        'impact' => __('Can reduce page size by 60-80%', 'prorank-seo'),
                    ];
                    break;
                    
                case 'broken_image':
                    $fixes[] = [
                        'id' => 'fix_broken_images',
                        'type' => 'fix_broken_images',
                        'title' => __('Fix broken images', 'prorank-seo'),
                        'description' => __('Remove or replace broken image references', 'prorank-seo'),
                        'severity' => 'high',
                        'impact' => __('Improves user experience and SEO', 'prorank-seo'),
                    ];
                    break;
            }
        }
        
        return array_slice($fixes, 0, 3); // Return top 3 fixes
    }
    
    /**
     * Clear audit data
     */
    public function clear_audit_data(): bool {
        global $wpdb;
        
        try {
            // Clear all scheduled audit tasks first
            $cron_array = _get_cron_array();
            $cleared_events = 0;
            foreach ($cron_array as $timestamp => $cron) {
                foreach ($cron as $hook => $values) {
                    if (strpos($hook, 'prorank_') === 0 && 
                        (strpos($hook, 'audit') !== false || 
                         strpos($hook, 'crawler') !== false || 
                         strpos($hook, 'check') !== false)) {
                        foreach ($values as $key => $args) {
                            wp_unschedule_event($timestamp, $hook, $args['args']);
                            $cleared_events++;
                        }
                    }
                }
            }
            
            prorank_log('ProRank SEO: Cleared ' . $cleared_events . ' scheduled audit events');
            
            // Clear all audit data
            $wpdb->query("TRUNCATE TABLE {$this->audits_table}");
            $wpdb->query("TRUNCATE TABLE {$this->audit_urls_table}");
            $wpdb->query("TRUNCATE TABLE {$this->audit_issues_table}");
            
            // Clear transients
            delete_transient('prorank_audit_status');
            delete_transient('prorank_audit_issues');
            
            // Clear any audit-related options
            $wpdb->query("DELETE FROM {$wpdb->options} WHERE option_name LIKE '%prorank_audit%'");
            
            return true;
        } catch (\Exception $e) {
            prorank_log('ProRank SEO: Clear audit data error: ' . $e->getMessage());
            return false;
        }
    }
    
    /**
     * Get audit history
     */
    public function get_audit_history(int $limit = 10): array {
        global $wpdb;
        
        $audits = $wpdb->get_results($wpdb->prepare("
            SELECT * FROM {$this->audits_table}
            WHERE status = 'completed' AND audit_id REGEXP '^audit_'
            ORDER BY completed_at DESC
            LIMIT %d
        ", $limit));
        
        $history = [];
        foreach ($audits as $audit) {
            $stats = $audit->stats ? json_decode($audit->stats, true) : [];
            $unique_issues_value = $stats['unique_issues'] ?? 0;
            if (is_array($unique_issues_value)) {
                $unique_issues_value = (int) ($unique_issues_value['total'] ?? 0);
            }
            
            $history[] = [
                'id' => $audit->audit_id,
                'date' => $audit->completed_at,
                'score' => (int)$audit->score,
                'critical' => (int)($stats['critical_issues'] ?? 0),
                'warnings' => (int)(($stats['medium_issues'] ?? 0) + ($stats['low_issues'] ?? 0)),
                'total_issues' => (int)($stats['total_issues'] ?? $unique_issues_value),
                'totalUrls' => (int)($stats['total_urls'] ?? 0),
                'duration' => strtotime($audit->completed_at) - strtotime($audit->started_at),
                'stats' => $stats,
                'state' => 'completed',
            ];
        }
        
        return $history;
    }
    
    /**
     * Calculate category scores
     */
    private function calculate_category_scores(string $audit_id): array {
        return $this->calculate_category_stats($audit_id);
    }

    /**
     * Get the category rows used by site-audit responses.
     *
     * @return array<string,array<string,int|string>>
     */
    private function get_empty_category_scores(): array {
        return [
            'technical-seo' => [
                'name' => 'Technical SEO',
                'score' => 100,
                'total' => 0,
                'passed' => 0,
                'critical' => 0,
                'high' => 0,
                'medium' => 0,
                'low' => 0,
                'warning' => 0,
                'warnings' => 0,
                'instances' => 0,
                'penalty' => 0,
            ],
            'on-page-seo' => [
                'name' => 'On-Page SEO',
                'score' => 100,
                'total' => 0,
                'passed' => 0,
                'critical' => 0,
                'high' => 0,
                'medium' => 0,
                'low' => 0,
                'warning' => 0,
                'warnings' => 0,
                'instances' => 0,
                'penalty' => 0,
            ],
            'performance' => [
                'name' => 'Performance',
                'score' => 100,
                'total' => 0,
                'passed' => 0,
                'critical' => 0,
                'high' => 0,
                'medium' => 0,
                'low' => 0,
                'warning' => 0,
                'warnings' => 0,
                'instances' => 0,
                'penalty' => 0,
            ],
            'security' => [
                'name' => 'Security',
                'score' => 100,
                'total' => 0,
                'passed' => 0,
                'critical' => 0,
                'high' => 0,
                'medium' => 0,
                'low' => 0,
                'warning' => 0,
                'warnings' => 0,
                'instances' => 0,
                'penalty' => 0,
            ],
            'content' => [
                'name' => 'Content',
                'score' => 100,
                'total' => 0,
                'passed' => 0,
                'critical' => 0,
                'high' => 0,
                'medium' => 0,
                'low' => 0,
                'warning' => 0,
                'warnings' => 0,
                'instances' => 0,
                'penalty' => 0,
            ],
        ];
    }

    /**
     * Map issue types to the dashboard category buckets.
     */
    private function map_audit_issue_to_category(string $type): string {
        $categories = [
            'technical-seo' => [
                'has_title', 'missing_title', 'duplicate_title', 'empty_title',
                'has_canonical', 'missing_canonical', 'duplicate_canonical', 'multiple_canonicals',
                'http_status', 'connectivity', 'redirect', 'indexable', 'noindex', 'noindex_set',
                'robots_noindex', 'missing_sitemap', 'missing_sitemap_reference', 'missing_robots_txt',
                'has_valid_sitemap', 'missing_lang_attribute', 'has_lang_attribute', 'missing_viewport',
                'has_viewport_meta',
            ],
            'on-page-seo' => [
                'short_title', 'long_title', 'title_length_optimal', 'title_keyword_stuffing',
                'missing_meta_description', 'empty_meta_description', 'short_meta_description',
                'long_meta_description', 'meta_description_optimal', 'missing_h1', 'multiple_h1',
                'duplicate_h1', 'has_single_h1', 'heading_hierarchy', 'missing_alt_text',
                'missing_og_tag', 'missing_og_tags', 'missing_twitter_cards', 'no_keyword_stuffing',
                'keyword_optimization', 'has_schema_markup', 'missing_schema',
            ],
            'performance' => [
                'cache_headers_set', 'no_cache_headers', 'gzip_enabled', 'no_compression',
                'fast_load_time', 'slow_load_time', 'moderate_load_time', 'optimal_page_size',
                'large_page_size', 'broken_image', 'broken_images', 'unoptimized_images',
                'render_blocking_resources', 'large_dom_size', 'cwv_not_measured',
            ],
            'security' => [
                'https_enabled', 'ssl_enabled', 'missing_https', 'mixed_content', 'no_mixed_content',
                'php_up_to_date', 'wordpress_up_to_date', 'plugins_need_updates',
            ],
            'content' => [
                'adequate_content_length', 'good_content_length', 'short_content', 'thin_content',
                'very_thin_content', 'duplicate_content', 'unique_content', 'poor_readability',
                'long_sentences', 'missing_headings', 'orphaned_page', 'has_internal_links',
                'no_internal_links', 'broken_link', 'broken_links', 'broken_internal_links',
                'external_broken_links',
            ],
        ];

        foreach ($categories as $category_key => $issue_types) {
            if (in_array($type, $issue_types, true)) {
                return $category_key;
            }
        }

        if (strpos($type, 'missing_security_header') === 0) {
            return 'security';
        }

        if (
            strpos($type, 'cwv_') === 0 ||
            strpos($type, 'core_web_vitals_') === 0 ||
            preg_match('/^(lcp|cls|inp|fcp|tti|tbt|speed_index)_/i', $type)
        ) {
            return 'performance';
        }

        return 'content';
    }

    /**
     * Calculate the score displayed for a category card.
     *
     * @param array<string,int|string> $stats
     */
    private function calculate_issue_category_score(array $stats): int {
        $total = (int) ($stats['total'] ?? 0);
        if ($total === 0) {
            return 100;
        }

        $penalty = (float) ($stats['penalty'] ?? 0);

        return max(0, min(100, 100 - (int) round($penalty)));
    }
    
    /**
     * Check canonical tags
     */
    private function check_canonical(\DOMDocument $dom, string $current_url): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        $xpath = new \DOMXPath($dom);
        $canonicals = $xpath->query('//link[@rel="canonical"]');
        
        if ($canonicals->length === 0) {
            $result['warnings'][] = [
                'type' => 'missing_canonical',
                'severity' => 'medium',
                'message' => 'Missing canonical tag',
            ];
        } elseif ($canonicals->length > 1) {
            $result['issues'][] = [
                'type' => 'multiple_canonicals',
                'severity' => 'high',
                'message' => 'Multiple canonical tags found',
            ];
        } else {
            $result['passed'][] = 'has_canonical';
        }
        
        return $result;
    }
    
    /**
     * Check Open Graph tags
     */
    private function check_open_graph(\DOMDocument $dom): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        $xpath = new \DOMXPath($dom);
        
        $required_og_tags = ['og:title', 'og:description', 'og:image', 'og:url'];
        $missing_tags = [];
        
        foreach ($required_og_tags as $property) {
            $tags = $xpath->query("//meta[@property='$property']");
            if ($tags->length === 0) {
                $missing_tags[] = $property;
            }
        }
        
        if (count($missing_tags) > 0) {
            $result['warnings'][] = [
                'type' => 'missing_og_tags',
                'severity' => 'low',
                'message' => 'Missing Open Graph tags: ' . implode(', ', $missing_tags),
                'data' => ['missing' => $missing_tags],
            ];
        } else {
            $result['passed'][] = 'has_all_og_tags';
        }
        
        return $result;
    }
    
    /**
     * Check Schema markup
     */
    private function check_schema_markup(\DOMDocument $dom): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        $xpath = new \DOMXPath($dom);
        
        // Check for JSON-LD
        $scripts = $xpath->query('//script[@type="application/ld+json"]');
        if ($scripts->length === 0) {
            $result['warnings'][] = [
                'type' => 'missing_schema',
                'severity' => 'medium',
                'message' => 'No structured data (Schema.org) found',
            ];
        } else {
            $result['passed'][] = 'has_schema_markup';
        }
        
        return $result;
    }
    
    /**
     * Check content quality
     */
    private function check_content_quality(\DOMDocument $dom, string $body): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        
        // Remove script and style content
        $text_content = wp_strip_all_tags($body);
        $word_count = str_word_count($text_content);
        
        if ($word_count < 300) {
            $result['issues'][] = [
                'type' => 'thin_content',
                'severity' => 'high',
                'message' => sprintf('Thin content (%d words)', $word_count),
                'data' => ['word_count' => $word_count],
            ];
        } elseif ($word_count < 500) {
            $result['warnings'][] = [
                'type' => 'short_content',
                'severity' => 'medium',
                'message' => sprintf('Content could be longer (%d words)', $word_count),
                'data' => ['word_count' => $word_count],
            ];
        } else {
            $result['passed'][] = 'good_content_length';
        }
        
        return $result;
    }
    
    /**
     * Check security
     */
    private function check_security($headers, string $url): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        
        // Check HTTPS
        if (strpos($url, 'https://') !== 0) {
            $result['issues'][] = [
                'type' => 'missing_https',
                'severity' => 'critical',
                'message' => 'Not using HTTPS',
            ];
        } else {
            $result['passed'][] = 'uses_https';
        }
        
        // Check security headers
        if (!isset($headers['x-frame-options'])) {
            $result['warnings'][] = [
                'type' => 'missing_security_header',
                'severity' => 'medium',
                'message' => 'Missing X-Frame-Options header',
            ];
        } else {
            $result['passed'][] = 'has_x_frame_options';
        }
        
        return $result;
    }
    
    /**
     * Check accessibility
     */
    private function check_accessibility(\DOMDocument $dom): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        
        // Check lang attribute
        $html = $dom->getElementsByTagName('html')->item(0);
        if ($html && !$html->hasAttribute('lang')) {
            $result['issues'][] = [
                'type' => 'missing_lang_attribute',
                'severity' => 'medium',
                'message' => 'Missing lang attribute on html element',
            ];
        } else if ($html) {
            $result['passed'][] = 'has_lang_attribute';
        }
        
        return $result;
    }
    
    /**
     * Make absolute URL
     */
    private function make_absolute_url(string $url, string $base): string {
        if (wp_parse_url($url, PHP_URL_SCHEME) !== null) {
            return $url;
        }
        
        if (strpos($url, '//') === 0) {
            $scheme = (string) (wp_parse_url($base, PHP_URL_SCHEME) ?: (is_ssl() ? 'https' : 'http'));
            return $scheme . ':' . $url;
        }
        
        if (strpos($url, '/') === 0) {
            $parsed = wp_parse_url($base);
            $scheme = (string) ($parsed['scheme'] ?? wp_parse_url(home_url('/'), PHP_URL_SCHEME) ?? (is_ssl() ? 'https' : 'http'));
            $host = (string) ($parsed['host'] ?? wp_parse_url(home_url('/'), PHP_URL_HOST) ?? '');
            if ($host === '') {
                return home_url($url);
            }
            return $scheme . '://' . $host . $url;
        }
        
        return rtrim($base, '/') . '/' . $url;
    }
    
    /**
     * Create database tables
     */
    private function maybe_create_tables(): void {
        global $wpdb;
        
        require_once(ABSPATH . 'wp-admin/includes/upgrade.php');
        
        $charset_collate = $wpdb->get_charset_collate();
        
        // Audits table
        $sql1 = "CREATE TABLE {$this->audits_table} (
            id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
            audit_id varchar(50) NOT NULL,
            status enum('idle','crawling','checking','completed','failed','stopped','stopping') DEFAULT 'idle',
            started_at datetime DEFAULT NULL,
            completed_at datetime DEFAULT NULL,
            total_urls int(11) DEFAULT 0,
            score int(3) DEFAULT NULL,
            stats longtext,
            options longtext,
            error text,
            PRIMARY KEY  (id),
            UNIQUE KEY audit_id (audit_id),
            KEY status (status),
            KEY started_at (started_at)
        ) $charset_collate;";
        
        // Audit URLs table
        $sql2 = "CREATE TABLE {$this->audit_urls_table} (
            id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
            audit_id varchar(50) NOT NULL,
            url varchar(500) NOT NULL,
            status enum('pending','checking','checked') DEFAULT 'pending',
            depth int(11) DEFAULT 0,
            issues_count int(11) DEFAULT 0,
            warnings_count int(11) DEFAULT 0,
            passed_count int(11) DEFAULT 0,
            score int(3) DEFAULT NULL,
            created_at datetime DEFAULT NULL,
            checked_at datetime DEFAULT NULL,
            PRIMARY KEY  (id),
            KEY audit_id (audit_id),
            KEY status (status),
            KEY url (url(255))
        ) $charset_collate;";
        
        // Audit issues table
        $sql3 = "CREATE TABLE {$this->audit_issues_table} (
            id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
            audit_id varchar(50) NOT NULL,
            url_id bigint(20) unsigned NOT NULL,
            type varchar(50) NOT NULL,
            severity enum('critical','high','medium','low','passed') DEFAULT 'medium',
            message text,
            data longtext,
            created_at datetime DEFAULT NULL,
            PRIMARY KEY  (id),
            KEY audit_id (audit_id),
            KEY url_id (url_id),
            KEY type (type),
            KEY severity (severity)
        ) $charset_collate;";
        
        // Execute table creation
        dbDelta($sql1);
        dbDelta($sql2);
        dbDelta($sql3);
        
        // Log any errors
        if ($wpdb->last_error) {
            prorank_log('ProRank SEO: Error creating audit tables: ' . $wpdb->last_error);
        }
    }
    
    /**
     * Check Core Web Vitals
     */
    private function check_core_web_vitals(string $url): array {
        return ['issues' => [], 'warnings' => [], 'passed' => []];
    }
    
    /**
     * Enhanced page speed check
     */
    private function check_page_speed(string $url, $headers, float $load_time, int $page_size): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        $headers = $this->normalize_response_headers($headers);
        
        // Check load time
        if ($load_time > 3.0) {
            $result['issues'][] = [
                'type' => 'slow_load_time',
                'severity' => 'high',
                'message' => sprintf('Page loads slowly (%.2fs, should be under 3s)', $load_time),
                'data' => ['load_time' => $load_time, 'fix' => 'Optimize server response time and resources'],
            ];
        } elseif ($load_time > 2.0) {
            $result['warnings'][] = [
                'type' => 'moderate_load_time',
                'severity' => 'medium',
                'message' => sprintf('Page load time could be improved (%.2fs)', $load_time),
                'data' => ['load_time' => $load_time],
            ];
        } else {
            $result['passed'][] = 'fast_load_time';
        }
        
        // Check page size
        $size_mb = $page_size / (1024 * 1024);
        if ($size_mb > 3.0) {
            $result['issues'][] = [
                'type' => 'large_page_size',
                'severity' => 'high',
                'message' => sprintf('Page size too large (%.1fMB, should be under 3MB)', $size_mb),
                'data' => ['size_mb' => $size_mb, 'fix' => 'Optimize images and remove unnecessary resources'],
            ];
        } elseif ($size_mb > 2.0) {
            $result['warnings'][] = [
                'type' => 'moderate_page_size',
                'severity' => 'medium',
                'message' => sprintf('Page size could be reduced (%.1fMB)', $size_mb),
                'data' => ['size_mb' => $size_mb],
            ];
        } else {
            $result['passed'][] = 'optimal_page_size';
        }
        
        // Check compression
        if (!$this->has_response_compression($headers)) {
            $result['issues'][] = [
                'type' => 'no_compression',
                'severity' => 'high',
                'message' => 'Response compression not enabled',
                'data' => ['fix' => 'Enable GZIP or Brotli compression on your server'],
            ];
        } else {
            $result['passed'][] = 'gzip_enabled';
        }
        
        // Check caching
        if (!$this->has_cache_headers($headers)) {
            $result['issues'][] = [
                'type' => 'no_cache_headers',
                'severity' => 'high',
                'message' => 'No cache headers set',
                'data' => ['fix' => 'Set appropriate Cache-Control headers'],
            ];
        } else {
            $result['passed'][] = 'cache_headers_set';
        }
        
        return $result;
    }

    /**
     * Normalize HTTP headers into a lowercase string-keyed array.
     *
     * @param mixed $headers Response headers from WP HTTP.
     * @return array<string,string>
     */
    private function normalize_response_headers($headers): array {
        if (is_object($headers) && method_exists($headers, 'getAll')) {
            $headers = $headers->getAll();
        } elseif (!is_array($headers)) {
            $headers = (array) $headers;
        }

        $normalized = [];
        foreach ($headers as $key => $value) {
            $header_key = strtolower((string) $key);
            if ($header_key === '') {
                continue;
            }

            if (is_array($value)) {
                $value = implode(', ', array_map('strval', $value));
            }

            $normalized[$header_key] = trim((string) $value);
        }

        return $normalized;
    }

    /**
     * Determine whether the response advertises compression.
     *
     * @param array<string,string> $headers
     */
    private function has_response_compression(array $headers): bool {
        $content_encoding = strtolower((string) ($headers['content-encoding'] ?? ''));
        if ($content_encoding !== '' && preg_match('/\b(gzip|br|deflate)\b/', $content_encoding)) {
            return true;
        }

        return false;
    }

    /**
     * Determine whether the response exposes meaningful cache metadata.
     *
     * @param array<string,string> $headers
     */
    private function has_cache_headers(array $headers): bool {
        $cache_control = strtolower((string) ($headers['cache-control'] ?? ''));
        if ($cache_control !== '') {
            return true;
        }

        foreach (['expires', 'etag', 'last-modified', 'age', 'cf-cache-status', 'x-cache', 'x-proxy-cache', 'surrogate-control'] as $header) {
            if (!empty($headers[$header])) {
                return true;
            }
        }

        return false;
    }
    
    /**
     * Check robots meta tag
     */
    private function check_robots_meta(\DOMDocument $dom): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        $xpath = new \DOMXPath($dom);
        $robots = $xpath->query('//meta[@name="robots"]');
        
        if ($robots->length > 0) {
            $content = $robots->item(0)->getAttribute('content');
            if (stripos($content, 'noindex') !== false) {
                $result['issues'][] = [
                    'type' => 'noindex_set',
                    'severity' => 'critical',
                    'message' => 'Page is set to noindex',
                    'data' => ['content' => $content, 'fix' => 'Remove noindex if you want this page in search results'],
                ];
            } else {
                $result['passed'][] = 'indexable';
            }
            
            if (stripos($content, 'nofollow') !== false) {
                $result['warnings'][] = [
                    'type' => 'nofollow_set',
                    'severity' => 'medium',
                    'message' => 'Page is set to nofollow',
                    'data' => ['content' => $content],
                ];
            }
        } else {
            $result['passed'][] = 'no_blocking_robots_meta';
        }
        
        return $result;
    }
    
    /**
     * Check hreflang tags
     */
    private function check_hreflang(\DOMDocument $dom): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        $xpath = new \DOMXPath($dom);
        $hreflangs = $xpath->query('//link[@rel="alternate"][@hreflang]');
        
        if ($hreflangs->length > 0) {
            $found_self_reference = false;
            $found_x_default = false;
            
            foreach ($hreflangs as $tag) {
                $hreflang = $tag->getAttribute('hreflang');
                if ($hreflang === 'x-default') {
                    $found_x_default = true;
                }
                // Check for self-referencing hreflang
                // This is simplified - in production would check actual URL match
            }
            
            if (!$found_x_default) {
                $result['warnings'][] = [
                    'type' => 'missing_x_default_hreflang',
                    'severity' => 'low',
                    'message' => 'No x-default hreflang tag found',
                    'data' => ['fix' => 'Add x-default hreflang for international targeting'],
                ];
            }
            
            $result['passed'][] = 'has_hreflang_tags';
        }
        
        return $result;
    }
    
    /**
     * Check sitemap reference
     */
    private function check_sitemap_reference(\DOMDocument $dom): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        $xpath = new \DOMXPath($dom);
        $sitemap_links = $xpath->query('//link[@rel="sitemap"]');
        
        if ($sitemap_links->length > 0) {
            $result['passed'][] = 'has_sitemap_reference';
        }
        
        // Note: Full sitemap check would verify robots.txt
        
        return $result;
    }
    
    /**
     * Check Twitter Cards
     */
    private function check_twitter_cards(\DOMDocument $dom): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        $xpath = new \DOMXPath($dom);
        
        $required_twitter_tags = ['twitter:card', 'twitter:title', 'twitter:description'];
        $missing = [];
        
        foreach ($required_twitter_tags as $name) {
            $tags = $xpath->query("//meta[@name='$name']");
            if ($tags->length === 0) {
                $missing[] = $name;
            }
        }
        
        if (count($missing) > 0) {
            $result['warnings'][] = [
                'type' => 'missing_twitter_cards',
                'severity' => 'low',
                'message' => 'Missing Twitter Card tags for better social sharing',
                'data' => ['missing' => $missing, 'fix' => 'Add Twitter Card meta tags'],
            ];
        } else {
            $result['passed'][] = 'has_twitter_cards';
        }
        
        return $result;
    }
    
    /**
     * Check keyword optimization
     */
    private function check_keyword_optimization(\DOMDocument $dom, string $body): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        
        // This is a simplified check - production would use NLP
        $text_content = wp_strip_all_tags($body);
        $word_count = str_word_count($text_content);
        
        if ($word_count > 0) {
            // Check keyword density (simplified)
            $result['passed'][] = 'has_content_for_keywords';
        }
        
        return $result;
    }
    
    /**
     * Check readability
     */
    private function check_readability(string $body): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        
        $text_content = wp_strip_all_tags($body);
        $sentences = preg_split('/[.!?]+/', $text_content, -1, PREG_SPLIT_NO_EMPTY);
        $avg_sentence_length = count($sentences) > 0 ? str_word_count($text_content) / count($sentences) : 0;
        
        if ($avg_sentence_length > 25) {
            $result['warnings'][] = [
                'type' => 'long_sentences',
                'severity' => 'low',
                'message' => sprintf('Average sentence length is high (%.1f words)', $avg_sentence_length),
                'data' => ['avg_length' => $avg_sentence_length, 'fix' => 'Use shorter sentences for better readability'],
            ];
        } else {
            $result['passed'][] = 'good_sentence_length';
        }
        
        return $result;
    }
    
    /**
     * Check mixed content
     */
    private function check_mixed_content(\DOMDocument $dom, string $base_url): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        
        if (strpos($base_url, 'https://') === 0) {
            // Check for HTTP resources on HTTPS page
            $xpath = new \DOMXPath($dom);
            $http_resources = $xpath->query('//*[@src[starts-with(., "http://")]] | //*[@href[starts-with(., "http://")]]');
            
            if ($http_resources->length > 0) {
                $result['issues'][] = [
                    'type' => 'mixed_content',
                    'severity' => 'high',
                    'message' => sprintf('Found %d HTTP resources on HTTPS page', $http_resources->length),
                    'data' => ['count' => $http_resources->length, 'fix' => 'Update all resources to use HTTPS'],
                ];
            } else {
                $result['passed'][] = 'no_mixed_content';
            }
        }
        
        return $result;
    }
    
    /**
     * Check mobile optimization
     */
    private function check_mobile_optimization(\DOMDocument $dom): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        $xpath = new \DOMXPath($dom);
        
        // Check viewport meta tag
        $viewport = $xpath->query('//meta[@name="viewport"]');
        if ($viewport->length === 0) {
            $result['issues'][] = [
                'type' => 'missing_viewport',
                'severity' => 'high',
                'message' => 'Missing viewport meta tag',
                'data' => ['fix' => 'Add <meta name="viewport" content="width=device-width, initial-scale=1">'],
            ];
        } else {
            $result['passed'][] = 'has_viewport_meta';
        }
        
        return $result;
    }
    
    /**
     * Check international SEO
     */
    private function check_international_seo(\DOMDocument $dom, $headers): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        
        // Check language declaration
        $html = $dom->getElementsByTagName('html')->item(0);
        if ($html && !$html->hasAttribute('lang')) {
            $result['warnings'][] = [
                'type' => 'missing_lang_attribute',
                'severity' => 'medium',
                'message' => 'Missing lang attribute on HTML element',
                'data' => ['fix' => 'Add lang attribute to specify page language'],
            ];
        } else {
            $result['passed'][] = 'has_lang_attribute';
        }
        
        return $result;
    }
    private function is_disallowed_by_robots(string $url): bool {
        $robots_url = trailingslashit(home_url()) . 'robots.txt';
        $robots = get_transient('prorank_audit_robots');

        if ($robots === false) {
            $resp = wp_remote_get($robots_url, ['timeout' => 4, 'sslverify' => false]);
            if (is_wp_error($resp) || wp_remote_retrieve_response_code($resp) >= 400) {
                set_transient('prorank_audit_robots', '', 15 * MINUTE_IN_SECONDS);
                return false;
            }
            $robots = wp_remote_retrieve_body($resp);
            set_transient('prorank_audit_robots', $robots, 15 * MINUTE_IN_SECONDS);
        }

        if (empty($robots)) {
            return false;
        }

        $lines = explode("
", (string) $robots);
        $apply = false;
        $disallows = [];
        foreach ($lines as $line) {
            $line = trim($line);
            if ($line === '' || strpos($line, '#') === 0) {
                continue;
            }
            if (stripos($line, 'User-agent:') === 0) {
                $ua = trim(substr($line, 11));
                $apply = ($ua === '*' || stripos($ua, 'prorank') !== false);
                continue;
            }
            if ($apply && stripos($line, 'Disallow:') === 0) {
                $path = trim(substr($line, 9));
                if ($path !== '') {
                    $disallows[] = $path;
                }
            }
        }

        $url_path = wp_parse_url($url, PHP_URL_PATH) ?: '/';
        foreach ($disallows as $path) {
            if ($path === '/') {
                return true;
            }
            if (strpos($url_path, $path) === 0) {
                return true;
            }
        }

        return false;
    }


    private function mark_url_checked(int $url_id): void {
        global $wpdb;
        $wpdb->update(
            $this->audit_urls_table,
            ['status' => 'checked'],
            ['id' => $url_id],
            ['%s'],
            ['%d']
        );
    }

    private function rate_limit_host(string $url): void {
        $host = wp_parse_url($url, PHP_URL_HOST);
        if (!$host) {
            return;
        }
        $key = 'prorank_audit_host_' . md5($host);
        $last = get_transient($key);
        $now = microtime(true);
        if ($last && ($now - (float)$last) < 0.35) {
            usleep(200000);
        }
        set_transient($key, $now, MINUTE_IN_SECONDS);
    }

    private function maybe_record_core_web_vitals(string $audit_id, string $url, $headers): void {
        $count_key = 'prorank_audit_vitals_count_' . $audit_id;
        $count = (int) get_option($count_key, 0);
        if ($count >= 10) {
            return;
        }

        $content_type = is_array($headers) && isset($headers['content-type']) ? (string) $headers['content-type'] : '';
        if ($content_type && stripos($content_type, 'text/html') === false) {
            return;
        }

        if (!class_exists('\ProRank\SEO\Core\Services\CoreWebVitalsService')) {
            return;
        }

        $service = new \ProRank\SEO\Core\Services\CoreWebVitalsService();
        $vitals = $service->get_vitals($url, 'mobile');
        if (is_wp_error($vitals)) {
            return;
        }

        $data_source = !empty($vitals['field_data']) ? $vitals['field_data'] : ($vitals['lab_data'] ?? []);
        if (empty($data_source)) {
            return;
        }

        $metrics = [];
        foreach (['lcp', 'cls', 'inp', 'ttfb'] as $metric) {
            if (isset($data_source[$metric])) {
                $value = $data_source[$metric]['value'] ?? 0;
                $category = strtolower($data_source[$metric]['category'] ?? $this->get_metric_rating($metric, $value));
                $metrics[$metric] = [
                    'value' => $value,
                    'rating' => $category,
                ];
            }
        }

        if (!empty($metrics)) {
            foreach ($metrics as $metric => $meta) {
                if ($meta['rating'] === 'poor') {
                    $this->add_issue($audit_id, $url, 'core_web_vitals_' . $metric, strtoupper($metric) . ' is poor', 'high');
                } elseif ($meta['rating'] === 'needs-improvement') {
                    $this->add_issue($audit_id, $url, 'core_web_vitals_' . $metric, strtoupper($metric) . ' needs improvement', 'medium');
                }
            }
        }

        update_option($count_key, $count + 1, false);
    }

    private function get_metric_rating(string $metric, float $value): string {
        $thresholds = [
            'lcp' => ['good' => 2500, 'poor' => 4000],
            'cls' => ['good' => 0.1, 'poor' => 0.25],
            'inp' => ['good' => 200, 'poor' => 500],
            'ttfb' => ['good' => 800, 'poor' => 1800],
        ];

        if (!isset($thresholds[$metric])) {
            return 'unknown';
        }

        if ($value <= $thresholds[$metric]['good']) {
            return 'good';
        }
        if ($value <= $thresholds[$metric]['poor']) {
            return 'needs-improvement';
        }
        return 'poor';
    }

    /**
     * Check XML Sitemap - validates sitemap exists and is properly formatted
     */
    private function check_xml_sitemap(): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        $base_urls = $this->get_configured_site_urls();

        // Check multiple common sitemap paths (WordPress core, Yoast, RankMath, etc.)
        $sitemap_paths = [
            'sitemap.xml',
            'wp-sitemap.xml',
            'sitemap_index.xml',
        ];

        $found_sitemap = false;
        $checked_urls = [];

        foreach ($base_urls as $base_url) {
            $base = trailingslashit($base_url);
            foreach ($sitemap_paths as $path) {
                $sitemap_url = $base . $path;
                if (in_array($sitemap_url, $checked_urls, true)) {
                    continue;
                }

                $checked_urls[] = $sitemap_url;
                $response = wp_remote_get($sitemap_url, [
                    'timeout' => 10,
                    'sslverify' => false, // Avoid false failures on self-signed or internal certs
                    'redirection' => 5,   // Follow redirects (sitemap.xml often 301s to wp-sitemap.xml)
                ]);

                if (is_wp_error($response)) {
                    continue;
                }

                $status_code = wp_remote_retrieve_response_code($response);
                if ($status_code === 200) {
                    $body = wp_remote_retrieve_body($response);
                    libxml_use_internal_errors(true);
                    $xml = simplexml_load_string($body);
                    libxml_clear_errors();
                    if ($xml !== false) {
                        $found_sitemap = true;
                        break 2;
                    }
                }
            }
        }

        if (!$found_sitemap) {
            $result['issues'][] = ['type' => 'sitemap_missing', 'severity' => 'high', 'message' => 'No XML sitemap found', 'data' => ['checked' => $checked_urls ?: $sitemap_paths]];
        } else {
            $result['passed'][] = 'has_valid_sitemap';
        }

        return $result;
    }

    private function check_thin_content(\DOMDocument $dom, string $body): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        $text_content = wp_strip_all_tags($body);
        $text_content = preg_replace('/\s+/', ' ', trim($text_content));
        $word_count = str_word_count($text_content);

        if ($word_count < 100) {
            $result['issues'][] = ['type' => 'very_thin_content', 'severity' => 'high', 'message' => sprintf('Very thin content: %d words', $word_count), 'data' => ['word_count' => $word_count]];
        } elseif ($word_count < 300) {
            $result['warnings'][] = ['type' => 'thin_content', 'severity' => 'medium', 'message' => sprintf('Thin content: %d words', $word_count), 'data' => ['word_count' => $word_count]];
        } else {
            $result['passed'][] = 'adequate_content_length';
        }
        return $result;
    }

    private function check_duplicate_content(string $audit_id, string $url, \DOMDocument $dom, string $body): array {
        global $wpdb;
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        $text_content = wp_strip_all_tags($body);
        $normalized = preg_replace('/\s+/', ' ', strtolower(trim($text_content)));
        $normalized = preg_replace('/[^a-z0-9\s]/', '', $normalized);
        if (strlen($normalized) < 100) return $result;

        $content_hash = md5($normalized);
        $meta_table = $wpdb->prefix . 'prorank_audit_meta';
        $this->maybe_create_meta_table();

        $existing = $wpdb->get_var($wpdb->prepare(
            "SELECT meta_value FROM {$meta_table} WHERE audit_id = %s AND meta_key = 'content_hash' AND meta_value = %s",
            $audit_id, $content_hash
        ));

        if ($existing) {
            $result['issues'][] = ['type' => 'duplicate_content', 'severity' => 'high', 'message' => 'Duplicate content detected', 'data' => []];
        } else {
            $wpdb->insert($meta_table, ['audit_id' => $audit_id, 'url' => $url, 'meta_key' => 'content_hash', 'meta_value' => $content_hash]);
            $result['passed'][] = 'unique_content';
        }
        return $result;
    }

    private function check_crawl_depth(string $audit_id, int $url_id): array {
        global $wpdb;
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        $depth = (int) $wpdb->get_var($wpdb->prepare("SELECT depth FROM {$this->audit_urls_table} WHERE id = %d", $url_id));

        if ($depth > 4) {
            $result['issues'][] = ['type' => 'deep_page', 'severity' => 'medium', 'message' => sprintf('Page is %d clicks from homepage', $depth), 'data' => ['depth' => $depth]];
        } elseif ($depth > 3) {
            $result['warnings'][] = ['type' => 'moderately_deep_page', 'severity' => 'low', 'message' => sprintf('Page is %d clicks deep', $depth), 'data' => ['depth' => $depth]];
        } else {
            $result['passed'][] = 'good_crawl_depth';
        }
        return $result;
    }

    private function check_orphaned_pages(string $audit_id): void {
        global $wpdb;
        $meta_table = $wpdb->prefix . 'prorank_audit_meta';
        $this->maybe_create_meta_table();

        $link_targets = $wpdb->get_col($wpdb->prepare(
            "SELECT DISTINCT meta_value FROM {$meta_table} WHERE audit_id = %s AND meta_key = 'internal_link_target'",
            $audit_id
        ));
        $audited_urls = $wpdb->get_results($wpdb->prepare(
            "SELECT id, url FROM {$this->audit_urls_table} WHERE audit_id = %s",
            $audit_id
        ));
        $home_url = home_url();

        // Build a normalized set of link targets for fast O(1) lookup
        $normalized_targets = [];
        foreach ($link_targets as $target) {
            // Normalize: lowercase, strip trailing slash, strip protocol for comparison
            $norm = strtolower(rtrim($target, '/'));
            $norm = preg_replace('#^https?://#', '', $norm);
            $normalized_targets[$norm] = true;
        }

        foreach ($audited_urls as $page) {
            // Skip homepage and site-level check URL
            if (rtrim($page->url, '/') === rtrim($home_url, '/')) continue;
            if (strpos($page->url, '?_site_level_checks') !== false) continue;

            // Normalize the page URL the same way
            $normalized_url = strtolower(rtrim($page->url, '/'));
            $normalized_url = preg_replace('#^https?://#', '', $normalized_url);

            if (!isset($normalized_targets[$normalized_url])) {
                $this->add_issue($audit_id, $page->url, 'orphaned_page', 'Orphaned page - no internal links point here', 'medium');
            }
        }
    }

    private function store_internal_links_for_orphan_detection(string $audit_id, \DOMDocument $dom, string $base_url): void {
        $this->maybe_create_meta_table();
        $home_host = strtolower(wp_parse_url(home_url(), PHP_URL_HOST));
        $links = $dom->getElementsByTagName('a');
        $stored = [];
        global $wpdb;
        $meta_table = $wpdb->prefix . 'prorank_audit_meta';

        foreach ($links as $link) {
            $href = $link->getAttribute('href');
            if (empty($href) || $href[0] === '#' || strpos($href, 'javascript:') === 0 || strpos($href, 'mailto:') === 0) continue;
            $absolute_url = $this->make_absolute_url($href, $base_url);
            if (!$absolute_url) continue;
            $link_host = strtolower(wp_parse_url($absolute_url, PHP_URL_HOST) ?? '');
            if ($link_host === $home_host) {
                // Normalize: remove fragment + query string, then trailing slash
                $normalized = strtok($absolute_url, '#');
                $normalized = strtok($normalized, '?');
                $normalized = rtrim($normalized, '/');
                if (!isset($stored[$normalized])) {
                    $wpdb->insert($meta_table, ['audit_id' => $audit_id, 'url' => '', 'meta_key' => 'internal_link_target', 'meta_value' => $normalized]);
                    $stored[$normalized] = true;
                }
            }
        }
    }

    private function check_wordpress_health(): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        global $wp_version;
        $latest = get_site_transient('update_core');
        if ($latest && isset($latest->updates[0]->version) && version_compare($wp_version, $latest->updates[0]->version, '<')) {
            $result['warnings'][] = ['type' => 'wordpress_outdated', 'severity' => 'medium', 'message' => sprintf('WordPress %s outdated', $wp_version), 'data' => []];
        } else {
            $result['passed'][] = 'wordpress_up_to_date';
        }
        if (version_compare(phpversion(), '8.0', '<')) {
            $result['warnings'][] = ['type' => 'php_outdated', 'severity' => 'medium', 'message' => sprintf('PHP %s outdated', phpversion()), 'data' => []];
        } else {
            $result['passed'][] = 'php_up_to_date';
        }
        $uses_https = false;
        foreach ($this->get_configured_site_urls() as $site_url) {
            if (wp_parse_url($site_url, PHP_URL_SCHEME) === 'https') {
                $uses_https = true;
                break;
            }
        }

        if (!$uses_https) {
            $result['issues'][] = ['type' => 'no_ssl', 'severity' => 'high', 'message' => 'Site not using HTTPS', 'data' => []];
        } else {
            $result['passed'][] = 'ssl_enabled';
        }
        return $result;
    }

    private function check_plugin_conflicts(): array {
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];

        if (!function_exists('get_plugins')) {
            require_once ABSPATH . 'wp-admin/includes/plugin.php';
        }

        $all_plugins = get_plugins();
        $active_plugins = get_option('active_plugins', []);

        // Known problematic plugins or combinations
        $problematic_plugins = [
            // SEO plugins
            'wordpress-seo/wp-seo.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank metadata, schema, and sitemaps'],
            'wordpress-seo-premium/wp-seo-premium.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank metadata, schema, and sitemaps'],
            'wpseo-local/wpseo-local.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank local SEO features'],
            'wpseo-video/wpseo-video.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank video schema'],
            'wpseo-news/wpseo-news.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank news sitemaps'],
            'woocommerce-seo/woocommerce-seo.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank WooCommerce SEO'],
            'wpseo-woocommerce/wpseo-woocommerce.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank WooCommerce SEO'],
            'seo-by-rank-math/rank-math.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank metadata, schema, and sitemaps'],
            'seo-by-rank-math-pro/rank-math-pro.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank metadata, schema, and sitemaps'],
            'all-in-one-seo-pack/all_in_one_seo_pack.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank metadata, schema, and sitemaps'],
            'all-in-one-seo-pack-pro/all_in_one_seo_pack.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank metadata, schema, and sitemaps'],
            'wp-seopress/seopress.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank metadata, schema, and sitemaps'],
            'wp-seopress-pro/seopress-pro.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank metadata, schema, and sitemaps'],
            'autodescription/autodescription.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank metadata and schema'],
            'slim-seo/slim-seo.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank metadata and schema'],
            'squirrly-seo/squirrly.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank SEO features'],
            'smartcrawl-seo/wpmu-dev-seo.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank SEO features'],
            'seokey/seokey.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank SEO features'],
            'seo-ultimate/seo-ultimate.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank SEO features'],
            'wp-meta-seo/wp-meta-seo.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank SEO features'],
            'premium-seo-pack/index.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank SEO features'],
            'seo-booster/seo-booster.php' => ['severity' => 'medium', 'reason' => 'Another SEO plugin - may conflict with ProRank SEO features'],

            // Performance/caching plugins
            'w3-total-cache/w3-total-cache.php' => ['severity' => 'low', 'reason' => 'Caching/optimization plugin - may overlap with ProRank performance features'],
            'wp-rocket/wp-rocket.php' => ['severity' => 'low', 'reason' => 'Caching/optimization plugin - may overlap with ProRank performance features'],
            'litespeed-cache/litespeed-cache.php' => ['severity' => 'low', 'reason' => 'Caching/optimization plugin - may overlap with ProRank performance features'],
            'wp-super-cache/wp-cache.php' => ['severity' => 'low', 'reason' => 'Caching/optimization plugin - may overlap with ProRank performance features'],
            'wp-fastest-cache/wpFastestCache.php' => ['severity' => 'low', 'reason' => 'Caching/optimization plugin - may overlap with ProRank performance features'],
            'cache-enabler/cache-enabler.php' => ['severity' => 'low', 'reason' => 'Caching/optimization plugin - may overlap with ProRank performance features'],
            'autoptimize/autoptimize.php' => ['severity' => 'low', 'reason' => 'Caching/optimization plugin - may overlap with ProRank performance features'],
            'hummingbird-performance/wp-hummingbird.php' => ['severity' => 'low', 'reason' => 'Caching/optimization plugin - may overlap with ProRank performance features'],
            'sg-cachepress/sg-cachepress.php' => ['severity' => 'low', 'reason' => 'Caching/optimization plugin - may overlap with ProRank performance features'],
            'nitropack/nitropack.php' => ['severity' => 'low', 'reason' => 'Caching/optimization plugin - may overlap with ProRank performance features'],
            'perfmatters/perfmatters.php' => ['severity' => 'low', 'reason' => 'Performance plugin - may overlap with ProRank optimizations'],
            'flying-press/flying-press.php' => ['severity' => 'low', 'reason' => 'Performance plugin - may overlap with ProRank optimizations'],
            'asset-cleanup/asset-cleanup.php' => ['severity' => 'low', 'reason' => 'Asset optimization plugin - may overlap with ProRank asset controls'],
            'async-javascript/async-javascript.php' => ['severity' => 'low', 'reason' => 'Asset optimization plugin - may overlap with ProRank asset controls'],
            'wp-optimize/wp-optimize.php' => ['severity' => 'low', 'reason' => 'Optimization plugin - may overlap with ProRank performance features'],
            'breeze/breeze.php' => ['severity' => 'low', 'reason' => 'Caching/optimization plugin - may overlap with ProRank performance features'],

            // Image optimization plugins
            'shortpixel-image-optimiser/wp-shortpixel.php' => ['severity' => 'low', 'reason' => 'Image optimization plugin - may overlap with ProRank image optimization'],
            'imagify/imagify.php' => ['severity' => 'low', 'reason' => 'Image optimization plugin - may overlap with ProRank image optimization'],
            'wp-smushit/wp-smush.php' => ['severity' => 'low', 'reason' => 'Image optimization plugin - may overlap with ProRank image optimization'],
            'ewww-image-optimizer/ewww-image-optimizer.php' => ['severity' => 'low', 'reason' => 'Image optimization plugin - may overlap with ProRank image optimization'],
            'optimole-wp/optimole-wp.php' => ['severity' => 'low', 'reason' => 'Image optimization plugin - may overlap with ProRank image optimization'],
            'tiny-compress-images/tiny-compress-images.php' => ['severity' => 'low', 'reason' => 'Image optimization plugin - may overlap with ProRank image optimization'],
            'webp-express/webp-express.php' => ['severity' => 'low', 'reason' => 'Image optimization plugin - may overlap with ProRank image optimization'],
            'converter-for-media/converter-for-media.php' => ['severity' => 'low', 'reason' => 'Image optimization plugin - may overlap with ProRank image optimization'],
            'seo-optimized-images/seo-optimized-images.php' => ['severity' => 'low', 'reason' => 'Image optimization plugin - may overlap with ProRank image optimization'],
        ];

        $conflicts_found = 0;

        foreach ($active_plugins as $plugin) {
            if (isset($problematic_plugins[$plugin])) {
                $info = $problematic_plugins[$plugin];
                $plugin_data = $all_plugins[$plugin] ?? ['Name' => $plugin];

                $result['warnings'][] = [
                    'type' => 'potential_plugin_conflict',
                    'severity' => $info['severity'],
                    'message' => sprintf('Potential conflict: %s', $plugin_data['Name']),
                    'data' => ['plugin' => $plugin, 'reason' => $info['reason']],
                ];
                $conflicts_found++;
            }
        }

        // Check for plugins needing updates
        $update_plugins = get_site_transient('update_plugins');
        $outdated_count = 0;

        if ($update_plugins && !empty($update_plugins->response)) {
            $outdated_count = count($update_plugins->response);
            if ($outdated_count > 0) {
                $result['warnings'][] = [
                    'type' => 'plugins_need_updates',
                    'severity' => 'medium',
                    'message' => sprintf('%d plugin(s) have updates available', $outdated_count),
                    'data' => ['count' => $outdated_count, 'fix' => 'Update plugins to latest versions'],
                ];
            }
        }

        // Check plugin count
        $active_count = count($active_plugins);
        if ($active_count > 30) {
            $result['warnings'][] = [
                'type' => 'too_many_plugins',
                'severity' => 'low',
                'message' => sprintf('%d active plugins (consider reducing for performance)', $active_count),
                'data' => ['count' => $active_count],
            ];
        }

        if ($conflicts_found === 0 && $outdated_count === 0) {
            $result['passed'][] = 'no_plugin_issues';
        }

        return $result;
    }

    private function check_database_optimization(): array {
        global $wpdb;
        $result = ['issues' => [], 'warnings' => [], 'passed' => []];
        $autoload = $wpdb->get_var("SELECT SUM(LENGTH(option_value)) FROM {$wpdb->options} WHERE autoload = 'yes'");
        $autoload_mb = $autoload / (1024 * 1024);
        if ($autoload_mb > 2) {
            $result['issues'][] = ['type' => 'large_autoload', 'severity' => 'high', 'message' => sprintf('Autoload %.1fMB', $autoload_mb), 'data' => []];
        } elseif ($autoload_mb > 1) {
            $result['warnings'][] = ['type' => 'moderate_autoload', 'severity' => 'medium', 'message' => sprintf('Autoload %.1fMB', $autoload_mb), 'data' => []];
        } else {
            $result['passed'][] = 'optimal_autoload_size';
        }
        $revisions = $wpdb->get_var("SELECT COUNT(*) FROM {$wpdb->posts} WHERE post_type = 'revision'");
        if ($revisions > 1000) {
            $result['warnings'][] = ['type' => 'many_revisions', 'severity' => 'low', 'message' => sprintf('%d revisions', $revisions), 'data' => []];
        }
        return $result;
    }

    private function maybe_create_meta_table(): void {
        global $wpdb;
        $meta_table = $wpdb->prefix . 'prorank_audit_meta';
        if ($wpdb->get_var("SHOW TABLES LIKE '{$meta_table}'") === $meta_table) return;
        $charset = $wpdb->get_charset_collate();
        $sql = "CREATE TABLE {$meta_table} (
            id bigint(20) unsigned NOT NULL AUTO_INCREMENT,
            audit_id varchar(50) NOT NULL,
            url varchar(500) DEFAULT '',
            meta_key varchar(255) NOT NULL,
            meta_value longtext,
            PRIMARY KEY (id),
            KEY audit_id (audit_id),
            KEY meta_key (meta_key(191))
        ) $charset;";
        require_once(ABSPATH . 'wp-admin/includes/upgrade.php');
        dbDelta($sql);
    }

    /**
     * Add issue to database
     */
    private function add_issue(string $audit_id, string $url, string $type, string $message, string $severity, array $data = []): void {
        global $wpdb;

        // Get URL ID
        $url_id = $wpdb->get_var($wpdb->prepare(
            "SELECT id FROM {$this->audit_urls_table} WHERE audit_id = %s AND url = %s LIMIT 1",
            $audit_id,
            $url
        ));

        if (!$url_id) {
            return;
        }

        $wpdb->insert($this->audit_issues_table, [
            'audit_id' => $audit_id,
            'url_id' => $url_id,
            'type' => $type,
            'severity' => $severity,
            'message' => $message,
            'data' => json_encode($data),
            'created_at' => current_time('mysql'),
        ], ['%s', '%d', '%s', '%s', '%s', '%s', '%s']);
    }

    /**
     * Enhanced structured data validation
     */
    private function check_structured_data_enhanced(string $audit_id, string $url, string $body): void {
        // Find all JSON-LD scripts
        if (!preg_match_all('/<script[^>]+type=["\']application\/ld\+json["\'][^>]*>(.*?)<\/script>/si', $body, $matches)) {
            $this->add_issue($audit_id, $url, 'missing_structured_data', 'No JSON-LD structured data found', 'medium');
            return;
        }

        $found_types = [];
        $has_errors = false;

        foreach ($matches[1] as $json_content) {
            $data = json_decode(trim($json_content), true);

            if (json_last_error() !== JSON_ERROR_NONE) {
                $this->add_issue($audit_id, $url, 'invalid_json_ld', 'Invalid JSON-LD: ' . json_last_error_msg(), 'high');
                $has_errors = true;
                continue;
            }

            // Handle @graph structures
            $items = isset($data['@graph']) ? $data['@graph'] : [$data];

            foreach ($items as $item) {
                if (isset($item['@type'])) {
                    $type = is_array($item['@type']) ? implode(', ', $item['@type']) : $item['@type'];
                    $found_types[] = $type;

                    // Validate required fields for common types
                    $this->validate_schema_type($audit_id, $url, $type, $item);
                }
            }
        }

        if (!$has_errors && !empty($found_types)) {
            // Structured data is present and valid
            return;
        }
    }

    /**
     * Validate specific schema types
     */
    private function validate_schema_type(string $audit_id, string $url, string $type, array $data): void {
        $required_fields = [
            'Article' => ['headline', 'author', 'datePublished'],
            'Product' => ['name', 'description'],
            'Organization' => ['name', 'url'],
            'LocalBusiness' => ['name', 'address'],
            'BreadcrumbList' => ['itemListElement'],
            'FAQPage' => ['mainEntity'],
            'HowTo' => ['name', 'step'],
            'Recipe' => ['name', 'recipeIngredient', 'recipeInstructions'],
        ];

        if (!isset($required_fields[$type])) {
            return;
        }

        $missing = [];
        foreach ($required_fields[$type] as $field) {
            if (!isset($data[$field]) || empty($data[$field])) {
                $missing[] = $field;
            }
        }

        if (!empty($missing)) {
            $this->add_issue(
                $audit_id,
                $url,
                'incomplete_schema_' . strtolower($type),
                sprintf('%s schema missing required fields: %s', $type, implode(', ', $missing)),
                'medium',
                ['type' => $type, 'missing_fields' => $missing]
            );
        }
    }

    /**
     * Check security headers
     */
    private function check_security_headers(string $audit_id, string $url, $headers): void {
        $security_headers = [
            'x-frame-options' => ['severity' => 'medium', 'message' => 'Missing X-Frame-Options header (clickjacking protection)'],
            'x-content-type-options' => ['severity' => 'medium', 'message' => 'Missing X-Content-Type-Options header'],
            'strict-transport-security' => ['severity' => 'high', 'message' => 'Missing Strict-Transport-Security (HSTS) header'],
            'content-security-policy' => ['severity' => 'low', 'message' => 'Missing Content-Security-Policy header'],
            'referrer-policy' => ['severity' => 'low', 'message' => 'Missing Referrer-Policy header'],
            'permissions-policy' => ['severity' => 'low', 'message' => 'Missing Permissions-Policy header'],
        ];

        $headers_lower = $this->normalize_response_headers($headers);

        foreach ($security_headers as $header => $config) {
            if (!isset($headers_lower[$header])) {
                $this->add_issue($audit_id, $url, 'missing_security_header_' . str_replace('-', '_', $header), $config['message'], $config['severity']);
            }
        }
    }

    /**
     * Check redirect chains
     */
    private function check_redirect_chain(string $audit_id, string $original_url): void {
        $redirects = [];
        $current_url = $original_url;
        $max_redirects = 10;

        for ($i = 0; $i < $max_redirects; $i++) {
            $response = wp_remote_head($current_url, [
                'timeout' => 5,
                'redirection' => 0, // Don't follow redirects
            ]);

            if (is_wp_error($response)) {
                break;
            }

            $code = wp_remote_retrieve_response_code($response);

            if ($code >= 300 && $code < 400) {
                $location = wp_remote_retrieve_header($response, 'location');
                if ($location) {
                    $redirects[] = ['from' => $current_url, 'to' => $location, 'code' => $code];
                    $current_url = $location;
                } else {
                    break;
                }
            } else {
                break;
            }
        }

        $chain_length = count($redirects);

        if ($chain_length >= 3) {
            $this->add_issue(
                $audit_id,
                $original_url,
                'long_redirect_chain',
                sprintf('Long redirect chain detected (%d redirects)', $chain_length),
                'high',
                ['chain' => $redirects]
            );
        } elseif ($chain_length === 2) {
            $this->add_issue(
                $audit_id,
                $original_url,
                'redirect_chain',
                'Redirect chain detected (2 redirects)',
                'medium',
                ['chain' => $redirects]
            );
        }

        // Check for redirect loops
        $urls_seen = array_column($redirects, 'from');
        if (count($urls_seen) !== count(array_unique($urls_seen))) {
            $this->add_issue($audit_id, $original_url, 'redirect_loop', 'Redirect loop detected', 'critical', ['chain' => $redirects]);
        }
    }

    /**
     * Enhanced hreflang validation
     */
    private function check_hreflang_enhanced(string $audit_id, string $url, \DOMDocument $dom): void {
        $xpath = new \DOMXPath($dom);
        $hreflangs = $xpath->query('//link[@rel="alternate"][@hreflang]');

        if ($hreflangs->length === 0) {
            return; // No hreflang - not necessarily an issue
        }

        $has_self = false;
        $has_x_default = false;
        $lang_codes = [];
        $issues = [];

        foreach ($hreflangs as $tag) {
            $hreflang = strtolower($tag->getAttribute('hreflang'));
            $href = $tag->getAttribute('href');

            if ($hreflang === 'x-default') {
                $has_x_default = true;
            }

            // Check for self-referencing
            if ($this->urls_match($href, $url)) {
                $has_self = true;
            }

            // Check for duplicate language codes
            if (isset($lang_codes[$hreflang])) {
                $issues[] = "Duplicate hreflang: $hreflang";
            }
            $lang_codes[$hreflang] = $href;

            // Validate language code format
            if ($hreflang !== 'x-default' && !preg_match('/^[a-z]{2}(-[a-z]{2})?$/i', $hreflang)) {
                $issues[] = "Invalid hreflang format: $hreflang";
            }
        }

        if (!$has_self && $hreflangs->length > 0) {
            $this->add_issue($audit_id, $url, 'missing_self_hreflang', 'Hreflang tags present but missing self-referencing tag', 'medium');
        }

        if (!$has_x_default && $hreflangs->length > 1) {
            $this->add_issue($audit_id, $url, 'missing_x_default_hreflang', 'Multiple hreflang tags but no x-default', 'low');
        }

        foreach ($issues as $issue) {
            $this->add_issue($audit_id, $url, 'hreflang_error', $issue, 'medium');
        }
    }

    /**
     * Check if two URLs match (ignoring trailing slashes and protocol)
     */
    private function urls_match(string $url1, string $url2): bool {
        $normalize = function($url) {
            $url = preg_replace('/^https?:\/\//', '', $url);
            $url = rtrim($url, '/');
            return strtolower($url);
        };

        return $normalize($url1) === $normalize($url2);
    }

    /**
     * Enhanced canonical check
     */
    private function check_canonical_enhanced(string $audit_id, string $url, \DOMDocument $dom): void {
        $xpath = new \DOMXPath($dom);
        $canonicals = $xpath->query('//link[@rel="canonical"]');

        if ($canonicals->length === 0) {
            $this->add_issue($audit_id, $url, 'missing_canonical', 'Missing canonical tag', 'medium');
            return;
        }

        if ($canonicals->length > 1) {
            $this->add_issue($audit_id, $url, 'multiple_canonicals', 'Multiple canonical tags found', 'high');
            return;
        }

        $canonical_href = $canonicals->item(0)->getAttribute('href');

        if (empty($canonical_href)) {
            $this->add_issue($audit_id, $url, 'empty_canonical', 'Canonical tag has empty href', 'high');
            return;
        }

        // Check if canonical points to a different page
        if (!$this->urls_match($canonical_href, $url)) {
            // This could be intentional (for duplicate content), just note it
            $this->add_issue(
                $audit_id,
                $url,
                'canonical_different_url',
                'Canonical points to different URL: ' . $canonical_href,
                'low',
                ['canonical' => $canonical_href]
            );
        }

        // Check if canonical URL is absolute
        if (!preg_match('/^https?:\/\//', $canonical_href)) {
            $this->add_issue($audit_id, $url, 'relative_canonical', 'Canonical URL should be absolute', 'medium');
        }
    }

}
