<?php
/* GENERATED FIX: actual values lookback/backfill news fetcher */
/**
 * fetch_news_events.php
 * ------------------------------------------------------------
 * Institutional-grade multi-source forex news fetcher.
 *
 * Provider priority:
 * 1. ForexFactory calendar scraping (primary)
 * 2. TradingEconomics API fallback/enrichment, if configured
 * 3. FinancialModelingPrep fallback/enrichment, if subscription allows
 * 4. RSS macro feeds fallback for unscheduled macro/speech/headline risk
 *
 * IMPORTANT:
 * This file DOES NOT block trading.
 * It only fetches, normalizes, classifies, stores and logs news events.
 *
 * Trading behavior such as:
 * - allow trading during red-folder news
 * - show warning only
 * - reduce confidence
 * - user toggle on/off
 * belongs in NewsRiskService.php / dashboard / signal engine.
 */

ini_set('display_errors', 1);
ini_set('display_startup_errors', 1);
error_reporting(E_ALL);

date_default_timezone_set('Africa/Lagos');

require_once __DIR__ . '/../config/database.php';
require_once __DIR__ . '/../services/SystemConfigService.php';

final class FetchNewsEventsCron
{

    private static function titleContainsSafe(string $haystack, string $needle): bool
    {
        return $needle !== '' && stripos($haystack, $needle) !== false;
    }

    private static function knownUsdReleaseTimestamp(string $title, string $date, string $currency = 'USD'): ?int
    {
        $title = strtoupper($title);
        $currency = strtoupper($currency);

        if ($currency !== 'USD') {
            return null;
        }

        $date = trim($date);
        if ($date === '') {
            return null;
        }

        $year = date('Y');
        $dateCandidates = [$date, $date . ' ' . $year, str_replace(['-', '.'], '/', $date)];

        $nyTime = null;

        if (
            self::titleContainsSafe($title, 'CORE PCE PRICE INDEX')
            || self::titleContainsSafe($title, 'PCE PRICE INDEX')
            || self::titleContainsSafe($title, 'PERSONAL INCOME')
            || self::titleContainsSafe($title, 'PERSONAL SPENDING')
            || self::titleContainsSafe($title, 'UNEMPLOYMENT CLAIMS')
            || self::titleContainsSafe($title, 'JOBLESS CLAIMS')
            || self::titleContainsSafe($title, 'CPI')
            || self::titleContainsSafe($title, 'PPI')
            || self::titleContainsSafe($title, 'NONFARM')
            || self::titleContainsSafe($title, 'NFP')
            || self::titleContainsSafe($title, 'GDP')
            || self::titleContainsSafe($title, 'RETAIL SALES')
            || self::titleContainsSafe($title, 'DURABLE GOODS')
        ) {
            $nyTime = '08:30:00';
        } elseif (
            self::titleContainsSafe($title, 'ISM')
            || self::titleContainsSafe($title, 'PMI')
            || self::titleContainsSafe($title, 'CONSUMER CONFIDENCE')
            || self::titleContainsSafe($title, 'JOLTS')
            || self::titleContainsSafe($title, 'NEW HOME SALES')
            || self::titleContainsSafe($title, 'EXISTING HOME SALES')
            || self::titleContainsSafe($title, 'PENDING HOME SALES')
            || self::titleContainsSafe($title, 'FACTORY ORDERS')
        ) {
            $nyTime = '10:00:00';
        }

        if ($nyTime === null) {
            return null;
        }

        foreach ($dateCandidates as $dateCandidate) {
            try {
                $dt = new DateTime(trim($dateCandidate) . ' ' . $nyTime, new DateTimeZone('America/New_York'));
                return (int)$dt->getTimestamp();
            } catch (Throwable $e) {
                // Try next date format.
            }
        }

        return null;
    }



    private static function containsTextSafe(string $haystack, string $needle): bool
    {
        return $needle !== '' && stripos($haystack, $needle) !== false;
    }


    private const LOCAL_TZ = 'Africa/Lagos';
    private const LOG_FILE = 'news_engine.log';

    /**
     * Scheduled calendar impact must match the source calendar folder:
     * LOW    = yellow folder / Low Impact Expected
     * MEDIUM = orange or semi-red folder / Medium Impact Expected
     * HIGH   = red folder / High Impact Expected
     *
     * Do NOT store CRITICAL as calendar impact. CRITICAL/EXTREME may be
     * used later by the risk engine, but it must not replace the source
     * calendar impact level.
     */
    private const STORE_IMPACTS = ['LOW', 'MEDIUM', 'HIGH'];

    public static function run(): void
    {
        self::outputHeader();
        self::log('==============================');
        self::log('Starting institutional multi-source news fetch.');

        self::ensureNewsTableExists();

        $providerPriority = [
            'forexfactory',
            'tradingeconomics',
            'fmp',
            'rss_macro',
        ];

        $results = [];
        $totalSaved = 0;

        foreach ($providerPriority as $provider) {
            try {
                self::log("Running provider: {$provider}");

                $rawEvents = match ($provider) {
                    'forexfactory' => self::fetchForexFactory(),
                    'tradingeconomics' => self::fetchTradingEconomics(),
                    'fmp' => self::fetchFmp(),
                    'rss_macro' => self::fetchRssMacro(),
                    default => [],
                };

                $rawActualCount = 0;
                foreach ($rawEvents as $eventForRawCount) {
                    if (is_array($eventForRawCount) && !empty($eventForRawCount['actual'])) {
                        $rawActualCount++;
                    }
                }

                $normalized = self::normalizeEvents($rawEvents, $provider);
                $saved = self::saveEvents($normalized);

                if ($rawActualCount > 0) {
                    self::backfillActualsFromRawEvents($rawEvents);
                }

                $actualCount = 0;
                foreach ($normalized as $eventForCount) {
                    if (!empty($eventForCount['actual'])) {
                        $actualCount++;
                    }
                }

                $results[$provider] = [
                    'raw' => count($rawEvents),
                    'normalized' => count($normalized),
                    'saved' => $saved,
                    'raw_actual_values' => $rawActualCount,
                    'actual_values' => $actualCount,
                ];

                $totalSaved += $saved;

                self::log("Provider {$provider}: raw=" . count($rawEvents) . ", normalized=" . count($normalized) . ", saved={$saved}, raw_actual_values={$rawActualCount}, actual_values={$actualCount}");
            } catch (Throwable $e) {
                $results[$provider] = [
                    'raw' => 0,
                    'normalized' => 0,
                    'saved' => 0,
                    'error' => $e->getMessage(),
                ];

                self::log("Provider {$provider} failed: " . $e->getMessage());
            }
        }

        self::cleanupOldEvents();

        self::log("Finished news fetch. Total saved/updated: {$totalSaved}");

        echo "SUCCESS\n";
        echo "Total saved/updated: {$totalSaved}\n\n";

        foreach ($results as $provider => $result) {
            echo strtoupper($provider) . "\n";
            echo "Raw: " . ($result['raw'] ?? 0) . "\n";
            echo "Normalized: " . ($result['normalized'] ?? 0) . "\n";
            echo "Saved/Updated: " . ($result['saved'] ?? 0) . "\n";
            echo "Raw Actual values found: " . ($result['raw_actual_values'] ?? 0) . "\n";
            echo "Actual values saved: " . ($result['actual_values'] ?? 0) . "\n";

            if (!empty($result['error'])) {
                echo "Error: " . $result['error'] . "\n";
            }

            echo "\n";
        }
    }

    /**
     * ------------------------------------------------------------
     * PROVIDER 1: FOREXFACTORY PRIMARY
     * ------------------------------------------------------------
     */
    private static function fetchForexFactory(): array
    {
        $urls = [
            'https://www.forexfactory.com/calendar',
            'https://www.forexfactory.com/calendar?day=today',
            'https://www.forexfactory.com/calendar?day=yesterday',
            'https://www.forexfactory.com/calendar?week=next',
        ];

        $events = [];

        // Official FairEconomy calendar files are the most reliable source for
        // released Actual values. Fetch them first, then enrich/merge with the
        // HTML calendar data below.
        $fileEvents = self::fetchForexFactoryCalendarFiles();
        if (!empty($fileEvents)) {
            $events = array_merge($events, $fileEvents);
            $fileActualCount = 0;
            foreach ($fileEvents as $fileEvent) {
                if (!empty($fileEvent['actual'])) {
                    $fileActualCount++;
                }
            }
            self::log('ForexFactory calendar files parsed ' . count($fileEvents) . ' events | actuals=' . $fileActualCount);
        }

        foreach ($urls as $url) {
            $html = self::httpGet($url, [
                'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/122 Safari/537.36',
                'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                'Accept-Language: en-US,en;q=0.9',
                'Cache-Control: no-cache',
            ]);

            if (!$html) {
                self::log("ForexFactory returned empty/blocked response for {$url}");
                continue;
            }

            $parsed = self::parseForexFactoryHtml($html);
            $plainParsed = self::parseForexFactoryPlainTextCalendar($html);

            /*
             * Critical fix:
             * The official calendar files usually do not carry Actual values.
             * The HTML/plain calendar can carry Actual values, but the DOM parser
             * may return zero rows depending on what ForexFactory sends to cURL.
             * Therefore we must merge plain-text values into BOTH:
             *  1) existing file-based events already in $events, and
             *  2) DOM-parsed HTML events in $parsed.
             * We also save actual-carrying plain rows as standalone rows so the
             * duplicate-key/backfill logic can update the existing database row.
             */
            $events = self::mergeForexFactoryPlainTextValues($events, $plainParsed);
            $parsed = self::mergeForexFactoryPlainTextValues($parsed, $plainParsed);
            $plainStandalone = self::enrichForexFactoryPlainTextEvents($plainParsed, array_merge($events, $parsed));

            $events = array_merge($events, $parsed, $plainStandalone);

            $plainActualCount = 0;
            foreach ($plainParsed as $plainEvent) {
                if (!empty($plainEvent['actual'])) {
                    $plainActualCount++;
                }
            }

            if ($plainActualCount === 0 && !empty($plainParsed)) {
                $samplePlain = array_slice($plainParsed, 0, 5);
                self::log("ForexFactory plain parser captured zero actuals. Sample=" . json_encode($samplePlain, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE));
            }

            self::log("ForexFactory parsed " . count($parsed) . " DOM events from {$url} | plain_text_rows=" . count($plainParsed) . " | plain_standalone=" . count($plainStandalone) . " | plain_text_actuals={$plainActualCount}");
        }

        return $events;
    }

    private static function fetchForexFactoryCalendarFiles(): array
    {
        $urls = [
            'https://nfs.faireconomy.media/ff_calendar_thisweek.xml',
            'https://nfs.faireconomy.media/ff_calendar_nextweek.xml',
            'https://nfs.faireconomy.media/ff_calendar_lastweek.xml',
            'https://nfs.faireconomy.media/ff_calendar_thisweek.csv',
            'https://nfs.faireconomy.media/ff_calendar_nextweek.csv',
            'https://nfs.faireconomy.media/ff_calendar_lastweek.csv',
            'https://cdn-nfs.faireconomy.media/ff_calendar_thisweek.xml',
            'https://cdn-nfs.faireconomy.media/ff_calendar_nextweek.xml',
            'https://cdn-nfs.faireconomy.media/ff_calendar_lastweek.xml',
        ];

        $events = [];

        foreach ($urls as $url) {
            $body = self::httpGet($url, [
                'User-Agent: Mozilla/5.0 compatible InstitutionalTradingAssistant/1.0',
                'Accept: application/xml,text/xml,text/csv,text/plain,*/*',
                'Cache-Control: no-cache',
            ]);

            if (!$body) {
                self::log('ForexFactory calendar file returned empty response: ' . $url);
                continue;
            }

            $path = strtolower((string)(parse_url($url, PHP_URL_PATH) ?? ''));
            $parsed = str_ends_with($path, '.csv')
                ? self::parseForexFactoryCsvFile($body, $url)
                : self::parseForexFactoryXmlFile($body, $url);

            $actuals = 0;
            foreach ($parsed as $event) {
                if (!empty($event['actual'])) {
                    $actuals++;
                }
            }

            self::log('ForexFactory file parsed ' . count($parsed) . ' events from ' . $url . ' | actuals=' . $actuals);
            $events = array_merge($events, $parsed);
        }

        return $events;
    }

    private static function parseForexFactoryXmlFile(string $xml, string $sourceUrl = ''): array
    {
        $events = [];
        $xml = trim($xml);
        if ($xml === '') {
            return [];
        }

        libxml_use_internal_errors(true);
        $feed = simplexml_load_string($xml, 'SimpleXMLElement', LIBXML_NOCDATA);
        if (!$feed) {
            self::log('ForexFactory XML parse failed for ' . $sourceUrl . ': ' . trim(strip_tags(substr($xml, 0, 180))));
            libxml_clear_errors();
            return [];
        }

        foreach ($feed->event ?? [] as $row) {
            $title = self::cleanTitle((string)($row->title ?? $row->event ?? ''));
            $currency = self::normalizeCurrency((string)($row->country ?? $row->currency ?? ''));
            $date = trim((string)($row->date ?? ''));
            $time = trim((string)($row->time ?? ''));

            if ($title === '' || $currency === '' || $date === '') {
                continue;
            }

            $timestamp = self::parseForexFactoryFileTimestamp($date, $time);
            if (!$timestamp) {
                continue;
            }

            $impact = self::forexFactoryFileImpact((string)($row->impact ?? ''));
            if (!in_array($impact, self::STORE_IMPACTS, true)) {
                continue;
            }

            $actual = self::cleanForexFactoryFileValue((string)($row->actual ?? ''));
            $forecast = self::cleanForexFactoryFileValue((string)($row->forecast ?? ''));
            $previous = self::cleanForexFactoryFileValue((string)($row->previous ?? ''));

            $events[] = self::forexFactoryFileEventArray($title, $currency, $impact, $timestamp, $date, $time, $actual, $forecast, $previous, 'forexfactory_file_xml', $sourceUrl);
        }

        return $events;
    }

    private static function parseForexFactoryCsvFile(string $csv, string $sourceUrl = ''): array
    {
        $events = [];
        $csv = trim($csv);
        if ($csv === '') {
            return [];
        }

        $handle = fopen('php://temp', 'r+');
        if (!$handle) {
            return [];
        }

        fwrite($handle, $csv);
        rewind($handle);
        $header = null;

        while (($row = fgetcsv($handle)) !== false) {
            if (!is_array($row) || count($row) < 5) {
                continue;
            }

            if ($header === null) {
                $possibleHeader = array_map(static fn($v) => strtolower(trim((string)$v)), $row);
                if (in_array('title', $possibleHeader, true) || in_array('event', $possibleHeader, true)) {
                    $header = $possibleHeader;
                    continue;
                }
                $header = [];
            }

            $assoc = [];
            if (!empty($header)) {
                foreach ($header as $i => $key) {
                    $assoc[$key] = $row[$i] ?? '';
                }
            } else {
                $assoc = [
                    'title' => $row[0] ?? '',
                    'country' => $row[1] ?? '',
                    'date' => $row[2] ?? '',
                    'time' => $row[3] ?? '',
                    'impact' => $row[4] ?? '',
                    'forecast' => $row[5] ?? '',
                    'previous' => $row[6] ?? '',
                    'actual' => $row[7] ?? '',
                ];
            }

            $title = self::cleanTitle((string)($assoc['title'] ?? $assoc['event'] ?? $assoc['description'] ?? ''));
            $currency = self::normalizeCurrency((string)($assoc['country'] ?? $assoc['currency'] ?? ''));
            $date = trim((string)($assoc['date'] ?? ''));
            $time = trim((string)($assoc['time'] ?? ''));

            if ($title === '' || $currency === '' || $date === '') {
                continue;
            }

            $timestamp = self::parseForexFactoryFileTimestamp($date, $time);
            if (!$timestamp) {
                continue;
            }

            $impact = self::forexFactoryFileImpact((string)($assoc['impact'] ?? ''));
            if (!in_array($impact, self::STORE_IMPACTS, true)) {
                continue;
            }

            $actual = self::cleanForexFactoryFileValue((string)($assoc['actual'] ?? ''));
            $forecast = self::cleanForexFactoryFileValue((string)($assoc['forecast'] ?? ''));
            $previous = self::cleanForexFactoryFileValue((string)($assoc['previous'] ?? ''));

            $events[] = self::forexFactoryFileEventArray($title, $currency, $impact, $timestamp, $date, $time, $actual, $forecast, $previous, 'forexfactory_file_csv', $sourceUrl);
        }

        fclose($handle);
        return $events;
    }

    private static function forexFactoryFileEventArray(string $title, string $currency, string $impact, int $timestamp, string $date, string $time, string $actual, string $forecast, string $previous, string $provider, string $sourceUrl): array
    {
        return [
            'title' => $title,
            'currency' => $currency,
            'impact' => $impact,
            'impact_raw' => $impact,
            'impact_level' => $impact,
            'impact_label' => self::impactLabel($impact),
            'impact_source' => 'forexfactory_calendar_file',
            'folder_class' => 'faireconomy_calendar_file',
            'timestamp' => $timestamp,
            'event_time' => date('Y-m-d H:i:s', $timestamp),
            'actual' => self::nullableString($actual),
            'forecast' => self::nullableString($forecast),
            'previous' => self::nullableString($previous),
            'source' => 'forexfactory',
            'raw' => [
                'provider' => $provider,
                'source_url' => $sourceUrl,
                'date' => $date,
                'time' => $time,
                'currency' => $currency,
                'actual' => $actual,
                'forecast' => $forecast,
                'previous' => $previous,
            ],
        ];
    }

    private static function parseForexFactoryFileTimestamp(string $date, string $time): ?int
    {
        $date = trim(preg_replace('/\s+/', ' ', $date));
        $time = trim(preg_replace('/\s+/', ' ', $time));

        if ($date === '') {
            return null;
        }

        if ($time === '' || strtolower($time) === 'all day' || strtolower($time) === 'tentative') {
            $time = '00:00';
        }

        /*
         * Use UTC for FairEconomy/ForexFactory feed normalization.
         * The dashboard then displays this timestamp in Africa/Lagos (WAT).
         * This prevents Core PCE 1:30 PM WAT from being shifted to evening.
         */
        $year = date('Y');
        $candidates = [
            $date . ' ' . $time,
            $date . ' ' . $year . ' ' . $time,
            str_replace(['-', '.'], '/', $date) . ' ' . $time,
        ];

        foreach ($candidates as $candidate) {
            try {
                $sourceTz = new DateTimeZone('UTC');
                $dt = new DateTime($candidate, $sourceTz);
                return (int)$dt->getTimestamp();
            } catch (Throwable $e) {
                // Try next candidate.
            }
        }

        return null;
    }

    private static function forexFactoryFileImpact(string $impact): string
    {
        $impact = strtoupper(trim($impact));
        if (self::containsTextSafe($impact, 'HIGH') || self::containsTextSafe($impact, 'RED')) {
            return 'HIGH';
        }
        if (self::containsTextSafe($impact, 'MEDIUM') || self::containsTextSafe($impact, 'ORANGE') || self::containsTextSafe($impact, 'SEMI')) {
            return 'MEDIUM';
        }
        return 'LOW';
    }

    private static function cleanForexFactoryFileValue(string $value): string
    {
        $value = html_entity_decode((string)$value, ENT_QUOTES | ENT_HTML5, 'UTF-8');
        $value = trim(preg_replace('/\s+/', ' ', $value));
        if ($value === '') {
            return '';
        }
        $invalid = ['-', '—', '–', 'N/A', 'NA', 'NULL', 'PENDING'];
        if (in_array(strtoupper($value), $invalid, true)) {
            return '';
        }
        return mb_substr($value, 0, 100);
    }

    private static function parseForexFactoryHtml(string $html): array
    {
        $events = [];

        libxml_use_internal_errors(true);

        $dom = new DOMDocument();

        if (!$dom->loadHTML($html)) {
            libxml_clear_errors();
            self::log('ForexFactory DOM parsing failed.');
            return [];
        }

        $xpath = new DOMXPath($dom);

        /*
         * ForexFactory frequently uses rows with calendar__row.
         * If the class changes, fallback searches table rows containing event/currency/date classes.
         */
        $rows = $xpath->query("//tr[contains(@class,'calendar__row')]");

        if (!$rows || $rows->length === 0) {
            $rows = $xpath->query("//tr[.//*[contains(@class,'calendar__event') or contains(@class,'calendar__currency')]]");
        }

        $currentDate = null;

        foreach ($rows as $row) {
            if (!$row instanceof DOMElement) {
                continue;
            }

            $rowClass = $row->getAttribute('class');

            $dateText = self::nodeText($xpath, ".//*[contains(@class,'calendar__date')]", $row);
            if ($dateText !== '') {
                $currentDate = $dateText;
            }

            $timeText = self::nodeText($xpath, ".//*[contains(@class,'calendar__time')]", $row);
            $currency = self::nodeText($xpath, ".//*[contains(@class,'calendar__currency')]", $row);
            $title = self::nodeText($xpath, ".//*[contains(@class,'calendar__event')]", $row);

            if ($title === '' || $currency === '') {
                continue;
            }

            $timestamp = self::parseForexFactoryDateTime($currentDate, $timeText);

            if (!$timestamp) {
                continue;
            }

            $impactMeta = self::forexFactoryImpact($xpath, $row, $rowClass, $title);
            $impact = $impactMeta['impact'];

            if (!in_array($impact, self::STORE_IMPACTS, true)) {
                continue;
            }

            $events[] = [
                'title' => $title,
                'currency' => self::normalizeCurrency($currency),
                'impact' => $impact,
                'impact_raw' => $impactMeta['impact_raw'],
                'impact_level' => $impact,
                'impact_label' => self::impactLabel($impact),
                'impact_source' => 'forexfactory_folder',
                'folder_class' => $impactMeta['folder_class'],
                'timestamp' => $timestamp,
                'event_time' => date('Y-m-d H:i:s', $timestamp),
                'actual' => self::forexFactoryCellValue($xpath, $row, 'actual'),
                'forecast' => self::forexFactoryCellValue($xpath, $row, 'forecast'),
                'previous' => self::forexFactoryCellValue($xpath, $row, 'previous'),
                'source' => 'forexfactory',
                'raw' => [
                    'provider' => 'forexfactory',
                    'date' => $currentDate,
                    'time' => $timeText,
                    'currency' => $currency,
                    'row_class' => $rowClass,
                    'impact_raw' => $impactMeta['impact_raw'],
                    'impact_level' => $impact,
                    'impact_label' => self::impactLabel($impact),
                    'impact_source' => 'forexfactory_folder',
                    'folder_class' => $impactMeta['folder_class'],
                    'actual' => self::forexFactoryCellValue($xpath, $row, 'actual'),
                    'forecast' => self::forexFactoryCellValue($xpath, $row, 'forecast'),
                    'previous' => self::forexFactoryCellValue($xpath, $row, 'previous'),
                ],
            ];
        }

        return $events;
    }

    private static function forexFactoryCellValue(DOMXPath $xpath, DOMElement $row, string $type): string
    {
        $type = strtolower(trim($type));

        $queries = match ($type) {
            'actual' => [
                ".//*[contains(@class,'calendar__actual')]",
                ".//*[contains(concat(' ', normalize-space(@class), ' '), ' actual ')]",
                ".//*[contains(@class,'actual')]",
                ".//*[@data-column='actual' or @data-field='actual' or @data-name='actual']",
                ".//*[@title and contains(translate(@title,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'actual')]",
            ],
            'forecast' => [
                ".//*[contains(@class,'calendar__forecast')]",
                ".//*[contains(concat(' ', normalize-space(@class), ' '), ' forecast ')]",
                ".//*[contains(@class,'forecast')]",
                ".//*[@data-column='forecast' or @data-field='forecast' or @data-name='forecast']",
            ],
            'previous' => [
                ".//*[contains(@class,'calendar__previous')]",
                ".//*[contains(concat(' ', normalize-space(@class), ' '), ' previous ')]",
                ".//*[contains(@class,'previous')]",
                ".//*[@data-column='previous' or @data-field='previous' or @data-name='previous']",
            ],
            default => [],
        };

        foreach ($queries as $query) {
            $value = self::nodeText($xpath, $query, $row);
            $value = self::cleanForexFactoryValue($value, $type);

            if ($value !== '') {
                return $value;
            }
        }

        /*
         * ForexFactory row fallback. The usual table order is:
         * date, time, currency, impact, event, actual, forecast, previous.
         * Some rows hide date/time cells but still keep classes on other cells,
         * so this is only a last resort.
         */
        $cells = $xpath->query("./td", $row);
        if ($cells && $cells->length >= 8) {
            $index = match ($type) {
                'actual' => 5,
                'forecast' => 6,
                'previous' => 7,
                default => -1,
            };

            if ($index >= 0 && $cells->item($index)) {
                $value = self::cleanForexFactoryValue($cells->item($index)->textContent ?? '', $type);
                if ($value !== '') {
                    return $value;
                }
            }
        }

        return '';
    }

    private static function cleanForexFactoryValue($value, string $type = ''): string
    {
        $value = html_entity_decode((string)$value, ENT_QUOTES | ENT_HTML5, 'UTF-8');
        $value = preg_replace('/\x{00a0}/u', ' ', $value);
        $value = trim(preg_replace('/\s+/', ' ', $value));

        if ($value === '') {
            return '';
        }

        /*
         * Remove labels/status words that can be attached to the cell by
         * responsive markup, but keep numeric/event values like 1.7%, 91.9, 2.25%.
         */
        $value = preg_replace('/\b(actual|forecast|previous)\b\s*:?/i', '', $value);
        $value = preg_replace('/\b(better|worse|as expected|revised from|revised)\b.*/i', '', $value);
        $value = trim(preg_replace('/\s+/', ' ', $value));

        $invalid = ['', '-', '—', '–', 'n/a', 'na', 'null', 'pending'];
        if (in_array(strtolower($value), $invalid, true)) {
            return '';
        }

        return mb_substr($value, 0, 100);
    }

    private static function parseForexFactoryPlainTextCalendar(string $html): array
    {
        $events = [];

        /*
         * DOM textContent often collapses ForexFactory rows on cPanel/cURL.
         * Add explicit separators around table/list/block tags before stripping.
         */
        $prepared = preg_replace(
            '~<(br|/td|/th|/tr|/div|/span|/p|/li|/h[1-6])\b[^>]*>~i',
            "\n",
            $html
        );

        $prepared = preg_replace(
            '~<(td|th|tr|div|span|p|li|h[1-6])\b[^>]*>~i',
            "\n",
            $prepared
        );

        $text = html_entity_decode(strip_tags((string)$prepared), ENT_QUOTES | ENT_HTML5, 'UTF-8');
        $text = preg_replace('/\x{00a0}/u', ' ', $text);
        $text = preg_replace('/[ \t]+/', ' ', $text);

        $lines = preg_split('/\R+/', (string)$text);

        $clean = [];
        foreach ($lines as $line) {
            $line = trim(preg_replace('/\s+/', ' ', (string)$line));
            if ($line !== '') {
                $clean[] = $line;
            }
        }

        $currentDate = null;
        $currentTime = null;
        $count = count($clean);

        for ($i = 0; $i < $count; $i++) {
            $line = $clean[$i];

            /*
             * Supported formats:
             *   Tue May 26
             *   Tue May 26 7:00am JPY
             *   7:00am JPY
             *   JPY
             */
            $time = null;
            $currency = null;

            if (preg_match('/^(Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+([A-Z][a-z]{2})\s+(\d{1,2})(?:\s+((?:\d{1,2}:\d{2}\s*(?:am|pm))|Tentative|All Day)\s+([A-Z]{3}))?$/i', $line, $m)) {
                $currentDate = trim($m[1] . ' ' . $m[2] . ' ' . $m[3]);

                if (!empty($m[4]) && !empty($m[5])) {
                    $time = trim($m[4]);
                    $currency = strtoupper($m[5]);
                    $currentTime = $time;
                } else {
                    continue;
                }
            } elseif (preg_match('/^((?:\d{1,2}:\d{2}\s*(?:am|pm))|Tentative|All Day)\s+([A-Z]{3})$/i', $line, $m)) {
                $time = trim($m[1]);
                $currency = strtoupper($m[2]);
                $currentTime = $time;
            } elseif (preg_match('/^[A-Z]{3}$/', $line)) {
                $currency = strtoupper($line);
                $time = $currentTime ?: '00:00';
            } else {
                continue;
            }

            if (!$currentDate || !$currency) {
                continue;
            }

            $titleIndex = self::nextForexFactoryTitleLine($clean, $i + 1);
            if ($titleIndex === null) {
                continue;
            }

            $title = trim($clean[$titleIndex]);

            $timestamp = self::parseForexFactoryDateTime($currentDate, (string)$time);
            if (!$timestamp) {
                continue;
            }

            $values = [];
            $j = $titleIndex + 1;

            while ($j < $count) {
                $candidate = trim($clean[$j]);

                if (self::looksLikeForexFactoryEventBoundary($candidate)) {
                    break;
                }

                foreach (self::extractForexFactoryValuesFromLine($candidate) as $value) {
                    $values[] = $value;
                }

                $j++;

                if (count($values) >= 3) {
                    break;
                }
            }

            $actual = '';
            $forecast = '';
            $previous = '';

            /*
             * ForexFactory display order:
             * Released event with all values: Actual / Forecast / Previous.
             * Unreleased event usually shows: Forecast / Previous.
             */
            if (count($values) >= 3) {
                /*
                 * Released rows show Actual / Forecast / Previous.
                 * Use the values directly; do not depend on server timezone.
                 */
                $actual = $values[0];
                $forecast = $values[1];
                $previous = $values[2];
            } elseif (count($values) === 2) {
                /*
                 * With only two values, unreleased rows are normally
                 * Forecast / Previous. Released rows inside the wider lookback
                 * can be Actual / Previous. Use a 12-hour tolerance to avoid
                 * losing just-released values due to source/server timezone.
                 */
                if ($timestamp <= (time() + (12 * 3600))) {
                    $actual = $values[0];
                    $previous = $values[1];
                } else {
                    $forecast = $values[0];
                    $previous = $values[1];
                }
            } elseif (count($values) === 1) {
                if ($timestamp <= (time() + (12 * 3600))) {
                    $actual = $values[0];
                } else {
                    $previous = $values[0];
                }
            }

            $events[] = [
                'title' => $title,
                'currency' => self::normalizeCurrency($currency),
                'timestamp' => $timestamp,
                'event_time' => date('Y-m-d H:i:s', $timestamp),
                'actual' => self::nullableString($actual),
                'forecast' => self::nullableString($forecast),
                'previous' => self::nullableString($previous),
            ];
        }

        return $events;
    }

    private static function nextForexFactoryTitleLine(array $lines, int $startIndex): ?int
    {
        $count = count($lines);

        for ($i = $startIndex; $i < $count; $i++) {
            $line = trim((string)$lines[$i]);

            if ($line === '') {
                continue;
            }

            if (self::looksLikeForexFactoryEventBoundary($line)) {
                return null;
            }

            if (self::looksLikeForexFactoryValueLine($line)) {
                continue;
            }

            if (preg_match('/^(Date|Currency|Impact|Actual|Forecast|Previous|Graph|Detail|Alerts|Navigation|Filter|More|Up Next)$/i', $line)) {
                continue;
            }

            if (stripos($line, 'Calendar Time Zone') !== false || stripos($line, 'Printed ') === 0) {
                continue;
            }

            return $i;
        }

        return null;
    }

    private static function looksLikeForexFactoryEventBoundary(string $line): bool
    {
        $line = trim($line);

        return (
            preg_match('/^(Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+[A-Z][a-z]{2}\s+\d{1,2}(?:\s+((?:\d{1,2}:\d{2}\s*(?:am|pm))|Tentative|All Day)\s+[A-Z]{3})?$/i', $line)
            || preg_match('/^((?:\d{1,2}:\d{2}\s*(?:am|pm))|Tentative|All Day)\s+[A-Z]{3}$/i', $line)
            || preg_match('/^[A-Z]{3}$/', $line)
            || stripos($line, '© Fair Economy') !== false
            || stripos($line, 'Top of Page') !== false
        );
    }

    private static function looksLikeForexFactoryValueLine(string $line): bool
    {
        return !empty(self::extractForexFactoryValuesFromLine($line));
    }

    private static function extractForexFactoryValuesFromLine(string $line): array
    {
        $line = self::cleanForexFactoryValue($line);

        if ($line === '') {
            return [];
        }

        /*
         * Match economic values:
         * 1.4%, -46, 42.3K, -2.0%, 93.1, 3.62|1.3, <0.75%
         */
        preg_match_all('/(?:<|>)?-?\d+(?:\.\d+)?(?:\|(?:<|>)?-?\d+(?:\.\d+)?)?\s*(?:%|K|M|B|T)?/i', $line, $matches);

        $values = [];
        foreach (($matches[0] ?? []) as $value) {
            $value = trim(preg_replace('/\s+/', '', (string)$value));
            if ($value !== '') {
                $values[] = $value;
            }
        }

        return $values;
    }

    private static function enrichForexFactoryPlainTextEvents(array $plainEvents, array $knownEvents): array
    {
        if (empty($plainEvents)) {
            return [];
        }

        $knownByExact = [];
        $knownByDate = [];

        foreach ($knownEvents as $event) {
            if (!is_array($event)) {
                continue;
            }

            $title = strtoupper(self::cleanTitle((string)($event['title'] ?? '')));
            $currency = strtoupper((string)($event['currency'] ?? ''));
            $timestamp = (int)($event['timestamp'] ?? 0);

            if ($title === '' || $currency === '' || $timestamp <= 0) {
                continue;
            }

            $exactKey = $title . '|' . $currency . '|' . date('Y-m-d H:i', $timestamp);
            $dateKey = $title . '|' . $currency . '|' . date('Y-m-d', $timestamp);

            $knownByExact[$exactKey] = $event;
            if (!isset($knownByDate[$dateKey])) {
                $knownByDate[$dateKey] = $event;
            }
        }

        $standalone = [];
        $seen = [];

        foreach ($plainEvents as $event) {
            if (!is_array($event)) {
                continue;
            }

            // Only save standalone plain rows when they carry useful value data.
            if (empty($event['actual']) && empty($event['forecast']) && empty($event['previous'])) {
                continue;
            }

            $title = strtoupper(self::cleanTitle((string)($event['title'] ?? '')));
            $currency = strtoupper((string)($event['currency'] ?? ''));
            $timestamp = (int)($event['timestamp'] ?? 0);

            if ($title === '' || $currency === '' || $timestamp <= 0) {
                continue;
            }

            $exactKey = $title . '|' . $currency . '|' . date('Y-m-d H:i', $timestamp);
            $dateKey = $title . '|' . $currency . '|' . date('Y-m-d', $timestamp);
            $known = $knownByExact[$exactKey] ?? $knownByDate[$dateKey] ?? [];

            $impact = self::normalizeImpact((string)($known['impact'] ?? $known['impact_level'] ?? 'LOW'), (string)($event['title'] ?? ''));
            $impactLevel = self::normalizeImpactLevel($known['impact_level'] ?? $impact);

            $event['impact'] = $impactLevel;
            $event['impact_level'] = $impactLevel;
            $event['impact_label'] = $known['impact_label'] ?? self::impactLabel($impactLevel);
            $event['impact_raw'] = $known['impact_raw'] ?? 'plain_text_calendar';
            $event['impact_source'] = $known['impact_source'] ?? 'forexfactory_plain_text';
            $event['folder_class'] = $known['folder_class'] ?? null;
            $event['source'] = 'forexfactory_plain_text';
            $event['raw'] = [
                'provider' => 'forexfactory_plain_text',
                'matched_existing_event' => !empty($known),
                'actual' => $event['actual'] ?? null,
                'forecast' => $event['forecast'] ?? null,
                'previous' => $event['previous'] ?? null,
            ];

            $dedupeKey = $exactKey . '|' . ($event['actual'] ?? '') . '|' . ($event['forecast'] ?? '') . '|' . ($event['previous'] ?? '');
            if (isset($seen[$dedupeKey])) {
                continue;
            }

            $seen[$dedupeKey] = true;
            $standalone[] = $event;
        }

        return $standalone;
    }

    private static function mergeForexFactoryPlainTextValues(array $domEvents, array $plainEvents): array
    {
        if (empty($domEvents) || empty($plainEvents)) {
            return $domEvents;
        }

        $plainByExact = [];
        $plainByDate = [];

        foreach ($plainEvents as $event) {
            $title = strtoupper(self::cleanTitle((string)($event['title'] ?? '')));
            $currency = strtoupper((string)($event['currency'] ?? ''));
            $timestamp = (int)($event['timestamp'] ?? 0);

            if ($title === '' || $currency === '' || $timestamp <= 0) {
                continue;
            }

            $exactKey = $title . '|' . $currency . '|' . date('Y-m-d H:i', $timestamp);
            $dateKey = $title . '|' . $currency . '|' . date('Y-m-d', $timestamp);

            $plainByExact[$exactKey] = $event;

            if (!isset($plainByDate[$dateKey]) || !empty($event['actual'])) {
                $plainByDate[$dateKey] = $event;
            }
        }

        foreach ($domEvents as &$event) {
            $title = strtoupper(self::cleanTitle((string)($event['title'] ?? '')));
            $currency = strtoupper((string)($event['currency'] ?? ''));
            $timestamp = (int)($event['timestamp'] ?? 0);

            if ($title === '' || $currency === '' || $timestamp <= 0) {
                continue;
            }

            $exactKey = $title . '|' . $currency . '|' . date('Y-m-d H:i', $timestamp);
            $dateKey = $title . '|' . $currency . '|' . date('Y-m-d', $timestamp);

            $plain = $plainByExact[$exactKey] ?? $plainByDate[$dateKey] ?? null;
            if (!$plain) {
                continue;
            }

            foreach (['actual', 'forecast', 'previous'] as $field) {
                if (empty($event[$field]) && !empty($plain[$field])) {
                    $event[$field] = $plain[$field];
                }
            }

            if (!isset($event['raw']) || !is_array($event['raw'])) {
                $event['raw'] = [];
            }

            $event['raw']['plain_text_actual'] = $plain['actual'] ?? null;
            $event['raw']['plain_text_forecast'] = $plain['forecast'] ?? null;
            $event['raw']['plain_text_previous'] = $plain['previous'] ?? null;
        }

        unset($event);

        return $domEvents;
    }

    private static function forexFactoryImpact(DOMXPath $xpath, DOMElement $row, string $rowClass, string $title): array
    {
        $parts = [];
        $folderClass = strtolower($rowClass);

        $collectElementMeta = static function (?DOMNode $node) use (&$parts, &$folderClass): void {
            if (!$node instanceof DOMElement) {
                return;
            }

            $parts[] = strtolower(trim($node->textContent));

            foreach ([
                'class',
                'title',
                'alt',
                'aria-label',
                'data-title',
                'data-impact',
                'data-tooltip',
                'data-original-title',
            ] as $attr) {
                if ($node->hasAttribute($attr)) {
                    $value = strtolower(trim($node->getAttribute($attr)));
                    if ($value !== '') {
                        $parts[] = $value;
                        if ($attr === 'class') {
                            $folderClass .= ' ' . $value;
                        }
                    }
                }
            }
        };

        $collectElementMeta($row);

        $impactNode = $xpath->query(".//*[contains(@class,'calendar__impact')]", $row)->item(0);
        $collectElementMeta($impactNode);

        /*
         * ForexFactory folder/icon information is usually in class/title/alt
         * values on icon elements. We read the source metadata directly and
         * do NOT promote impact based on the event title.
         */
        $iconNodes = $xpath->query(".//*[@title or @alt or @aria-label or @data-impact or @data-tooltip or @data-original-title or contains(@class,'impact') or contains(@class,'ff-impact') or contains(@class,'folder')]", $row);
        foreach ($iconNodes as $iconNode) {
            $collectElementMeta($iconNode);
        }

        $combined = trim(preg_replace('/\s+/', ' ', implode(' ', array_filter($parts))));
        $combined = strtolower($combined);

        /*
         * ForexFactory folder mapping:
         * yellow folder             => LOW
         * orange / semi-red folder  => MEDIUM
         * red folder                => HIGH
         *
         * Check semi-red/orange/medium before red so "semi-red" is not
         * accidentally treated as high-impact.
         */
        if (
            self::containsTextSafe($combined, 'yellow') ||
            self::containsTextSafe($combined, 'low impact') ||
            self::containsTextSafe($combined, 'impact--low') ||
            self::containsTextSafe($combined, 'impact-low') ||
            self::containsTextSafe($combined, 'ff-impact-yel') ||
            self::containsTextSafe($combined, 'impact-yel')
        ) {
            $impact = 'LOW';
        } elseif (
            self::containsTextSafe($combined, 'semi-red') ||
            self::containsTextSafe($combined, 'semired') ||
            self::containsTextSafe($combined, 'orange') ||
            self::containsTextSafe($combined, 'medium impact') ||
            self::containsTextSafe($combined, 'impact--medium') ||
            self::containsTextSafe($combined, 'impact-medium') ||
            self::containsTextSafe($combined, 'ff-impact-ora') ||
            self::containsTextSafe($combined, 'impact-ora')
        ) {
            $impact = 'MEDIUM';
        } elseif (
            self::containsTextSafe($combined, 'red') ||
            self::containsTextSafe($combined, 'high impact') ||
            self::containsTextSafe($combined, 'impact--high') ||
            self::containsTextSafe($combined, 'impact-high') ||
            self::containsTextSafe($combined, 'ff-impact-red') ||
            self::containsTextSafe($combined, 'impact-red')
        ) {
            $impact = 'HIGH';
        } else {
            /*
             * If the folder metadata is missing because ForexFactory changed
             * markup or blocked HTML, do not guess from title. Store LOW with
             * raw=unknown and let the log/database reveal the parsing issue.
             */
            $impact = 'LOW';
            self::log('ForexFactory impact metadata not found for event "' . $title . '". Stored as LOW pending parser review.');
        }

        return [
            'impact' => $impact,
            'impact_raw' => mb_substr($combined !== '' ? $combined : 'unknown', 0, 255),
            'folder_class' => mb_substr(trim($folderClass), 0, 255),
        ];
    }

    private static function parseForexFactoryDateTime(?string $dateText, string $timeText): ?int
    {
        if (!$dateText) {
            return null;
        }

        $dateText = trim(preg_replace('/\s+/', ' ', $dateText));
        $timeText = trim(preg_replace('/\s+/', ' ', $timeText));

        if ($timeText === '' || strtolower($timeText) === 'all day' || strtolower($timeText) === 'tentative') {
            $timeText = '00:00';
        }

        $year = date('Y');

        $candidates = [
            "{$dateText} {$year} {$timeText}",
            "{$dateText} {$timeText} {$year}",
            "{$year} {$dateText} {$timeText}",
        ];

        foreach ($candidates as $candidate) {
            $ts = strtotime($candidate);
            if ($ts) {
                return $ts;
            }
        }

        return null;
    }

    /**
     * ------------------------------------------------------------
     * PROVIDER 2: TRADINGECONOMICS FALLBACK/ENRICHMENT
     * ------------------------------------------------------------
     */
    private static function fetchTradingEconomics(): array
    {
        $key = self::apiKey(['tradingeconomics', 'trading_economics']);

        if (!$key) {
            self::log('TradingEconomics API key not configured. Skipping.');
            return [];
        }

        $from = gmdate('Y-m-d');
        $to = gmdate('Y-m-d', strtotime('+7 days'));

        $url = 'https://api.tradingeconomics.com/calendar?' . http_build_query([
            'c' => $key,
            'd1' => $from,
            'd2' => $to,
            'format' => 'json',
        ]);

        $json = self::httpJson($url);

        if (!is_array($json) || empty($json)) {
            return [];
        }

        $events = [];

        foreach ($json as $row) {
            if (!is_array($row)) {
                continue;
            }

            $title = trim((string)($row['Event'] ?? $row['event'] ?? ''));
            $country = trim((string)($row['Country'] ?? $row['country'] ?? ''));
            $timestamp = self::parseGenericTimestamp($row['Date'] ?? $row['date'] ?? null, 'UTC');

            if ($title === '' || !$timestamp) {
                continue;
            }

            $impact = self::normalizeImpact((string)($row['Importance'] ?? $row['importance'] ?? ''), $title);

            if (!in_array($impact, self::STORE_IMPACTS, true)) {
                continue;
            }

            $events[] = [
                'title' => $title,
                'currency' => self::countryToCurrency($country),
                'impact' => $impact,
                'timestamp' => $timestamp,
                'event_time' => date('Y-m-d H:i:s', $timestamp),
                'actual' => $row['Actual'] ?? $row['actual'] ?? null,
                'forecast' => $row['Forecast'] ?? $row['forecast'] ?? null,
                'previous' => $row['Previous'] ?? $row['previous'] ?? null,
                'source' => 'tradingeconomics',
                'raw' => $row,
            ];
        }

        return $events;
    }

    /**
     * ------------------------------------------------------------
     * PROVIDER 3: FMP FALLBACK/ENRICHMENT
     * ------------------------------------------------------------
     */
    private static function fetchFmp(): array
    {
        $key = self::apiKey([
            'fmp',
            'financialmodelingprep',
            'financial_modeling_prep',
            'financialmodelingprep_api',
            'economic_calendar',
        ]);

        if (!$key) {
            self::log('FMP API key not configured. Skipping.');
            return [];
        }

        $from = gmdate('Y-m-d');
        $to = gmdate('Y-m-d', strtotime('+7 days'));

        $urls = [
            'https://financialmodelingprep.com/stable/economic-calendar?' . http_build_query([
                'from' => $from,
                'to' => $to,
                'apikey' => $key,
            ]),
            'https://financialmodelingprep.com/api/v3/economic_calendar?' . http_build_query([
                'from' => $from,
                'to' => $to,
                'apikey' => $key,
            ]),
        ];

        $rows = [];

        foreach ($urls as $url) {
            $json = self::httpJson($url);

            if (isset($json['Error Message']) || isset($json['error'])) {
                self::log('FMP returned error: ' . json_encode($json));
                continue;
            }

            if (is_array($json) && !empty($json)) {
                $rows = $json;
                break;
            }
        }

        $events = [];

        foreach ($rows as $row) {
            if (!is_array($row)) {
                continue;
            }

            $title = trim((string)($row['event'] ?? $row['title'] ?? $row['name'] ?? ''));
            $timestamp = self::parseGenericTimestamp($row['date'] ?? $row['time'] ?? $row['datetime'] ?? null, 'UTC');

            if ($title === '' || !$timestamp) {
                continue;
            }

            $impact = self::normalizeImpact((string)($row['impact'] ?? $row['importance'] ?? ''), $title);

            if (!in_array($impact, self::STORE_IMPACTS, true)) {
                continue;
            }

            $events[] = [
                'title' => $title,
                'currency' => self::countryToCurrency((string)($row['currency'] ?? $row['country'] ?? '')),
                'impact' => $impact,
                'timestamp' => $timestamp,
                'event_time' => date('Y-m-d H:i:s', $timestamp),
                'actual' => $row['actual'] ?? null,
                'forecast' => $row['estimate'] ?? $row['forecast'] ?? null,
                'previous' => $row['previous'] ?? null,
                'source' => 'fmp',
                'raw' => $row,
            ];
        }

        return $events;
    }

    /**
     * ------------------------------------------------------------
     * PROVIDER 4: RSS MACRO HEADLINE FALLBACK
     * ------------------------------------------------------------
     */
    private static function fetchRssMacro(): array
    {
        $feeds = [
            'https://www.investing.com/rss/news_25.rss',
            'https://feeds.reuters.com/reuters/businessNews',
            'https://feeds.reuters.com/news/economy',
        ];

        $events = [];

        foreach ($feeds as $feed) {
            $xml = self::httpGet($feed, [
                'User-Agent: Mozilla/5.0 compatible TradingNewsBot/1.0',
                'Accept: application/rss+xml, application/xml, text/xml, */*',
            ]);

            if (!$xml) {
                continue;
            }

            $rss = @simplexml_load_string($xml);

            if (!$rss || empty($rss->channel->item)) {
                continue;
            }

            foreach ($rss->channel->item as $item) {
                $title = trim((string)($item->title ?? ''));
                $pubDate = trim((string)($item->pubDate ?? ''));

                if ($title === '') {
                    continue;
                }

                $impact = self::inferImpact($title);

                // RSS fallback is unscheduled headline risk. Keep only material headlines.
                if (!in_array($impact, ['MEDIUM', 'HIGH'], true)) {
                    continue;
                }

                $timestamp = self::parseGenericTimestamp($pubDate ?: 'now', 'UTC') ?: time();

                $events[] = [
                    'title' => $title,
                    'currency' => self::inferCurrencyFromTitle($title),
                    'impact' => $impact,
                    'timestamp' => $timestamp,
                    'event_time' => date('Y-m-d H:i:s', $timestamp),
                    'actual' => null,
                    'forecast' => null,
                    'previous' => null,
                    'source' => 'rss_macro',
                    'raw' => [
                        'feed' => $feed,
                        'link' => (string)($item->link ?? ''),
                        'pubDate' => $pubDate,
                    ],
                ];
            }
        }

        return $events;
    }

    /**
     * ------------------------------------------------------------
     * NORMALIZE + SAVE
     * ------------------------------------------------------------
     */
    private static function normalizeEvents(array $events, string $provider): array
    {
        $normalized = [];

        $now = time();

        /*
         * IMPORTANT:
         * Actual values are often published after release and the weekly
         * ForexFactory/FairEconomy page contains released rows from yesterday
         * and earlier this week. The old 6-hour lookback dropped those rows
         * before they reached saveEvents(), which is why Actual stayed NULL.
         *
         * Keep a wider rolling window so released values can be captured and
         * backfilled reliably.
         */
        $min = $now - (7 * 86400);
        $max = $now + (14 * 86400);

        foreach ($events as $event) {
            if (!is_array($event)) {
                continue;
            }

            $title = self::cleanTitle((string)($event['title'] ?? $event['event'] ?? $event['name'] ?? ''));

            if ($title === '') {
                continue;
            }

            $timestamp = (int)($event['timestamp'] ?? 0);

            if (!$timestamp) {
                $timestamp = self::parseGenericTimestamp($event['event_time'] ?? $event['time'] ?? $event['date'] ?? null, 'UTC') ?? 0;
            }

            if (!$timestamp || $timestamp < $min || $timestamp > $max) {
                continue;
            }

            $currency = self::normalizeCurrency((string)($event['currency'] ?? $event['country'] ?? ''));

            if ($currency === '') {
                $currency = self::inferCurrencyFromTitle($title);
            }

            $impact = self::normalizeImpact((string)($event['impact'] ?? ''), $title);
            $impactLevel = self::normalizeImpactLevel($event['impact_level'] ?? $impact);
            $impactRaw = self::nullableString($event['impact_raw'] ?? $event['impact'] ?? null);
            $impactLabel = self::nullableString($event['impact_label'] ?? self::impactLabel($impactLevel));
            $impactSource = self::nullableString($event['impact_source'] ?? $event['source'] ?? $provider);
            $folderClass = self::nullableString($event['folder_class'] ?? null);

            if (!in_array($impactLevel, self::STORE_IMPACTS, true)) {
                continue;
            }

            $normalized[] = [
                'event_uid' => self::eventUid($title, $currency, $timestamp),
                'currency' => $currency,
                'title' => $title,
                'impact' => $impactLevel,
                'impact_raw' => $impactRaw,
                'impact_level' => $impactLevel,
                'impact_label' => $impactLabel,
                'impact_source' => $impactSource,
                'folder_class' => $folderClass,
                'event_time' => date('Y-m-d H:i:s', $timestamp),
                'event_timestamp' => $timestamp,
                'actual' => self::nullableString($event['actual'] ?? null),
                'forecast' => self::nullableString($event['forecast'] ?? null),
                'previous' => self::nullableString($event['previous'] ?? null),
                'source' => $event['source'] ?? $provider,
                'raw_json' => json_encode($event['raw'] ?? $event, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE),
            ];
        }

        usort($normalized, fn($a, $b) => $a['event_timestamp'] <=> $b['event_timestamp']);

        return $normalized;
    }

    private static function saveEvents(array $events): int
    {
        if (empty($events)) {
            return 0;
        }

        $sql = "
            INSERT INTO news_events
            (
                event_uid,
                currency,
                title,
                impact,
                impact_raw,
                impact_level,
                impact_label,
                impact_source,
                folder_class,
                event_time,
                event_timestamp,
                actual,
                forecast,
                previous,
                source,
                raw_json
            )
            VALUES
            (
                :event_uid,
                :currency,
                :title,
                :impact,
                :impact_raw,
                :impact_level,
                :impact_label,
                :impact_source,
                :folder_class,
                :event_time,
                :event_timestamp,
                :actual,
                :forecast,
                :previous,
                :source,
                :raw_json
            )
            ON DUPLICATE KEY UPDATE
                currency = VALUES(currency),
                title = VALUES(title),
                impact = VALUES(impact),
                impact_raw = VALUES(impact_raw),
                impact_level = VALUES(impact_level),
                impact_label = VALUES(impact_label),
                impact_source = VALUES(impact_source),
                folder_class = VALUES(folder_class),
                event_time = VALUES(event_time),
                event_timestamp = VALUES(event_timestamp),
                actual = CASE WHEN VALUES(actual) IS NOT NULL AND VALUES(actual) <> '' THEN VALUES(actual) ELSE actual END,
                forecast = CASE WHEN VALUES(forecast) IS NOT NULL AND VALUES(forecast) <> '' THEN VALUES(forecast) ELSE forecast END,
                previous = CASE WHEN VALUES(previous) IS NOT NULL AND VALUES(previous) <> '' THEN VALUES(previous) ELSE previous END,
                source = VALUES(source),
                raw_json = VALUES(raw_json)
        ";

        $stmt = db()->prepare($sql);

        $saved = 0;

        foreach ($events as $event) {
            try {
                $stmt->execute([
                    ':event_uid' => $event['event_uid'],
                    ':currency' => $event['currency'],
                    ':title' => $event['title'],
                    ':impact' => $event['impact'],
                    ':impact_raw' => $event['impact_raw'],
                    ':impact_level' => $event['impact_level'],
                    ':impact_label' => $event['impact_label'],
                    ':impact_source' => $event['impact_source'],
                    ':folder_class' => $event['folder_class'],
                    ':event_time' => $event['event_time'],
                    ':event_timestamp' => $event['event_timestamp'],
                    ':actual' => $event['actual'],
                    ':forecast' => $event['forecast'],
                    ':previous' => $event['previous'],
                    ':source' => $event['source'],
                    ':raw_json' => $event['raw_json'],
                ]);

                if (!empty($event['actual'])) {
                    self::backfillActualForSameEvent($event);
                }

                $saved++;
            } catch (Throwable $e) {
                self::log('DB save failed: ' . $e->getMessage() . ' | event=' . json_encode($event));
            }
        }

        return $saved;
    }

    private static function backfillActualsFromRawEvents(array $events): void
    {
        foreach ($events as $event) {
            if (!is_array($event) || empty($event['actual'])) {
                continue;
            }

            try {
                $title = self::cleanTitle((string)($event['title'] ?? $event['event'] ?? $event['name'] ?? ''));
                $currency = self::normalizeCurrency((string)($event['currency'] ?? $event['country'] ?? ''));
                $timestamp = (int)($event['timestamp'] ?? 0);

                if ($title === '' || $currency === '' || $timestamp <= 0) {
                    continue;
                }

                $actual = self::nullableString($event['actual'] ?? null);
                if ($actual === null) {
                    continue;
                }

                $eventTime = date('Y-m-d H:i:s', $timestamp);

                /*
                 * Match by title/currency/date rather than event_uid only.
                 * This repairs rows created earlier with blank Actual even when
                 * the source time shifts slightly after release.
                 */
                $stmt = db()->prepare("
                    UPDATE news_events
                    SET actual = :actual
                    WHERE currency = :currency
                      AND DATE(event_time) = DATE(:event_time)
                      AND UPPER(TRIM(title)) = UPPER(TRIM(:title))
                      AND (actual IS NULL OR actual = '')
                ");

                $stmt->execute([
                    ':actual' => $actual,
                    ':currency' => $currency,
                    ':event_time' => $eventTime,
                    ':title' => $title,
                ]);
            } catch (Throwable $e) {
                self::log('Raw Actual backfill failed: ' . $e->getMessage());
            }
        }
    }

    private static function backfillActualForSameEvent(array $event): void
    {
        try {
            $actual = self::nullableString($event['actual'] ?? null);
            if ($actual === null) {
                return;
            }

            $stmt = db()->prepare("
                UPDATE news_events
                SET actual = :actual,
                    raw_json = COALESCE(raw_json, :raw_json)
                WHERE currency = :currency
                  AND DATE(event_time) = DATE(:event_time)
                  AND UPPER(TRIM(title)) = UPPER(TRIM(:title))
                  AND (actual IS NULL OR actual = '')
            ");

            $stmt->execute([
                ':actual' => $actual,
                ':raw_json' => $event['raw_json'] ?? null,
                ':currency' => $event['currency'] ?? '',
                ':event_time' => $event['event_time'] ?? date('Y-m-d H:i:s'),
                ':title' => $event['title'] ?? '',
            ]);
        } catch (Throwable $e) {
            self::log('Actual backfill failed: ' . $e->getMessage());
        }
    }

    /**
     * ------------------------------------------------------------
     * CLASSIFICATION
     * ------------------------------------------------------------
     */
    private static function normalizeImpact(string $impact, string $title): string
    {
        $impact = strtoupper(trim($impact));

        if (
            in_array($impact, ['3', 'HIGH', 'RED'], true) ||
            self::containsTextSafe($impact, 'HIGH') ||
            self::containsTextSafe($impact, 'RED')
        ) {
            return 'HIGH';
        }

        if (
            in_array($impact, ['2', 'MEDIUM', 'ORANGE', 'SEMI-RED', 'SEMI RED'], true) ||
            self::containsTextSafe($impact, 'MEDIUM') ||
            self::containsTextSafe($impact, 'ORANGE') ||
            self::containsTextSafe($impact, 'SEMI-RED') ||
            self::containsTextSafe($impact, 'SEMI RED')
        ) {
            return 'MEDIUM';
        }

        if (
            in_array($impact, ['1', 'LOW', 'YELLOW'], true) ||
            self::containsTextSafe($impact, 'LOW') ||
            self::containsTextSafe($impact, 'YELLOW')
        ) {
            return 'LOW';
        }

        // Non-ForexFactory fallback only. ForexFactory itself is parsed from folder metadata.
        return self::inferImpact($title);
    }

    private static function normalizeImpactLevel($impact): string
    {
        $impact = strtoupper(trim((string)$impact));

        return match ($impact) {
            'HIGH' => 'HIGH',
            'MEDIUM' => 'MEDIUM',
            default => 'LOW',
        };
    }

    private static function impactLabel(string $impact): string
    {
        return match (strtoupper($impact)) {
            'HIGH' => 'High Impact Expected',
            'MEDIUM' => 'Medium Impact Expected',
            default => 'Low Impact Expected',
        };
    }

    private static function inferImpact(string $title): string
    {
        $upper = strtoupper($title);

        if (self::isCriticalTitle($upper)) {
            return 'HIGH';
        }

        $high = [
            'GDP',
            'INFLATION',
            'UNEMPLOYMENT',
            'RETAIL SALES',
            'ISM MANUFACTURING',
            'ISM SERVICES',
            'CENTRAL BANK',
            'MONETARY POLICY',
            'PRESS CONFERENCE',
        ];

        foreach ($high as $word) {
            if (self::containsTextSafe($upper, $word)) {
                return 'HIGH';
            }
        }

        $medium = [
            'PMI',
            'JOBLESS',
            'CLAIMS',
            'SPEECH',
            'SPEAKS',
            'TESTIFIES',
            'TESTIMONY',
            'AUCTION',
            'CONSUMER CONFIDENCE',
            'PPI',
            'HOUSING',
            'BUILDING PERMITS',
            'DURABLE GOODS',
            'TRADE BALANCE',
            'INDUSTRIAL PRODUCTION',
            'FOMC MEMBER',
            'FED MEMBER',
            'ECB MEMBER',
            'BOE MEMBER',
            'BOJ MEMBER',
        ];

        foreach ($medium as $word) {
            if (self::containsTextSafe($upper, $word)) {
                return 'MEDIUM';
            }
        }

        return 'LOW';
    }

    private static function isCriticalTitle(string $title): bool
    {
        $upper = strtoupper($title);

        $critical = [
            'NON-FARM',
            'NONFARM',
            'NFP',
            'FOMC',
            'FED INTEREST RATE',
            'FED RATE',
            'RATE DECISION',
            'INTEREST RATE DECISION',
            'CPI',
            'CORE CPI',
            'PCE PRICE INDEX',
            'CORE PCE',
            'POWELL',
            'ECB INTEREST RATE',
            'BOE INTEREST RATE',
            'BOJ INTEREST RATE',
            'RBA INTEREST RATE',
            'BOC INTEREST RATE',
        ];

        foreach ($critical as $word) {
            if (self::containsTextSafe($upper, $word)) {
                return true;
            }
        }

        return false;
    }

    /**
     * ------------------------------------------------------------
     * DB + TABLE
     * ------------------------------------------------------------
     */
    private static function ensureNewsTableExists(): void
    {
        db()->exec("
            CREATE TABLE IF NOT EXISTS news_events (
                id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
                event_uid VARCHAR(255) UNIQUE NULL,
                currency VARCHAR(10) NULL,
                title VARCHAR(255) NOT NULL,
                impact ENUM('LOW','MEDIUM','HIGH','CRITICAL') DEFAULT 'LOW',
                impact_raw VARCHAR(255) NULL,
                impact_level ENUM('LOW','MEDIUM','HIGH') DEFAULT 'LOW',
                impact_label VARCHAR(50) NULL,
                impact_source VARCHAR(100) NULL,
                folder_class VARCHAR(255) NULL,
                event_time DATETIME NOT NULL,
                event_timestamp BIGINT NULL,
                actual VARCHAR(100) NULL,
                forecast VARCHAR(100) NULL,
                previous VARCHAR(100) NULL,
                source VARCHAR(100) NULL,
                raw_json LONGTEXT NULL,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                INDEX idx_event_time(event_time),
                INDEX idx_currency(currency),
                INDEX idx_impact(impact),
                INDEX idx_impact_level(impact_level),
                INDEX idx_event_timestamp(event_timestamp)
            ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
        ");

        self::ensureNewsTableColumns();
    }

    private static function ensureNewsTableColumns(): void
    {
        self::addColumnIfMissing('news_events', 'impact_raw', "VARCHAR(255) NULL AFTER impact");
        self::addColumnIfMissing('news_events', 'impact_level', "ENUM('LOW','MEDIUM','HIGH') DEFAULT 'LOW' AFTER impact_raw");
        self::addColumnIfMissing('news_events', 'impact_label', "VARCHAR(50) NULL AFTER impact_level");
        self::addColumnIfMissing('news_events', 'impact_source', "VARCHAR(100) NULL AFTER impact_label");
        self::addColumnIfMissing('news_events', 'folder_class', "VARCHAR(255) NULL AFTER impact_source");
        self::addIndexIfMissing('news_events', 'idx_impact_level', 'impact_level');
    }

    private static function addColumnIfMissing(string $table, string $column, string $definition): void
    {
        try {
            $stmt = db()->prepare("
                SELECT COUNT(*)
                FROM INFORMATION_SCHEMA.COLUMNS
                WHERE TABLE_SCHEMA = DATABASE()
                  AND TABLE_NAME = ?
                  AND COLUMN_NAME = ?
            ");
            $stmt->execute([$table, $column]);

            if ((int)$stmt->fetchColumn() === 0) {
                db()->exec("ALTER TABLE {$table} ADD COLUMN {$column} {$definition}");
                self::log("Added missing column {$table}.{$column}");
            }
        } catch (Throwable $e) {
            self::log("Column check/add failed for {$table}.{$column}: " . $e->getMessage());
        }
    }

    private static function addIndexIfMissing(string $table, string $index, string $column): void
    {
        try {
            $stmt = db()->prepare("
                SELECT COUNT(*)
                FROM INFORMATION_SCHEMA.STATISTICS
                WHERE TABLE_SCHEMA = DATABASE()
                  AND TABLE_NAME = ?
                  AND INDEX_NAME = ?
            ");
            $stmt->execute([$table, $index]);

            if ((int)$stmt->fetchColumn() === 0) {
                db()->exec("ALTER TABLE {$table} ADD INDEX {$index} ({$column})");
                self::log("Added missing index {$table}.{$index}");
            }
        } catch (Throwable $e) {
            self::log("Index check/add failed for {$table}.{$index}: " . $e->getMessage());
        }
    }

    private static function cleanupOldEvents(): void
    {
        try {
            db()->prepare("DELETE FROM news_events WHERE event_timestamp < ?")->execute([
                time() - (14 * 86400),
            ]);
        } catch (Throwable $e) {
            self::log('Old news cleanup failed: ' . $e->getMessage());
        }
    }

    /**
     * ------------------------------------------------------------
     * HELPERS
     * ------------------------------------------------------------
     */
    private static function apiKey(array $providers): ?string
    {
        foreach ($providers as $provider) {
            try {
                if (class_exists('SystemConfigService') && method_exists('SystemConfigService', 'getApiKey')) {
                    $key = SystemConfigService::getApiKey($provider);
                    if ($key) {
                        return (string)$key;
                    }
                }
            } catch (Throwable $e) {
                self::log("SystemConfigService lookup failed for {$provider}: " . $e->getMessage());
            }

            try {
                $stmt = db()->prepare("SELECT api_key FROM api_settings WHERE LOWER(provider) = LOWER(?) AND is_active = 1 LIMIT 1");
                $stmt->execute([$provider]);
                $key = $stmt->fetchColumn();
                if ($key) {
                    return (string)$key;
                }
            } catch (Throwable $e) {
                self::log("DB API lookup failed for {$provider}: " . $e->getMessage());
            }
        }

        return null;
    }

    private static function httpGet(string $url, array $headers = []): ?string
    {
        $ch = curl_init();

        curl_setopt_array($ch, [
            CURLOPT_URL => $url,
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_TIMEOUT => 30,
            CURLOPT_CONNECTTIMEOUT => 15,
            CURLOPT_SSL_VERIFYPEER => false,
            CURLOPT_ENCODING => '',
            CURLOPT_HTTPHEADER => $headers,
        ]);

        $body = curl_exec($ch);
        $http = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE);
        $error = curl_error($ch);

        curl_close($ch);

        if ($body === false || $http < 200 || $http >= 300) {
            self::log("HTTP GET failed: " . self::maskApiKey($url) . " | HTTP={$http} | ERROR={$error}");
            return null;
        }

        return (string)$body;
    }

    private static function httpJson(string $url): array
    {
        $body = self::httpGet($url, [
            'Accept: application/json',
            'User-Agent: Mozilla/5.0 compatible TradingNewsBot/1.0',
        ]);

        if (!$body) {
            return [];
        }

        $json = json_decode($body, true);

        if (!is_array($json)) {
            self::log('JSON decode failed for ' . self::maskApiKey($url));
            return [];
        }

        return $json;
    }

    private static function nodeText(DOMXPath $xpath, string $query, DOMNode $context): string
    {
        $node = $xpath->query($query, $context)->item(0);

        if (!$node) {
            return '';
        }

        return trim(preg_replace('/\s+/', ' ', html_entity_decode($node->textContent, ENT_QUOTES | ENT_HTML5, 'UTF-8')));
    }

    private static function parseGenericTimestamp($value, string $assumedTimezone = 'UTC'): ?int
    {
        if (!$value) {
            return null;
        }

        if (is_numeric($value)) {
            $ts = (int)$value;
            return $ts > 20000000000 ? (int)floor($ts / 1000) : $ts;
        }

        try {
            $raw = trim((string)$value);

            if ($raw === '') {
                return null;
            }

            $hasTimezone = (bool)preg_match('/(Z|[+\-]\d{2}:?\d{2}|UTC|GMT)$/i', $raw);

            $dt = $hasTimezone
                ? new DateTime($raw)
                : new DateTime($raw, new DateTimeZone($assumedTimezone));

            $dt->setTimezone(new DateTimeZone(self::LOCAL_TZ));

            return $dt->getTimestamp();
        } catch (Throwable $e) {
            self::log('Timestamp parse failed for "' . (string)$value . '": ' . $e->getMessage());
            return null;
        }
    }

    private static function normalizeCurrency(string $value): string
    {
        $value = strtoupper(trim($value));

        if ($value === '') {
            return '';
        }

        $map = [
            'UNITED STATES' => 'USD',
            'UNITED STATES OF AMERICA' => 'USD',
            'USA' => 'USD',
            'US' => 'USD',
            'U.S.' => 'USD',
            'EURO AREA' => 'EUR',
            'EUROZONE' => 'EUR',
            'EUROPEAN UNION' => 'EUR',
            'GERMANY' => 'EUR',
            'FRANCE' => 'EUR',
            'ITALY' => 'EUR',
            'SPAIN' => 'EUR',
            'UNITED KINGDOM' => 'GBP',
            'UK' => 'GBP',
            'GREAT BRITAIN' => 'GBP',
            'JAPAN' => 'JPY',
            'CANADA' => 'CAD',
            'AUSTRALIA' => 'AUD',
            'NEW ZEALAND' => 'NZD',
            'SWITZERLAND' => 'CHF',
            'CHINA' => 'CNY',
        ];

        if (isset($map[$value])) {
            return $map[$value];
        }

        if (preg_match('/^[A-Z]{3}$/', $value)) {
            return $value;
        }

        return $value;
    }

    private static function countryToCurrency(string $country): string
    {
        $currency = self::normalizeCurrency($country);

        return $currency !== '' ? $currency : self::inferCurrencyFromTitle($country);
    }

    private static function inferCurrencyFromTitle(string $title): string
    {
        $upper = strtoupper($title);

        $map = [
            'USD' => ['USD', 'DOLLAR', 'FED', 'FOMC', 'POWELL', 'UNITED STATES', 'U.S.', 'NFP', 'NONFARM', 'JOBLESS'],
            'EUR' => ['EUR', 'EURO', 'ECB', 'LAGARDE', 'EUROZONE', 'GERMANY', 'FRANCE'],
            'GBP' => ['GBP', 'POUND', 'BOE', 'BAILEY', 'UNITED KINGDOM', 'UK '],
            'JPY' => ['JPY', 'YEN', 'BOJ', 'JAPAN'],
            'CAD' => ['CAD', 'CANADA', 'BOC'],
            'AUD' => ['AUD', 'AUSTRALIA', 'RBA'],
            'NZD' => ['NZD', 'NEW ZEALAND', 'RBNZ'],
            'CHF' => ['CHF', 'SWISS', 'SNB'],
            'CNY' => ['CNY', 'CHINA', 'PBOC'],
        ];

        foreach ($map as $currency => $needles) {
            foreach ($needles as $needle) {
                if (self::containsTextSafe($upper, $needle)) {
                    return $currency;
                }
            }
        }

        return 'USD';
    }

    private static function cleanTitle(string $title): string
    {
        $title = html_entity_decode($title, ENT_QUOTES | ENT_HTML5, 'UTF-8');
        $title = preg_replace('/\s+/', ' ', $title);

        return trim((string)$title);
    }

    private static function nullableString($value): ?string
    {
        if ($value === null) {
            return null;
        }

        $text = trim((string)$value);

        if ($text === '') {
            return null;
        }

        return mb_substr($text, 0, 100);
    }

    private static function eventUid(string $title, string $currency, int $timestamp): string
    {
        /*
         * Provider excluded intentionally so the same event from different providers
         * updates one row instead of duplicating.
         */
        return md5(strtoupper(self::cleanTitle($title)) . '|' . strtoupper($currency) . '|' . date('Y-m-d H:i', $timestamp));
    }

    private static function outputHeader(): void
    {
        if (PHP_SAPI !== 'cli' && !headers_sent()) {
            header('Content-Type: text/plain; charset=utf-8');
        }
    }

    private static function maskApiKey(string $text): string
    {
        return preg_replace('/(apikey|api_key|c)=([^&\s]+)/i', '$1=***', $text);
    }

    private static function log(string $message): void
    {
        try {
            $dir = __DIR__ . '/../storage/logs';

            if (!is_dir($dir)) {
                @mkdir($dir, 0755, true);
            }

            $line = '[' . date('Y-m-d H:i:s') . '] ' . self::maskApiKey($message) . PHP_EOL;

            @file_put_contents($dir . '/' . self::LOG_FILE, $line, FILE_APPEND);
        } catch (Throwable $e) {
            // Logging should never break cron execution.
        }
    }
}

FetchNewsEventsCron::run();
