From 79de691eef16a9a3e70a237f76c555f5d71665ed Mon Sep 17 00:00:00 2001 From: Eddy G Date: Tue, 24 Jun 2025 23:05:51 -0400 Subject: [PATCH] Augment missing weather data from METAR when possible; use centralized error handling - Add utility function to augment missing weather observation data from METAR - Switch from json() to safeJson() for centralized error handling - Data quality validation and age checks - Add null/undefined value handling for wind direction calculations --- server/scripts/modules/currentweather.mjs | 115 ++++++++----- server/scripts/modules/latestobservations.mjs | 107 ++++++++---- .../modules/regionalforecast-utils.mjs | 39 +++-- server/scripts/modules/utils/calc.mjs | 4 + server/scripts/modules/utils/metar.mjs | 153 ++++++++++++++++++ 5 files changed, 344 insertions(+), 74 deletions(-) create mode 100644 server/scripts/modules/utils/metar.mjs diff --git a/server/scripts/modules/currentweather.mjs b/server/scripts/modules/currentweather.mjs index 4280c1a..51db3bb 100644 --- a/server/scripts/modules/currentweather.mjs +++ b/server/scripts/modules/currentweather.mjs @@ -1,26 +1,21 @@ // current weather conditions display import STATUS from './status.mjs'; import { preloadImg } from './utils/image.mjs'; -import { json } from './utils/fetch.mjs'; +import { safeJson } from './utils/fetch.mjs'; import { directionToNSEW } from './utils/calc.mjs'; import { locationCleanup } from './utils/string.mjs'; import { getLargeIcon } from './icons.mjs'; import WeatherDisplay from './weatherdisplay.mjs'; import { registerDisplay } from './navigation.mjs'; +import augmentObservationWithMetar from './utils/metar.mjs'; import { temperature, windSpeed, pressure, distanceMeters, distanceKilometers, } from './utils/units.mjs'; +import { debugFlag } from './utils/debug.mjs'; // some stations prefixed do not provide all the necessary data const skipStations = ['U', 'C', 'H', 'W', 'Y', 'T', 'S', 'M', 'O', 'L', 'A', 'F', 'B', 'N', 'V', 'R', 'D', 'E', 'I', 'G', 'J']; -const REQUIRED_VALUES = [ - 'windSpeed', - 'dewpoint', - 'barometricPressure', - 'visibility', - 'relativeHumidity', -]; class CurrentWeather extends WeatherDisplay { constructor(navId, elemId) { super(navId, elemId, 'Current Conditions', true); @@ -45,47 +40,93 @@ class CurrentWeather extends WeatherDisplay { // get the station station = filteredStations[stationNum]; stationNum += 1; + + let candidateObservation; try { - // station observations // eslint-disable-next-line no-await-in-loop - observations = await json(`${station.id}/observations`, { + candidateObservation = await safeJson(`${station.id}/observations`, { data: { - limit: 2, + limit: 2, // we need the two most recent observations to calculate pressure direction }, retryCount: 3, stillWaiting: () => this.stillWaiting(), }); - - if (observations.features.length === 0) throw new Error(`No features returned for station: ${station.properties.stationIdentifier}, trying next station`); - - // one weather value in the right side column is allowed to be missing. Count them up. - // eslint-disable-next-line no-loop-func - const valuesCount = REQUIRED_VALUES.reduce((prev, cur) => { - const value = observations.features[0].properties?.[cur]?.value; - if (value !== null && value !== undefined) return prev + 1; - // ceiling is a special case :,-( - const ceiling = observations.features[0].properties?.cloudLayers[0]?.base?.value; - if (cur === 'ceiling' && ceiling !== null && ceiling !== undefined) return prev + 1; - return prev; - }, 0); - - // test data quality - if (observations.features[0].properties.temperature.value === null - || observations.features[0].properties.textDescription === null - || observations.features[0].properties.textDescription === '' - || observations.features[0].properties.icon === null - || valuesCount < REQUIRED_VALUES.length - 1) { - observations = undefined; - throw new Error(`Incomplete data set for: ${station.properties.stationIdentifier}, trying next station`); - } } catch (error) { - console.error(error); - observations = undefined; + console.error(`Unexpected error getting Current Conditions for station ${station.properties.stationIdentifier}: ${error.message} (trying next station)`); + candidateObservation = undefined; + } + + // Check if request was successful and has data + if (candidateObservation && candidateObservation.features?.length > 0) { + // Check if the observation data is old + const observationTime = new Date(candidateObservation.features[0].properties.timestamp); + const ageInMinutes = (new Date() - observationTime) / (1000 * 60); + + if (ageInMinutes > 180 && debugFlag('currentweather')) { + console.warn(`Current Observations for station ${station.properties.stationIdentifier} are ${ageInMinutes.toFixed(0)} minutes old (from ${observationTime.toISOString()}), trying next station`); + } + + // Attempt making observation data usable with METAR data + const originalData = { ...candidateObservation.features[0].properties }; + candidateObservation.features[0].properties = augmentObservationWithMetar(candidateObservation.features[0].properties); + const metarFields = [ + { name: 'temperature', check: (orig, metar) => orig.temperature?.value === null && metar.temperature?.value !== null }, + { name: 'windSpeed', check: (orig, metar) => orig.windSpeed?.value === null && metar.windSpeed?.value !== null }, + { name: 'windDirection', check: (orig, metar) => orig.windDirection?.value === null && metar.windDirection?.value !== null }, + { name: 'windGust', check: (orig, metar) => orig.windGust?.value === null && metar.windGust?.value !== null }, + { name: 'dewpoint', check: (orig, metar) => orig.dewpoint?.value === null && metar.dewpoint?.value !== null }, + { name: 'barometricPressure', check: (orig, metar) => orig.barometricPressure?.value === null && metar.barometricPressure?.value !== null }, + { name: 'relativeHumidity', check: (orig, metar) => orig.relativeHumidity?.value === null && metar.relativeHumidity?.value !== null }, + { name: 'visibility', check: (orig, metar) => orig.visibility?.value === null && metar.visibility?.value !== null }, + { name: 'ceiling', check: (orig, metar) => orig.cloudLayers?.[0]?.base?.value === null && metar.cloudLayers?.[0]?.base?.value !== null }, + ]; + const augmentedData = candidateObservation.features[0].properties; + const metarReplacements = metarFields.filter((field) => field.check(originalData, augmentedData)).map((field) => field.name); + if (debugFlag('currentweather') && metarReplacements.length > 0) { + console.log(`Current Conditions for station ${station.properties.stationIdentifier} were augmented with METAR data for ${metarReplacements.join(', ')}`); + } + + // test data quality - check required fields and allow one optional field to be missing + const requiredFields = [ + { name: 'temperature', check: (props) => props.temperature?.value === null, required: true }, + { name: 'textDescription', check: (props) => props.textDescription === null || props.textDescription === '', required: true }, + { name: 'icon', check: (props) => props.icon === null, required: true }, + { name: 'windSpeed', check: (props) => props.windSpeed?.value === null, required: false }, + { name: 'dewpoint', check: (props) => props.dewpoint?.value === null, required: false }, + { name: 'barometricPressure', check: (props) => props.barometricPressure?.value === null, required: false }, + { name: 'visibility', check: (props) => props.visibility?.value === null, required: false }, + { name: 'relativeHumidity', check: (props) => props.relativeHumidity?.value === null, required: false }, + { name: 'ceiling', check: (props) => props.cloudLayers?.[0]?.base?.value === null, required: false }, + ]; + + const missingRequired = requiredFields.filter((field) => field.required && field.check(augmentedData)).map((field) => field.name); + const missingOptional = requiredFields.filter((field) => !field.required && field.check(augmentedData)).map((field) => field.name); + const missingOptionalCount = missingOptional.length; + + // Allow one optional field to be missing + if (missingRequired.length === 0 && missingOptionalCount <= 1) { + // Station data is good, use it + observations = candidateObservation; + if (debugFlag('currentweather') && missingOptional.length > 0) { + console.log(`Data for station ${station.properties.stationIdentifier} is missing optional fields: ${missingOptional.join(', ')} (acceptable)`); + } + } else { + const allMissing = [...missingRequired, ...missingOptional]; + if (debugFlag('currentweather')) { + console.log(`Data for station ${station.properties.stationIdentifier} is missing fields: ${allMissing.join(', ')} (${missingRequired.length} required, ${missingOptionalCount} optional) (trying next station)`); + } + } + } else if (debugFlag('verbose-failures')) { + if (!candidateObservation) { + console.log(`Current Observations for station ${station.properties.stationIdentifier} failed, trying next station`); + } else { + console.log(`No features returned for station ${station.properties.stationIdentifier}, trying next station`); + } } } // test for data received if (!observations) { - console.error('All current weather stations exhausted'); + console.error('Current Conditions failure: all nearby weather stations exhausted!'); if (this.isEnabled) this.setStatus(STATUS.failed); // send failed to subscribers this.getDataCallback(undefined); diff --git a/server/scripts/modules/latestobservations.mjs b/server/scripts/modules/latestobservations.mjs index 93e0d2e..dcb69f3 100644 --- a/server/scripts/modules/latestobservations.mjs +++ b/server/scripts/modules/latestobservations.mjs @@ -1,12 +1,14 @@ // current weather conditions display import { distance as calcDistance, directionToNSEW } from './utils/calc.mjs'; -import { json } from './utils/fetch.mjs'; +import { safeJson, safePromiseAll } from './utils/fetch.mjs'; import STATUS from './status.mjs'; import { locationCleanup } from './utils/string.mjs'; import { temperature, windSpeed } from './utils/units.mjs'; import WeatherDisplay from './weatherdisplay.mjs'; import { registerDisplay } from './navigation.mjs'; +import augmentObservationWithMetar from './utils/metar.mjs'; import settings from './settings.mjs'; +import { debugFlag } from './utils/debug.mjs'; class LatestObservations extends WeatherDisplay { constructor(navId, elemId) { @@ -32,14 +34,17 @@ class LatestObservations extends WeatherDisplay { // try up to 30 regional stations const regionalStations = sortedStations.slice(0, 30); - // get data for regional stations - // get first 7 stations + // Fetch stations sequentially in batches to avoid unnecessary API calls. + // We start with the 7 closest stations and only fetch more if some fail, + // stopping as soon as we have 7 valid stations with data. const actualConditions = []; let lastStation = Math.min(regionalStations.length, 7); let firstStation = 0; while (actualConditions.length < 7 && (lastStation) <= regionalStations.length) { + // Sequential fetching is intentional here - we want to try closest stations first + // and only fetch additional batches if needed, rather than hitting all 30 stations at once // eslint-disable-next-line no-await-in-loop - const someStations = await getStations(regionalStations.slice(firstStation, lastStation)); + const someStations = await this.getStations(regionalStations.slice(firstStation, lastStation)); actualConditions.push(...someStations); // update counters @@ -58,6 +63,76 @@ class LatestObservations extends WeatherDisplay { this.setStatus(STATUS.loaded); } + // This is a class method because it needs access to the instance's `stillWaiting` method + async getStations(stations) { + // Use centralized safe Promise handling to avoid unhandled AbortError rejections + const stationData = await safePromiseAll(stations.map(async (station) => { + try { + const data = await safeJson(`https://api.weather.gov/stations/${station.id}/observations/latest`, { + retryCount: 1, + stillWaiting: () => this.stillWaiting(), + }); + + if (!data) { + if (debugFlag('verbose-failures')) { + console.log(`Failed to get Latest Observations for station ${station.id}`); + } + return false; + } + + // Check if the observation data is old + const observationTime = new Date(data.properties.timestamp); + const ageInMinutes = (new Date() - observationTime) / (1000 * 60); + + if (ageInMinutes > 180 && debugFlag('latestobservations')) { + console.warn(`Latest Observations for station ${station.id} are ${ageInMinutes.toFixed(0)} minutes old (from ${observationTime.toISOString()})`); + } + + // Enhance observation data with METAR parsing for missing fields + const originalData = { ...data.properties }; + data.properties = augmentObservationWithMetar(data.properties); + const metarFields = [ + { name: 'temperature', check: (orig, metar) => orig.temperature.value === null && metar.temperature.value !== null }, + { name: 'windSpeed', check: (orig, metar) => orig.windSpeed.value === null && metar.windSpeed.value !== null }, + { name: 'windDirection', check: (orig, metar) => orig.windDirection.value === null && metar.windDirection.value !== null }, + ]; + const augmentedData = data.properties; + const metarReplacements = metarFields.filter((field) => field.check(originalData, augmentedData)).map((field) => field.name); + if (debugFlag('latestobservations') && metarReplacements.length > 0) { + console.log(`Latest Observations for station ${station.id} were augmented with METAR data for ${metarReplacements.join(', ')}`); + } + + // test data quality + const requiredFields = [ + { name: 'temperature', check: (props) => props.temperature?.value === null }, + { name: 'windSpeed', check: (props) => props.windSpeed?.value === null }, + { name: 'windDirection', check: (props) => props.windDirection?.value === null }, + { name: 'textDescription', check: (props) => props.textDescription === null || props.textDescription === '' }, + ]; + const missingFields = requiredFields.filter((field) => field.check(augmentedData)).map((field) => field.name); + + if (missingFields.length > 0) { + if (debugFlag('latestobservations')) { + console.log(`Latest Observations for station ${station.id} are missing required fields: ${missingFields.join(', ')}`); + } + return false; + } + + // format the return values + return { + ...data.properties, + StationId: station.id, + city: station.city, + }; + } catch (error) { + console.error(`Unexpected error getting latest observations for station ${station.id}: ${error.message}`); + return false; + } + })); + // filter false (no data or other error) + return stationData.filter((d) => d); + } + async drawCanvas() { super.drawCanvas(); const conditions = this.data; @@ -106,6 +181,7 @@ class LatestObservations extends WeatherDisplay { this.finishDraw(); } } + const shortenCurrentConditions = (_condition) => { let condition = _condition; condition = condition.replace(/Light/, 'L'); @@ -124,28 +200,5 @@ const shortenCurrentConditions = (_condition) => { condition = condition.replace(/ with /, '/'); return condition; }; - -const getStations = async (stations) => { - const stationData = await Promise.all(stations.map(async (station) => { - try { - const data = await json(`https://api.weather.gov/stations/${station.id}/observations/latest`, { retryCount: 1, stillWaiting: () => this.stillWaiting() }); - // test for temperature, weather and wind values present - if (data.properties.temperature.value === null - || data.properties.textDescription === '' - || data.properties.windSpeed.value === null) return false; - // format the return values - return { - ...data.properties, - StationId: station.id, - city: station.city, - }; - } catch { - console.log(`Unable to get latest observations for ${station.id}`); - return false; - } - })); - // filter false (no data or other error) - return stationData.filter((d) => d); -}; // register display registerDisplay(new LatestObservations(2, 'latest-observations')); diff --git a/server/scripts/modules/regionalforecast-utils.mjs b/server/scripts/modules/regionalforecast-utils.mjs index e5cf702..6080b7b 100644 --- a/server/scripts/modules/regionalforecast-utils.mjs +++ b/server/scripts/modules/regionalforecast-utils.mjs @@ -1,7 +1,9 @@ import { getSmallIcon } from './icons.mjs'; import { preloadImg } from './utils/image.mjs'; -import { json } from './utils/fetch.mjs'; +import { safeJson } from './utils/fetch.mjs'; import { temperature as temperatureUnit } from './utils/units.mjs'; +import augmentObservationWithMetar from './utils/metar.mjs'; +import { debugFlag } from './utils/debug.mjs'; const buildForecast = (forecast, city, cityXY) => { // get a unit converter @@ -19,23 +21,40 @@ const buildForecast = (forecast, city, cityXY) => { const getRegionalObservation = async (point, city) => { try { - // get stations - const stations = await json(`https://api.weather.gov/gridpoints/${point.wfo}/${point.x},${point.y}/stations?limit=1`); + // get stations using centralized safe handling + const stations = await safeJson(`https://api.weather.gov/gridpoints/${point.wfo}/${point.x},${point.y}/stations?limit=1`); + + if (!stations || !stations.features || stations.features.length === 0) { + if (debugFlag('verbose-failures')) { + console.warn(`Unable to get regional stations for ${city.Name ?? city.city}`); + } + return false; + } // get the first station const station = stations.features[0].id; - // get the observation data - const observation = await json(`${station}/observations/latest`); + // get the observation data using centralized safe handling + const observation = await safeJson(`${station}/observations/latest`); + + if (!observation) { + if (debugFlag('verbose-failures')) { + console.warn(`Unable to get regional observations for ${city.Name ?? city.city}`); + } + return false; + } + + // Enhance observation data with METAR parsing for missing fields + const augmentedObservation = augmentObservationWithMetar(observation.properties); + // preload the image - if (!observation.properties.icon) return false; - const icon = getSmallIcon(observation.properties.icon, !observation.properties.daytime); + if (!augmentedObservation.icon) return false; + const icon = getSmallIcon(augmentedObservation.icon, !augmentedObservation.daytime); if (!icon) return false; preloadImg(icon); // return the observation - return observation.properties; + return augmentedObservation; } catch (error) { - console.log(`Unable to get regional observations for ${city.Name ?? city.city}`); - console.error(error.status, error.responseJSON); + console.error(`Unexpected error getting Regional Observation for ${city.Name ?? city.city}: ${error.message}`); return false; } }; diff --git a/server/scripts/modules/utils/calc.mjs b/server/scripts/modules/utils/calc.mjs index b4c7469..40d6548 100644 --- a/server/scripts/modules/utils/calc.mjs +++ b/server/scripts/modules/utils/calc.mjs @@ -1,5 +1,9 @@ // wind direction const directionToNSEW = (Direction) => { + // Handle null, undefined, or invalid direction values + if (Direction === null || Direction === undefined || typeof Direction !== 'number' || Number.isNaN(Direction)) { + return 'VAR'; // Variable (or unknown) direction + } const val = Math.floor((Direction / 22.5) + 0.5); const arr = ['N', 'NNE', 'NE', 'ENE', 'E', 'ESE', 'SE', 'SSE', 'S', 'SSW', 'SW', 'WSW', 'W', 'WNW', 'NW', 'NNW']; return arr[(val % 16)]; diff --git a/server/scripts/modules/utils/metar.mjs b/server/scripts/modules/utils/metar.mjs new file mode 100644 index 0000000..98b371e --- /dev/null +++ b/server/scripts/modules/utils/metar.mjs @@ -0,0 +1,153 @@ +// METAR parsing utilities using metar-taf-parser library +import { parseMetar } from '../../vendor/auto/metar-taf-parser.mjs'; + +/** + * Augment observation data by parsing METAR when API fields are missing + * @param {Object} observation - The observation object from the API + * @returns {Object} - Augmented observation with parsed METAR data filled in + */ +const augmentObservationWithMetar = (observation) => { + if (!observation?.rawMessage) { + return observation; + } + + const metar = { ...observation }; + + try { + const metarData = parseMetar(observation.rawMessage); + + if (observation.windSpeed?.value === null && metarData.wind?.speed !== undefined) { + metar.windSpeed = { + ...observation.windSpeed, + value: metarData.wind.speed * 1.852, // Convert knots to km/h (API uses km/h) + qualityControl: 'M', // M for METAR-derived + }; + } + + if (observation.windDirection?.value === null && metarData.wind?.degrees !== undefined) { + metar.windDirection = { + ...observation.windDirection, + value: metarData.wind.degrees, + qualityControl: 'M', + }; + } + + if (observation.windGust?.value === null && metarData.wind?.gust !== undefined) { + metar.windGust = { + ...observation.windGust, + value: metarData.wind.gust * 1.852, // Convert knots to km/h + qualityControl: 'M', + }; + } + + if (observation.temperature?.value === null && metarData.temperature !== undefined) { + metar.temperature = { + ...observation.temperature, + value: metarData.temperature, + qualityControl: 'M', + }; + } + + if (observation.dewpoint?.value === null && metarData.dewPoint !== undefined) { + metar.dewpoint = { + ...observation.dewpoint, + value: metarData.dewPoint, + qualityControl: 'M', + }; + } + + if (observation.barometricPressure?.value === null && metarData.altimeter !== undefined) { + // Convert inHg to Pascals + const pascals = Math.round(metarData.altimeter * 3386.39); + metar.barometricPressure = { + ...observation.barometricPressure, + value: pascals, + qualityControl: 'M', + }; + } + + // Calculate relative humidity if missing from API but we have temp and dewpoint + if (observation.relativeHumidity?.value === null && metar.temperature?.value !== null && metar.dewpoint?.value !== null) { + const humidity = calculateRelativeHumidity(metar.temperature.value, metar.dewpoint.value); + metar.relativeHumidity = { + ...observation.relativeHumidity, + value: humidity, + qualityControl: 'M', // M for METAR-derived + }; + } + + if (observation.visibility?.value === null && metarData.visibility?.value !== undefined) { + let visibilityKm; + if (metarData.visibility.unit === 'SM') { + // Convert statute miles to kilometers + visibilityKm = metarData.visibility.value * 1.609344; + } else if (metarData.visibility.unit === 'm') { + // Convert meters to kilometers + visibilityKm = metarData.visibility.value / 1000; + } else { + // Assume it's already in the right unit + visibilityKm = metarData.visibility.value; + } + + metar.visibility = { + ...observation.visibility, + value: Math.round(visibilityKm * 10) / 10, // Round to 1 decimal place + qualityControl: 'M', + }; + } + + if (observation.cloudLayers?.[0]?.base?.value === null && metarData.clouds?.length > 0) { + // Find the lowest broken (BKN) or overcast (OVC) layer for ceiling + const ceilingLayer = metarData.clouds + .filter((cloud) => cloud.type === 'BKN' || cloud.type === 'OVC') + .sort((a, b) => a.height - b.height)[0]; + + if (ceilingLayer) { + // Convert feet to meters + const heightMeters = Math.round(ceilingLayer.height * 0.3048); + + // Create cloud layer structure if it doesn't exist + if (!metar.cloudLayers || !metar.cloudLayers[0]) { + metar.cloudLayers = [{ + base: { + value: heightMeters, + qualityControl: 'M', + }, + }]; + } else { + metar.cloudLayers[0].base = { + ...observation.cloudLayers[0].base, + value: heightMeters, + qualityControl: 'M', + }; + } + } + } + } catch (error) { + // If METAR parsing fails, just return the original observation + console.warn(`Failed to parse METAR: ${error.message}`); + return observation; + } + + return metar; +}; + +/** + * Calculate relative humidity from temperature and dewpoint + * @param {number} temperature - Temperature in Celsius + * @param {number} dewpoint - Dewpoint in Celsius + * @returns {number} Relative humidity as a percentage (0-100) + */ +const calculateRelativeHumidity = (temperature, dewpoint) => { + // Using the Magnus formula approximation + const a = 17.625; + const b = 243.04; + + const alpha = Math.log(Math.exp((a * dewpoint) / (b + dewpoint)) / Math.exp((a * temperature) / (b + temperature))); + const relativeHumidity = Math.exp(alpha) * 100; + + // Clamp between 0 and 100 and round to nearest integer + return Math.round(Math.max(0, Math.min(100, relativeHumidity))); +}; + +export default augmentObservationWithMetar;