Augment missing weather data from METAR when possible; use centralized error handling

- Add utility function to augment missing weather observation data from METAR
- Switch from json() to safeJson() for centralized error handling
- Data quality validation and age checks
- Add null/undefined value handling for wind direction calculations

server/scripts/modules/currentweather.mjs

@@ -1,26 +1,21 @@
 // current weather conditions display
 import STATUS from './status.mjs';
 import { preloadImg } from './utils/image.mjs';
-import { json } from './utils/fetch.mjs';
+import { safeJson } from './utils/fetch.mjs';
 import { directionToNSEW } from './utils/calc.mjs';
 import { locationCleanup } from './utils/string.mjs';
 import { getLargeIcon } from './icons.mjs';
 import WeatherDisplay from './weatherdisplay.mjs';
 import { registerDisplay } from './navigation.mjs';
+import augmentObservationWithMetar from './utils/metar.mjs';
 import {
 	temperature, windSpeed, pressure, distanceMeters, distanceKilometers,
 } from './utils/units.mjs';
+import { debugFlag } from './utils/debug.mjs';
 
 // some stations prefixed do not provide all the necessary data
 const skipStations = ['U', 'C', 'H', 'W', 'Y', 'T', 'S', 'M', 'O', 'L', 'A', 'F', 'B', 'N', 'V', 'R', 'D', 'E', 'I', 'G', 'J'];
 
-const REQUIRED_VALUES = [
-	'windSpeed',
-	'dewpoint',
-	'barometricPressure',
-	'visibility',
-	'relativeHumidity',
-];
 class CurrentWeather extends WeatherDisplay {
 	constructor(navId, elemId) {
 		super(navId, elemId, 'Current Conditions', true);
@@ -45,47 +40,93 @@ class CurrentWeather extends WeatherDisplay {
 			// get the station
 			station = filteredStations[stationNum];
 			stationNum += 1;
+
+			let candidateObservation;
 			try {
-				// station observations
 				// eslint-disable-next-line no-await-in-loop
-				observations = await json(`${station.id}/observations`, {
+				candidateObservation = await safeJson(`${station.id}/observations`, {
 					data: {
-						limit: 2,
+						limit: 2, // we need the two most recent observations to calculate pressure direction
 					},
 					retryCount: 3,
 					stillWaiting: () => this.stillWaiting(),
 				});
-
-				if (observations.features.length === 0) throw new Error(`No features returned for station: ${station.properties.stationIdentifier}, trying next station`);
-
-				// one weather value in the right side column is allowed to be missing. Count them up.
-				// eslint-disable-next-line no-loop-func
-				const valuesCount = REQUIRED_VALUES.reduce((prev, cur) => {
-					const value = observations.features[0].properties?.[cur]?.value;
-					if (value !== null && value !== undefined) return prev + 1;
-					// ceiling is a special case :,-(
-					const ceiling = observations.features[0].properties?.cloudLayers[0]?.base?.value;
-					if (cur === 'ceiling' && ceiling !== null && ceiling !== undefined) return prev + 1;
-					return prev;
-				}, 0);
-
-				// test data quality
-				if (observations.features[0].properties.temperature.value === null
-					|| observations.features[0].properties.textDescription === null
-					|| observations.features[0].properties.textDescription === ''
-					|| observations.features[0].properties.icon === null
-					|| valuesCount < REQUIRED_VALUES.length - 1) {
-					observations = undefined;
-					throw new Error(`Incomplete data set for: ${station.properties.stationIdentifier}, trying next station`);
-				}
 			} catch (error) {
-				console.error(error);
-				observations = undefined;
+				console.error(`Unexpected error getting Current Conditions for station ${station.properties.stationIdentifier}: ${error.message} (trying next station)`);
+				candidateObservation = undefined;
+			}
+
+			// Check if request was successful and has data
+			if (candidateObservation && candidateObservation.features?.length > 0) {
+				// Check if the observation data is old
+				const observationTime = new Date(candidateObservation.features[0].properties.timestamp);
+				const ageInMinutes = (new Date() - observationTime) / (1000 * 60);
+
+				if (ageInMinutes > 180 && debugFlag('currentweather')) {
+					console.warn(`Current Observations for station ${station.properties.stationIdentifier} are ${ageInMinutes.toFixed(0)} minutes old (from ${observationTime.toISOString()}), trying next station`);
+				}
+
+				// Attempt making observation data usable with METAR data
+				const originalData = { ...candidateObservation.features[0].properties };
+				candidateObservation.features[0].properties = augmentObservationWithMetar(candidateObservation.features[0].properties);
+				const metarFields = [
+					{ name: 'temperature', check: (orig, metar) => orig.temperature?.value === null && metar.temperature?.value !== null },
+					{ name: 'windSpeed', check: (orig, metar) => orig.windSpeed?.value === null && metar.windSpeed?.value !== null },
+					{ name: 'windDirection', check: (orig, metar) => orig.windDirection?.value === null && metar.windDirection?.value !== null },
+					{ name: 'windGust', check: (orig, metar) => orig.windGust?.value === null && metar.windGust?.value !== null },
+					{ name: 'dewpoint', check: (orig, metar) => orig.dewpoint?.value === null && metar.dewpoint?.value !== null },
+					{ name: 'barometricPressure', check: (orig, metar) => orig.barometricPressure?.value === null && metar.barometricPressure?.value !== null },
+					{ name: 'relativeHumidity', check: (orig, metar) => orig.relativeHumidity?.value === null && metar.relativeHumidity?.value !== null },
+					{ name: 'visibility', check: (orig, metar) => orig.visibility?.value === null && metar.visibility?.value !== null },
+					{ name: 'ceiling', check: (orig, metar) => orig.cloudLayers?.[0]?.base?.value === null && metar.cloudLayers?.[0]?.base?.value !== null },
+				];
+				const augmentedData = candidateObservation.features[0].properties;
+				const metarReplacements = metarFields.filter((field) => field.check(originalData, augmentedData)).map((field) => field.name);
+				if (debugFlag('currentweather') && metarReplacements.length > 0) {
+					console.log(`Current Conditions for station ${station.properties.stationIdentifier} were augmented with METAR data for ${metarReplacements.join(', ')}`);
+				}
+
+				// test data quality - check required fields and allow one optional field to be missing
+				const requiredFields = [
+					{ name: 'temperature', check: (props) => props.temperature?.value === null, required: true },
+					{ name: 'textDescription', check: (props) => props.textDescription === null || props.textDescription === '', required: true },
+					{ name: 'icon', check: (props) => props.icon === null, required: true },
+					{ name: 'windSpeed', check: (props) => props.windSpeed?.value === null, required: false },
+					{ name: 'dewpoint', check: (props) => props.dewpoint?.value === null, required: false },
+					{ name: 'barometricPressure', check: (props) => props.barometricPressure?.value === null, required: false },
+					{ name: 'visibility', check: (props) => props.visibility?.value === null, required: false },
+					{ name: 'relativeHumidity', check: (props) => props.relativeHumidity?.value === null, required: false },
+					{ name: 'ceiling', check: (props) => props.cloudLayers?.[0]?.base?.value === null, required: false },
+				];
+
+				const missingRequired = requiredFields.filter((field) => field.required && field.check(augmentedData)).map((field) => field.name);
+				const missingOptional = requiredFields.filter((field) => !field.required && field.check(augmentedData)).map((field) => field.name);
+				const missingOptionalCount = missingOptional.length;
+
+				// Allow one optional field to be missing
+				if (missingRequired.length === 0 && missingOptionalCount <= 1) {
+					// Station data is good, use it
+					observations = candidateObservation;
+					if (debugFlag('currentweather') && missingOptional.length > 0) {
+						console.log(`Data for station ${station.properties.stationIdentifier} is missing optional fields: ${missingOptional.join(', ')} (acceptable)`);
+					}
+				} else {
+					const allMissing = [...missingRequired, ...missingOptional];
+					if (debugFlag('currentweather')) {
+						console.log(`Data for station ${station.properties.stationIdentifier} is missing fields: ${allMissing.join(', ')} (${missingRequired.length} required, ${missingOptionalCount} optional) (trying next station)`);
+					}
+				}
+			} else if (debugFlag('verbose-failures')) {
+				if (!candidateObservation) {
+					console.log(`Current Observations for station ${station.properties.stationIdentifier} failed, trying next station`);
+				} else {
+					console.log(`No features returned for station ${station.properties.stationIdentifier}, trying next station`);
+				}
 			}
 		}
 		// test for data received
 		if (!observations) {
-			console.error('All current weather stations exhausted');
+			console.error('Current Conditions failure: all nearby weather stations exhausted!');
 			if (this.isEnabled) this.setStatus(STATUS.failed);
 			// send failed to subscribers
 			this.getDataCallback(undefined);

server/scripts/modules/latestobservations.mjs

@@ -1,12 +1,14 @@
 // current weather conditions display
 import { distance as calcDistance, directionToNSEW } from './utils/calc.mjs';
-import { json } from './utils/fetch.mjs';
+import { safeJson, safePromiseAll } from './utils/fetch.mjs';
 import STATUS from './status.mjs';
 import { locationCleanup } from './utils/string.mjs';
 import { temperature, windSpeed } from './utils/units.mjs';
 import WeatherDisplay from './weatherdisplay.mjs';
 import { registerDisplay } from './navigation.mjs';
+import augmentObservationWithMetar from './utils/metar.mjs';
 import settings from './settings.mjs';
+import { debugFlag } from './utils/debug.mjs';
 
 class LatestObservations extends WeatherDisplay {
 	constructor(navId, elemId) {
@@ -32,14 +34,17 @@ class LatestObservations extends WeatherDisplay {
 		// try up to 30 regional stations
 		const regionalStations = sortedStations.slice(0, 30);
 
-		// get data for regional stations
-		// get first 7 stations
+		// Fetch stations sequentially in batches to avoid unnecessary API calls.
+		// We start with the 7 closest stations and only fetch more if some fail,
+		// stopping as soon as we have 7 valid stations with data.
 		const actualConditions = [];
 		let lastStation = Math.min(regionalStations.length, 7);
 		let firstStation = 0;
 		while (actualConditions.length < 7 && (lastStation) <= regionalStations.length) {
+			// Sequential fetching is intentional here - we want to try closest stations first
+			// and only fetch additional batches if needed, rather than hitting all 30 stations at once
 			// eslint-disable-next-line no-await-in-loop
-			const someStations = await getStations(regionalStations.slice(firstStation, lastStation));
+			const someStations = await this.getStations(regionalStations.slice(firstStation, lastStation));
 
 			actualConditions.push(...someStations);
 			// update counters
@@ -58,6 +63,76 @@ class LatestObservations extends WeatherDisplay {
 		this.setStatus(STATUS.loaded);
 	}
 
+	// This is a class method because it needs access to the instance's `stillWaiting` method
+	async getStations(stations) {
+		// Use centralized safe Promise handling to avoid unhandled AbortError rejections
+		const stationData = await safePromiseAll(stations.map(async (station) => {
+			try {
+				const data = await safeJson(`https://api.weather.gov/stations/${station.id}/observations/latest`, {
+					retryCount: 1,
+					stillWaiting: () => this.stillWaiting(),
+				});
+
+				if (!data) {
+					if (debugFlag('verbose-failures')) {
+						console.log(`Failed to get Latest Observations for station ${station.id}`);
+					}
+					return false;
+				}
+
+				// Check if the observation data is old
+				const observationTime = new Date(data.properties.timestamp);
+				const ageInMinutes = (new Date() - observationTime) / (1000 * 60);
+
+				if (ageInMinutes > 180 && debugFlag('latestobservations')) {
+					console.warn(`Latest Observations for station ${station.id} are ${ageInMinutes.toFixed(0)} minutes old (from ${observationTime.toISOString()})`);
+				}
+
+				// Enhance observation data with METAR parsing for missing fields
+				const originalData = { ...data.properties };
+				data.properties = augmentObservationWithMetar(data.properties);
+				const metarFields = [
+					{ name: 'temperature', check: (orig, metar) => orig.temperature.value === null && metar.temperature.value !== null },
+					{ name: 'windSpeed', check: (orig, metar) => orig.windSpeed.value === null && metar.windSpeed.value !== null },
+					{ name: 'windDirection', check: (orig, metar) => orig.windDirection.value === null && metar.windDirection.value !== null },
+				];
+				const augmentedData = data.properties;
+				const metarReplacements = metarFields.filter((field) => field.check(originalData, augmentedData)).map((field) => field.name);
+				if (debugFlag('latestobservations') && metarReplacements.length > 0) {
+					console.log(`Latest Observations for station ${station.id} were augmented with METAR data for ${metarReplacements.join(', ')}`);
+				}
+
+				// test data quality
+				const requiredFields = [
+					{ name: 'temperature', check: (props) => props.temperature?.value === null },
+					{ name: 'windSpeed', check: (props) => props.windSpeed?.value === null },
+					{ name: 'windDirection', check: (props) => props.windDirection?.value === null },
+					{ name: 'textDescription', check: (props) => props.textDescription === null || props.textDescription === '' },
+				];
+				const missingFields = requiredFields.filter((field) => field.check(augmentedData)).map((field) => field.name);
+
+				if (missingFields.length > 0) {
+					if (debugFlag('latestobservations')) {
+						console.log(`Latest Observations for station ${station.id} are missing required fields: ${missingFields.join(', ')}`);
+					}
+					return false;
+				}
+
+				// format the return values
+				return {
+					...data.properties,
+					StationId: station.id,
+					city: station.city,
+				};
+			} catch (error) {
+				console.error(`Unexpected error getting latest observations for station ${station.id}: ${error.message}`);
+				return false;
+			}
+		}));
+		// filter false (no data or other error)
+		return stationData.filter((d) => d);
+	}
+
 	async drawCanvas() {
 		super.drawCanvas();
 		const conditions = this.data;
@@ -106,6 +181,7 @@ class LatestObservations extends WeatherDisplay {
 		this.finishDraw();
 	}
 }
+
 const shortenCurrentConditions = (_condition) => {
 	let condition = _condition;
 	condition = condition.replace(/Light/, 'L');
@@ -124,28 +200,5 @@ const shortenCurrentConditions = (_condition) => {
 	condition = condition.replace(/ with /, '/');
 	return condition;
 };
-
-const getStations = async (stations) => {
-	const stationData = await Promise.all(stations.map(async (station) => {
-		try {
-			const data = await json(`https://api.weather.gov/stations/${station.id}/observations/latest`, { retryCount: 1, stillWaiting: () => this.stillWaiting() });
-			// test for temperature, weather and wind values present
-			if (data.properties.temperature.value === null
-				|| data.properties.textDescription === ''
-				|| data.properties.windSpeed.value === null) return false;
-			// format the return values
-			return {
-				...data.properties,
-				StationId: station.id,
-				city: station.city,
-			};
-		} catch {
-			console.log(`Unable to get latest observations for ${station.id}`);
-			return false;
-		}
-	}));
-	// filter false (no data or other error)
-	return stationData.filter((d) => d);
-};
 // register display
 registerDisplay(new LatestObservations(2, 'latest-observations'));

server/scripts/modules/regionalforecast-utils.mjs

@@ -1,7 +1,9 @@
 import { getSmallIcon } from './icons.mjs';
 import { preloadImg } from './utils/image.mjs';
-import { json } from './utils/fetch.mjs';
+import { safeJson } from './utils/fetch.mjs';
 import { temperature as temperatureUnit } from './utils/units.mjs';
+import augmentObservationWithMetar from './utils/metar.mjs';
+import { debugFlag } from './utils/debug.mjs';
 
 const buildForecast = (forecast, city, cityXY) => {
 	// get a unit converter
@@ -19,23 +21,40 @@ const buildForecast = (forecast, city, cityXY) => {
 
 const getRegionalObservation = async (point, city) => {
 	try {
-		// get stations
-		const stations = await json(`https://api.weather.gov/gridpoints/${point.wfo}/${point.x},${point.y}/stations?limit=1`);
+		// get stations using centralized safe handling
+		const stations = await safeJson(`https://api.weather.gov/gridpoints/${point.wfo}/${point.x},${point.y}/stations?limit=1`);
+
+		if (!stations || !stations.features || stations.features.length === 0) {
+			if (debugFlag('verbose-failures')) {
+				console.warn(`Unable to get regional stations for ${city.Name ?? city.city}`);
+			}
+			return false;
+		}
 
 		// get the first station
 		const station = stations.features[0].id;
-		// get the observation data
-		const observation = await json(`${station}/observations/latest`);
+		// get the observation data using centralized safe handling
+		const observation = await safeJson(`${station}/observations/latest`);
+
+		if (!observation) {
+			if (debugFlag('verbose-failures')) {
+				console.warn(`Unable to get regional observations for ${city.Name ?? city.city}`);
+			}
+			return false;
+		}
+
+		// Enhance observation data with METAR parsing for missing fields
+		const augmentedObservation = augmentObservationWithMetar(observation.properties);
+
 		// preload the image
-		if (!observation.properties.icon) return false;
-		const icon = getSmallIcon(observation.properties.icon, !observation.properties.daytime);
+		if (!augmentedObservation.icon) return false;
+		const icon = getSmallIcon(augmentedObservation.icon, !augmentedObservation.daytime);
 		if (!icon) return false;
 		preloadImg(icon);
 		// return the observation
-		return observation.properties;
+		return augmentedObservation;
 	} catch (error) {
-		console.log(`Unable to get regional observations for ${city.Name ?? city.city}`);
-		console.error(error.status, error.responseJSON);
+		console.error(`Unexpected error getting Regional Observation for ${city.Name ?? city.city}: ${error.message}`);
 		return false;
 	}
 };

server/scripts/modules/utils/calc.mjs

@@ -1,5 +1,9 @@
 // wind direction
 const directionToNSEW = (Direction) => {
+	// Handle null, undefined, or invalid direction values
+	if (Direction === null || Direction === undefined || typeof Direction !== 'number' || Number.isNaN(Direction)) {
+		return 'VAR'; // Variable (or unknown) direction
+	}
 	const val = Math.floor((Direction / 22.5) + 0.5);
 	const arr = ['N', 'NNE', 'NE', 'ENE', 'E', 'ESE', 'SE', 'SSE', 'S', 'SSW', 'SW', 'WSW', 'W', 'WNW', 'NW', 'NNW'];
 	return arr[(val % 16)];

server/scripts/modules/utils/metar.mjs

@@ -0,0 +1,153 @@
+// METAR parsing utilities using metar-taf-parser library
+import { parseMetar } from '../../vendor/auto/metar-taf-parser.mjs';
+
+/**
+ * Augment observation data by parsing METAR when API fields are missing
+ * @param {Object} observation - The observation object from the API
+ * @returns {Object} - Augmented observation with parsed METAR data filled in
+ */
+const augmentObservationWithMetar = (observation) => {
+	if (!observation?.rawMessage) {
+		return observation;
+	}
+
+	const metar = { ...observation };
+
+	try {
+		const metarData = parseMetar(observation.rawMessage);
+
+		if (observation.windSpeed?.value === null && metarData.wind?.speed !== undefined) {
+			metar.windSpeed = {
+				...observation.windSpeed,
+				value: metarData.wind.speed * 1.852, // Convert knots to km/h (API uses km/h)
+				qualityControl: 'M', // M for METAR-derived
+			};
+		}
+
+		if (observation.windDirection?.value === null && metarData.wind?.degrees !== undefined) {
+			metar.windDirection = {
+				...observation.windDirection,
+				value: metarData.wind.degrees,
+				qualityControl: 'M',
+			};
+		}
+
+		if (observation.windGust?.value === null && metarData.wind?.gust !== undefined) {
+			metar.windGust = {
+				...observation.windGust,
+				value: metarData.wind.gust * 1.852, // Convert knots to km/h
+				qualityControl: 'M',
+			};
+		}
+
+		if (observation.temperature?.value === null && metarData.temperature !== undefined) {
+			metar.temperature = {
+				...observation.temperature,
+				value: metarData.temperature,
+				qualityControl: 'M',
+			};
+		}
+
+		if (observation.dewpoint?.value === null && metarData.dewPoint !== undefined) {
+			metar.dewpoint = {
+				...observation.dewpoint,
+				value: metarData.dewPoint,
+				qualityControl: 'M',
+			};
+		}
+
+		if (observation.barometricPressure?.value === null && metarData.altimeter !== undefined) {
+			// Convert inHg to Pascals
+			const pascals = Math.round(metarData.altimeter * 3386.39);
+			metar.barometricPressure = {
+				...observation.barometricPressure,
+				value: pascals,
+				qualityControl: 'M',
+			};
+		}
+
+		// Calculate relative humidity if missing from API but we have temp and dewpoint
+		if (observation.relativeHumidity?.value === null && metar.temperature?.value !== null && metar.dewpoint?.value !== null) {
+			const humidity = calculateRelativeHumidity(metar.temperature.value, metar.dewpoint.value);
+			metar.relativeHumidity = {
+				...observation.relativeHumidity,
+				value: humidity,
+				qualityControl: 'M', // M for METAR-derived
+			};
+		}
+
+		if (observation.visibility?.value === null && metarData.visibility?.value !== undefined) {
+			let visibilityKm;
+			if (metarData.visibility.unit === 'SM') {
+				// Convert statute miles to kilometers
+				visibilityKm = metarData.visibility.value * 1.609344;
+			} else if (metarData.visibility.unit === 'm') {
+				// Convert meters to kilometers
+				visibilityKm = metarData.visibility.value / 1000;
+			} else {
+				// Assume it's already in the right unit
+				visibilityKm = metarData.visibility.value;
+			}
+
+			metar.visibility = {
+				...observation.visibility,
+				value: Math.round(visibilityKm * 10) / 10, // Round to 1 decimal place
+				qualityControl: 'M',
+			};
+		}
+
+		if (observation.cloudLayers?.[0]?.base?.value === null && metarData.clouds?.length > 0) {
+			// Find the lowest broken (BKN) or overcast (OVC) layer for ceiling
+			const ceilingLayer = metarData.clouds
+				.filter((cloud) => cloud.type === 'BKN' || cloud.type === 'OVC')
+				.sort((a, b) => a.height - b.height)[0];
+
+			if (ceilingLayer) {
+				// Convert feet to meters
+				const heightMeters = Math.round(ceilingLayer.height * 0.3048);
+
+				// Create cloud layer structure if it doesn't exist
+				if (!metar.cloudLayers || !metar.cloudLayers[0]) {
+					metar.cloudLayers = [{
+						base: {
+							value: heightMeters,
+							qualityControl: 'M',
+						},
+					}];
+				} else {
+					metar.cloudLayers[0].base = {
+						...observation.cloudLayers[0].base,
+						value: heightMeters,
+						qualityControl: 'M',
+					};
+				}
+			}
+		}
+	} catch (error) {
+		// If METAR parsing fails, just return the original observation
+		console.warn(`Failed to parse METAR: ${error.message}`);
+		return observation;
+	}
+
+	return metar;
+};
+
+/**
+ * Calculate relative humidity from temperature and dewpoint
+ * @param {number} temperature - Temperature in Celsius
+ * @param {number} dewpoint - Dewpoint in Celsius
+ * @returns {number} Relative humidity as a percentage (0-100)
+ */
+const calculateRelativeHumidity = (temperature, dewpoint) => {
+	// Using the Magnus formula approximation
+	const a = 17.625;
+	const b = 243.04;
+
+	const alpha = Math.log(Math.exp((a * dewpoint) / (b + dewpoint)) / Math.exp((a * temperature) / (b + temperature)));
+	const relativeHumidity = Math.exp(alpha) * 100;
+
+	// Clamp between 0 and 100 and round to nearest integer
+	return Math.round(Math.max(0, Math.min(100, relativeHumidity)));
+};
+
+export default augmentObservationWithMetar;