Replace CORS proxy with complete server-side cache

- Replace cors/ directory and cors.mjs utility with comprehensive
    HTTP caching proxy in proxy/ directory
- Implement RFC-compliant caching with cache-control headers,
    conditional requests, and in-flight deduplication
- Centralized error handling with "safe" fetch utilities
- Add unified proxy handlers for weather.gov, SPC, radar, and mesonet APIs
- Include cache management endpoint and extensive diagnostic logging
- Migrate client-side URL rewriting from cors.mjs to url-rewrite.mjs

server/scripts/modules/utils/cache.mjs

@@ -0,0 +1,40 @@
+import { rewriteUrl } from './url-rewrite.mjs';
+
+// Clear cache utility for client-side use
+const clearCacheEntry = async (url, baseUrl = '') => {
+	try {
+		// Rewrite the URL to get the local proxy path
+		const rewrittenUrl = rewriteUrl(url);
+		const urlObj = typeof rewrittenUrl === 'string' ? new URL(rewrittenUrl, baseUrl || window.location.origin) : rewrittenUrl;
+		let cachePath = urlObj.pathname + urlObj.search;
+
+		// Strip the route designator (first path segment) to match actual cache keys
+		const firstSlashIndex = cachePath.indexOf('/', 1); // Find second slash
+		if (firstSlashIndex > 0) {
+			cachePath = cachePath.substring(firstSlashIndex);
+		}
+
+		// Call the cache clear endpoint
+		const fetchUrl = baseUrl ? `${baseUrl}/cache${cachePath}` : `/cache${cachePath}`;
+		const response = await fetch(fetchUrl, {
+			method: 'DELETE',
+		});
+
+		if (response.ok) {
+			const result = await response.json();
+			if (result.cleared) {
+				console.log(`🗑️ Cleared cache entry: ${cachePath}`);
+				return true;
+			}
+			console.log(`🔍 Cache entry not found: ${cachePath}`);
+			return false;
+		}
+		console.warn(`⚠️ Failed to clear cache entry: ${response.status} ${response.statusText}`);
+		return false;
+	} catch (error) {
+		console.error(`❌ Error clearing cache entry for ${url}:`, error.message);
+		return false;
+	}
+};
+
+export default clearCacheEntry;

server/scripts/modules/utils/cors.mjs

@@ -1,12 +0,0 @@
-// rewrite some urls for local server
-const rewriteUrl = (_url) => {
-	let url = _url;
-	url = url.replace('https://api.weather.gov/', `${window.location.protocol}//${window.location.host}/`);
-	url = url.replace('https://www.cpc.ncep.noaa.gov/', `${window.location.protocol}//${window.location.host}/`);
-	return url;
-};
-
-export {
-	// eslint-disable-next-line import/prefer-default-export
-	rewriteUrl,
-};

server/scripts/modules/utils/fetch.mjs

@@ -1,31 +1,107 @@
-import { rewriteUrl } from './cors.mjs';
+import { rewriteUrl } from './url-rewrite.mjs';
+
+// Centralized utilities for handling errors in Promise contexts
+const safeJson = async (url, params) => {
+	try {
+		const result = await json(url, params);
+		// Return an object with both data and url if params.returnUrl is true
+		if (params?.returnUrl) {
+			return result;
+		}
+		// If caller didn't specify returnUrl, result is the raw API response
+		return result;
+	} catch (error) {
+		// Error already logged in fetchAsync; return null to be "safe"
+		return null;
+	}
+};
+
+const safeText = async (url, params) => {
+	try {
+		const result = await text(url, params);
+		// Return an object with both data and url if params.returnUrl is true
+		if (params?.returnUrl) {
+			return result;
+		}
+		// If caller didn't specify returnUrl, result is the raw API response
+		return result;
+	} catch (error) {
+		// Error already logged in fetchAsync; return null to be "safe"
+		return null;
+	}
+};
+
+const safeBlob = async (url, params) => {
+	try {
+		const result = await blob(url, params);
+		// Return an object with both data and url if params.returnUrl is true
+		if (params?.returnUrl) {
+			return result;
+		}
+		// If caller didn't specify returnUrl, result is the raw API response
+		return result;
+	} catch (error) {
+		// Error already logged in fetchAsync; return null to be "safe"
+		return null;
+	}
+};
+
+const safePromiseAll = async (promises) => {
+	try {
+		const results = await Promise.allSettled(promises);
+
+		return results.map((result, index) => {
+			if (result.status === 'fulfilled') {
+				return result.value;
+			}
+			// Log rejected promises for debugging (except AbortErrors which are expected)
+			if (result.reason?.name !== 'AbortError') {
+				console.warn(`Promise ${index} rejected:`, result.reason?.message || result.reason);
+			}
+			return null;
+		});
+	} catch (error) {
+		console.error('safePromiseAll encountered an unexpected error:', error);
+		// Return array of nulls matching the input length
+		return new Array(promises.length).fill(null);
+	}
+};
 
 const json = (url, params) => fetchAsync(url, 'json', params);
 const text = (url, params) => fetchAsync(url, 'text', params);
 const blob = (url, params) => fetchAsync(url, 'blob', params);
 
+// Hosts that don't allow custom User-Agent headers due to CORS restrictions
+const USER_AGENT_EXCLUDED_HOSTS = [
+	'geocode.arcgis.com',
+	'services.arcgis.com',
+];
+
 const fetchAsync = async (_url, responseType, _params = {}) => {
-	// add user agent header to json request at api.weather.gov
 	const headers = {};
-	if (_url.toString().match(/api\.weather\.gov/)) {
+
+	const checkUrl = new URL(_url, window.location.origin);
+	const shouldExcludeUserAgent = USER_AGENT_EXCLUDED_HOSTS.some((host) => checkUrl.hostname.includes(host));
+
+	if (!shouldExcludeUserAgent) {
 		headers['user-agent'] = 'Weatherstar 4000+; weatherstar@netbymatt.com';
 	}
+
 	// combine default and provided parameters
 	const params = {
 		method: 'GET',
 		mode: 'cors',
 		type: 'GET',
 		retryCount: 0,
+		timeout: 30000,
 		..._params,
 		headers,
 	};
 	// store original number of retries
 	params.originalRetries = params.retryCount;
 
-	// build a url, including the rewrite for cors if necessary
-	let corsUrl = _url;
-	if (params.cors === true) corsUrl = rewriteUrl(_url);
-	const url = new URL(corsUrl, `${window.location.origin}/`);
+	// rewrite URLs for various services to use the backend proxy server for proper caching (and request logging)
+	const url = rewriteUrl(_url);
 	// match the security protocol when not on localhost
 	// url.protocol = window.location.hostname === 'localhost' ? url.protocol : window.location.protocol;
 	// add parameters if necessary
@@ -39,53 +115,148 @@ const fetchAsync = async (_url, responseType, _params = {}) => {
 	}
 
 	// make the request
-	const response = await doFetch(url, params);
+	try {
+		const response = await doFetch(url, params);
 
-	// check for ok response
-	if (!response.ok) throw new Error(`Fetch error ${response.status} ${response.statusText} while fetching ${response.url}`);
-	// return the requested response
-	switch (responseType) {
-		case 'json':
-			return response.json();
-		case 'text':
-			return response.text();
-		case 'blob':
-			return response.blob();
-		default:
-			return response;
+		// check for ok response
+		if (!response.ok) throw new Error(`Fetch error ${response.status} ${response.statusText} while fetching ${response.url}`);
+		// process the response based on type
+		let result;
+		switch (responseType) {
+			case 'json':
+				result = await response.json();
+				break;
+			case 'text':
+				result = await response.text();
+				break;
+			case 'blob':
+				result = await response.blob();
+				break;
+			default:
+				result = response;
+		}
+
+		// Return both data and URL if requested
+		if (params.returnUrl) {
+			return {
+				data: result,
+				url: response.url,
+			};
+		}
+
+		return result;
+	} catch (error) {
+		// Enhanced error handling for different error types
+		if (error.name === 'AbortError') {
+			// AbortError is always handled gracefully (background tab throttling)
+			console.log(`🛑 Fetch aborted for ${_url} (background tab throttling?)`);
+			return null; // Always return null for AbortError instead of throwing
+		} if (error.message.includes('502')) {
+			console.warn(`🚪 Bad Gateway error for ${_url}`);
+		} else if (error.message.includes('503')) {
+			console.warn(`⌛ Temporarily unavailable for ${_url}`);
+		} else if (error.message.includes('504')) {
+			console.warn(`⏱️  Gateway Timeout for ${_url}`);
+		} else if (error.message.includes('500')) {
+			console.warn(`💥 Internal Server Error for ${_url}`);
+		} else if (error.message.includes('CORS') || error.message.includes('Access-Control')) {
+			console.warn(`🔒 CORS or Access Control error for ${_url}`);
+		} else {
+			console.warn(`❌ Fetch failed for ${_url} (${error.message})`);
+		}
+
+		// Add standard error properties that calling code expects
+		if (!error.status) error.status = 0;
+		if (!error.responseJSON) error.responseJSON = null;
+
+		throw error;
 	}
 };
 
 // fetch with retry and back-off
 const doFetch = (url, params) => new Promise((resolve, reject) => {
-	fetch(url, params).then((response) => {
-		if (params.retryCount > 0) {
-			// 500 status codes should be retried after a short backoff
-			if (response.status >= 500 && response.status <= 599 && params.retryCount > 0) {
-				// call the "still waiting" function
-				if (typeof params.stillWaiting === 'function' && params.retryCount === params.originalRetries) {
-					params.stillWaiting();
-				}
-				// decrement and retry
-				const newParams = {
-					...params,
-					retryCount: params.retryCount - 1,
-				};
-				return resolve(delay(retryDelay(params.originalRetries - newParams.retryCount), doFetch, url, newParams));
-			}
-			// not 500 status
-			return resolve(response);
-		}
-		// out of retries
-		return resolve(response);
-	})
-		.catch(reject);
-});
+	// Create AbortController for timeout
+	const controller = new AbortController();
+	const timeoutId = setTimeout(() => {
+		controller.abort();
+	}, params.timeout);
 
-const delay = (time, func, ...args) => new Promise((resolve) => {
-	setTimeout(() => {
-		resolve(func(...args));
-	}, time);
+	// Add signal to fetch params
+	const fetchParams = {
+		...params,
+		signal: controller.signal,
+	};
+
+	// Shared retry logic to avoid duplication
+	const attemptRetry = (reason) => {
+		// Safety check for params
+		if (!params || typeof params.retryCount !== 'number' || typeof params.originalRetries !== 'number') {
+			console.error(`❌ Invalid params for retry: ${url}`);
+			return reject(new Error('Invalid retry parameters'));
+		}
+
+		const retryAttempt = params.originalRetries - params.retryCount + 1;
+		const remainingRetries = params.retryCount - 1;
+		const delayMs = retryDelay(retryAttempt);
+
+		console.warn(`🔄 Retry ${retryAttempt}/${params.originalRetries} for ${url} - ${reason} (retrying in ${delayMs}ms, ${remainingRetries} retries left)`);
+
+		// call the "still waiting" function on first retry
+		if (params && params.stillWaiting && typeof params.stillWaiting === 'function' && params.retryCount === params.originalRetries) {
+			try {
+				params.stillWaiting();
+			} catch (callbackError) {
+				console.warn(`⚠️ stillWaiting callback error for ${url}:`, callbackError.message);
+			}
+		}
+		// decrement and retry with safe parameter copying
+		const newParams = {
+			...params,
+			retryCount: Math.max(0, params.retryCount - 1), // Ensure retryCount doesn't go negative
+		};
+		// Use setTimeout directly instead of the delay wrapper to avoid Promise resolution issues
+		setTimeout(() => {
+			doFetch(url, newParams).then(resolve).catch(reject);
+		}, delayMs);
+		return undefined; // Explicit return for linter
+	};
+
+	fetch(url, fetchParams).then((response) => {
+		clearTimeout(timeoutId); // Clear timeout on successful response
+
+		// Retry 500 status codes if we have retries left
+		if (params && params.retryCount > 0 && response.status >= 500 && response.status <= 599) {
+			let errorType = 'Server error';
+			if (response.status === 502) {
+				errorType = 'Bad Gateway';
+			} else if (response.status === 503) {
+				errorType = 'Service Unavailable';
+			} else if (response.status === 504) {
+				errorType = 'Gateway Timeout';
+			}
+			return attemptRetry(`${errorType} ${response.status} ${response.statusText}`);
+		}
+
+		// Log when we're out of retries for server errors
+		// if (response.status >= 500 && response.status <= 599) {
+		// 	console.warn(`⚠️ Server error ${response.status} ${response.statusText} for ${url} - no retries remaining`);
+		// }
+
+		// successful response or out of retries
+		return resolve(response);
+	}).catch((error) => {
+		clearTimeout(timeoutId); // Clear timeout on error
+
+		// Retry network errors if we have retries left (but not AbortError)
+		if (params && params.retryCount > 0 && error.name !== 'AbortError') {
+			const reason = error.name === 'TimeoutError' ? 'Request timeout' : `Network error: ${error.message}`;
+			return attemptRetry(reason);
+		}
+
+		// out of retries or AbortError - reject
+		reject(error);
+		return undefined; // Explicit return for linter
+	});
 });
 
 const retryDelay = (retryNumber) => {
@@ -102,4 +273,8 @@ export {
 	json,
 	text,
 	blob,
+	safeJson,
+	safeText,
+	safeBlob,
+	safePromiseAll,
 };

server/scripts/modules/utils/url-rewrite.mjs

@@ -0,0 +1,41 @@
+// rewrite URLs to use local proxy server
+const rewriteUrl = (_url) => {
+	// Handle relative URLs - return them as-is since they don't need rewriting
+	if (typeof _url === 'string' && !_url.startsWith('http')) {
+		return _url;
+	}
+
+	// Handle both string URLs and URL objects
+	const url = typeof _url === 'string' ? new URL(_url) : new URL(_url.toString());
+
+	// Rewrite the origin to use local proxy server
+	if (url.origin === 'https://api.weather.gov') {
+		url.protocol = window.location.protocol;
+		url.host = window.location.host;
+		url.pathname = `/api${url.pathname}`;
+	} else if (url.origin === 'https://www.spc.noaa.gov') {
+		url.protocol = window.location.protocol;
+		url.host = window.location.host;
+		url.pathname = `/spc${url.pathname}`;
+	} else if (url.origin === 'https://radar.weather.gov') {
+		url.protocol = window.location.protocol;
+		url.host = window.location.host;
+		url.pathname = `/radar${url.pathname}`;
+	} else if (url.origin === 'https://mesonet.agron.iastate.edu') {
+		url.protocol = window.location.protocol;
+		url.host = window.location.host;
+		url.pathname = `/mesonet${url.pathname}`;
+	} else if (typeof OVERRIDES !== 'undefined' && OVERRIDES?.RADAR_HOST && url.origin === `https://${OVERRIDES.RADAR_HOST}`) {
+		// Handle override radar host
+		url.protocol = window.location.protocol;
+		url.host = window.location.host;
+		url.pathname = `/mesonet${url.pathname}`;
+	}
+
+	return url;
+};
+
+export {
+	// eslint-disable-next-line import/prefer-default-export
+	rewriteUrl,
+};