Refactor data loading: move from inline JSON to client-side fetch

- Remove large JSON data injection from EJS templates
- Add client-side data-loader utility with cache-busting support
- Create server endpoints for JSON data with long-term caching
- Add graceful failure handling if core data fails to load
- Copy JSON data files to dist/data for static hosting
- Update app initialization to load data asynchronously
- Set serverAvailable flag for static builds in gulp task

This reduces HTML payload size and enables better caching strategies
for both server and static deployment modes.

gulp/publish-frontend.mjs

@@ -133,11 +133,6 @@ const compressHtml = async () => {
 	const packageJson = await readFile('package.json');
 	const { version } = JSON.parse(packageJson);
 
-	// Load the same data that the main server uses
-	const travelCities = JSON.parse(await readFile('./datagenerators/output/travelcities.json'));
-	const regionalCities = JSON.parse(await readFile('./datagenerators/output/regionalcities.json'));
-	const stationInfo = JSON.parse(await readFile('./datagenerators/output/stations.json'));
-
 	return src(htmlSources)
 		.pipe(ejs({
 			production: version,
@@ -145,9 +140,6 @@ const compressHtml = async () => {
 			version,
 			OVERRIDES,
 			query: {},
-			travelCities,
-			regionalCities,
-			stationInfo,
 		}))
 		.pipe(rename({ extname: '.html' }))
 		.pipe(htmlmin({ collapseWhitespace: true }))
@@ -162,6 +154,13 @@ const otherFiles = [
 const copyOtherFiles = () => src(otherFiles, { base: 'server/', encoding: false })
 	.pipe(dest('./dist'));
 
+// Copy JSON data files for static hosting
+const copyDataFiles = () => src([
+	'datagenerators/output/travelcities.json',
+	'datagenerators/output/regionalcities.json',
+	'datagenerators/output/stations.json',
+]).pipe(dest('./dist/data'));
+
 const s3 = s3Upload({
 	useIAM: true,
 }, {
@@ -222,7 +221,7 @@ const buildPlaylist = async () => {
 	return file('playlist.json', JSON.stringify(playlist)).pipe(dest('./dist'));
 };
 
-const buildDist = series(clean, parallel(buildJs, buildWorkers, compressJsVendor, copyMetarVendor, copyCss, compressHtml, copyOtherFiles, copyImageSources, buildPlaylist));
+const buildDist = series(clean, parallel(buildJs, buildWorkers, compressJsVendor, copyMetarVendor, copyCss, compressHtml, copyOtherFiles, copyDataFiles, copyImageSources, buildPlaylist));
 
 // upload_images could be in parallel with upload, but _images logs a lot and has little changes
 // by running upload last the majority of the changes will be at the bottom of the log for easy viewing

index.mjs

@@ -58,9 +58,6 @@ const renderIndex = (req, res, production = false) => {
 		version,
 		OVERRIDES,
 		query: req.query,
-		travelCities,
-		regionalCities,
-		stationInfo,
 	});
 };
 
@@ -137,6 +134,23 @@ if (!process.env?.STATIC) {
 	app.get('/playlist.json', playlist);
 }
 
+// Data endpoints - serve JSON data with long-term caching
+const dataEndpoints = {
+	travelcities: travelCities,
+	regionalcities: regionalCities,
+	stations: stationInfo,
+};
+
+Object.entries(dataEndpoints).forEach(([name, data]) => {
+	app.get(`/data/${name}.json`, (req, res) => {
+		res.set({
+			'Cache-Control': 'public, max-age=31536000, immutable',
+			'Content-Type': 'application/json',
+		});
+		res.json(data);
+	});
+});
+
 if (process.env?.DIST === '1') {
 	// Production ("distribution") mode uses pre-baked files in the dist directory
 	// 'npm run build' and then 'DIST=1 npm start'

server/scripts/index.mjs

@@ -7,6 +7,7 @@ import { round2 } from './modules/utils/units.mjs';
 import { parseQueryString } from './modules/share.mjs';
 import settings from './modules/settings.mjs';
 import AutoComplete from './modules/autocomplete.mjs';
+import { loadAllData } from './modules/utils/data-loader.mjs';
 
 document.addEventListener('DOMContentLoaded', () => {
 	init();
@@ -28,7 +29,23 @@ const TXT_ADDRESS_SELECTOR = '#txtAddress';
 const TOGGLE_FULL_SCREEN_SELECTOR = '#ToggleFullScreen';
 const BNT_GET_GPS_SELECTOR = '#btnGetGps';
 
-const init = () => {
+const init = async () => {
+	// Load core data first - app cannot function without it
+	try {
+		await loadAllData(typeof OVERRIDES !== 'undefined' && OVERRIDES.VERSION ? OVERRIDES.VERSION : '');
+	} catch (error) {
+		console.error('Failed to load core application data:', error);
+		// Show error message to user and halt initialization
+		document.body.innerHTML = `
+			<div>
+				<h2>Unable to load Weather Data</h2>
+				<p>The application cannot start because core data failed to load.</p>
+				<p>Please check your connection and try refreshing.</p>
+			</div>
+		`;
+		return; // Stop initialization
+	}
+
 	document.querySelector(TXT_ADDRESS_SELECTOR).addEventListener('focus', (e) => {
 		e.target.select();
 	});

server/scripts/modules/utils/data-loader.mjs

@@ -0,0 +1,53 @@
+// Data loader utility for fetching JSON data with cache-busting
+
+let dataCache = {};
+
+// Load data with version-based cache busting
+const loadData = async (dataType, version = '') => {
+	if (dataCache[dataType]) {
+		return dataCache[dataType];
+	}
+
+	try {
+		const url = `/data/${dataType}.json${version ? `?_=${version}` : ''}`;
+		const response = await fetch(url);
+
+		if (!response.ok) {
+			throw new Error(`Failed to load ${dataType}: ${response.status}`);
+		}
+
+		const data = await response.json();
+		dataCache[dataType] = data;
+		return data;
+	} catch (error) {
+		console.error(`Error loading ${dataType}:`, error);
+		throw error;
+	}
+};
+
+// Load all data types
+const loadAllData = async (version = '') => {
+	const [travelCities, regionalCities, stationInfo] = await Promise.all([
+		loadData('travelcities', version),
+		loadData('regionalcities', version),
+		loadData('stations', version),
+	]);
+
+	// Set global variables for backward compatibility
+	window.TravelCities = travelCities;
+	window.RegionalCities = regionalCities;
+	window.StationInfo = stationInfo;
+
+	return { travelCities, regionalCities, stationInfo };
+};
+
+// Clear cache (useful for development)
+const clearDataCache = () => {
+	dataCache = {};
+};
+
+export {
+	loadData,
+	loadAllData,
+	clearDataCache,
+};

views/index.ejs

@@ -28,12 +28,6 @@
 
   <% if (production) { %>
   <link rel="stylesheet" type="text/css" href="resources/ws.min.css?_=<%=production%>" />
-  <!-- data must be loaded before main script -->
-  <script>
-    window.TravelCities = <%- JSON.stringify(travelCities) %>;
-    window.RegionalCities = <%- JSON.stringify(regionalCities) %>;
-    window.StationInfo = <%- JSON.stringify(stationInfo) %>;
-  </script>
   <script type="text/javascript" src="resources/vendor.min.js?_=<%=production%>"></script>
   <script type="text/javascript" src="resources/ws.min.js?_=<%=production%>"></script>
 	<script type="text/javascript">const OVERRIDES=<%-JSON.stringify(OVERRIDES)%>;</script>
@@ -63,12 +57,6 @@
   <script type="module" src="scripts/modules/settings.mjs"></script>
 	<script type="module" src="scripts/modules/media.mjs"></script>
 	<script type="module" src="scripts/index.mjs"></script>
-  <!-- data -->
-  <script>
-    window.TravelCities = <%- JSON.stringify(travelCities) %>;
-    window.RegionalCities = <%- JSON.stringify(regionalCities) %>;
-    window.StationInfo = <%- JSON.stringify(stationInfo) %>;
-  </script>
   <% } %>
 
 </head>