From 517cafe40a7c65ac51c2684fd6d8a6f2d4ada53c Mon Sep 17 00:00:00 2001 From: Eddy G Date: Thu, 26 Jun 2025 17:21:23 -0400 Subject: [PATCH] Refactor data loading: move from inline JSON to client-side fetch - Remove large JSON data injection from EJS templates - Add client-side data-loader utility with cache-busting support - Create server endpoints for JSON data with long-term caching - Add graceful failure handling if core data fails to load - Copy JSON data files to dist/data for static hosting - Update app initialization to load data asynchronously - Set serverAvailable flag for static builds in gulp task This reduces HTML payload size and enables better caching strategies for both server and static deployment modes. --- gulp/publish-frontend.mjs | 17 +++---- index.mjs | 20 ++++++-- server/scripts/index.mjs | 19 ++++++- server/scripts/modules/utils/data-loader.mjs | 53 ++++++++++++++++++++ views/index.ejs | 12 ----- 5 files changed, 96 insertions(+), 25 deletions(-) create mode 100644 server/scripts/modules/utils/data-loader.mjs diff --git a/gulp/publish-frontend.mjs b/gulp/publish-frontend.mjs index c7a9164..d493da9 100644 --- a/gulp/publish-frontend.mjs +++ b/gulp/publish-frontend.mjs @@ -133,11 +133,6 @@ const compressHtml = async () => { const packageJson = await readFile('package.json'); const { version } = JSON.parse(packageJson); - // Load the same data that the main server uses - const travelCities = JSON.parse(await readFile('./datagenerators/output/travelcities.json')); - const regionalCities = JSON.parse(await readFile('./datagenerators/output/regionalcities.json')); - const stationInfo = JSON.parse(await readFile('./datagenerators/output/stations.json')); - return src(htmlSources) .pipe(ejs({ production: version, @@ -145,9 +140,6 @@ const compressHtml = async () => { version, OVERRIDES, query: {}, - travelCities, - regionalCities, - stationInfo, })) .pipe(rename({ extname: '.html' })) .pipe(htmlmin({ collapseWhitespace: true })) @@ -162,6 +154,13 @@ const otherFiles = [ const copyOtherFiles = () => src(otherFiles, { base: 'server/', encoding: false }) .pipe(dest('./dist')); +// Copy JSON data files for static hosting +const copyDataFiles = () => src([ + 'datagenerators/output/travelcities.json', + 'datagenerators/output/regionalcities.json', + 'datagenerators/output/stations.json', +]).pipe(dest('./dist/data')); + const s3 = s3Upload({ useIAM: true, }, { @@ -222,7 +221,7 @@ const buildPlaylist = async () => { return file('playlist.json', JSON.stringify(playlist)).pipe(dest('./dist')); }; -const buildDist = series(clean, parallel(buildJs, buildWorkers, compressJsVendor, copyMetarVendor, copyCss, compressHtml, copyOtherFiles, copyImageSources, buildPlaylist)); +const buildDist = series(clean, parallel(buildJs, buildWorkers, compressJsVendor, copyMetarVendor, copyCss, compressHtml, copyOtherFiles, copyDataFiles, copyImageSources, buildPlaylist)); // upload_images could be in parallel with upload, but _images logs a lot and has little changes // by running upload last the majority of the changes will be at the bottom of the log for easy viewing diff --git a/index.mjs b/index.mjs index 6bb782f..ec48d2e 100644 --- a/index.mjs +++ b/index.mjs @@ -58,9 +58,6 @@ const renderIndex = (req, res, production = false) => { version, OVERRIDES, query: req.query, - travelCities, - regionalCities, - stationInfo, }); }; @@ -137,6 +134,23 @@ if (!process.env?.STATIC) { app.get('/playlist.json', playlist); } +// Data endpoints - serve JSON data with long-term caching +const dataEndpoints = { + travelcities: travelCities, + regionalcities: regionalCities, + stations: stationInfo, +}; + +Object.entries(dataEndpoints).forEach(([name, data]) => { + app.get(`/data/${name}.json`, (req, res) => { + res.set({ + 'Cache-Control': 'public, max-age=31536000, immutable', + 'Content-Type': 'application/json', + }); + res.json(data); + }); +}); + if (process.env?.DIST === '1') { // Production ("distribution") mode uses pre-baked files in the dist directory // 'npm run build' and then 'DIST=1 npm start' diff --git a/server/scripts/index.mjs b/server/scripts/index.mjs index 40021eb..4b86dc6 100644 --- a/server/scripts/index.mjs +++ b/server/scripts/index.mjs @@ -7,6 +7,7 @@ import { round2 } from './modules/utils/units.mjs'; import { parseQueryString } from './modules/share.mjs'; import settings from './modules/settings.mjs'; import AutoComplete from './modules/autocomplete.mjs'; +import { loadAllData } from './modules/utils/data-loader.mjs'; document.addEventListener('DOMContentLoaded', () => { init(); @@ -28,7 +29,23 @@ const TXT_ADDRESS_SELECTOR = '#txtAddress'; const TOGGLE_FULL_SCREEN_SELECTOR = '#ToggleFullScreen'; const BNT_GET_GPS_SELECTOR = '#btnGetGps'; -const init = () => { +const init = async () => { + // Load core data first - app cannot function without it + try { + await loadAllData(typeof OVERRIDES !== 'undefined' && OVERRIDES.VERSION ? OVERRIDES.VERSION : ''); + } catch (error) { + console.error('Failed to load core application data:', error); + // Show error message to user and halt initialization + document.body.innerHTML = ` +
+

Unable to load Weather Data

+

The application cannot start because core data failed to load.

+

Please check your connection and try refreshing.

+
+ `; + return; // Stop initialization + } + document.querySelector(TXT_ADDRESS_SELECTOR).addEventListener('focus', (e) => { e.target.select(); }); diff --git a/server/scripts/modules/utils/data-loader.mjs b/server/scripts/modules/utils/data-loader.mjs new file mode 100644 index 0000000..59cf7b7 --- /dev/null +++ b/server/scripts/modules/utils/data-loader.mjs @@ -0,0 +1,53 @@ +// Data loader utility for fetching JSON data with cache-busting + +let dataCache = {}; + +// Load data with version-based cache busting +const loadData = async (dataType, version = '') => { + if (dataCache[dataType]) { + return dataCache[dataType]; + } + + try { + const url = `/data/${dataType}.json${version ? `?_=${version}` : ''}`; + const response = await fetch(url); + + if (!response.ok) { + throw new Error(`Failed to load ${dataType}: ${response.status}`); + } + + const data = await response.json(); + dataCache[dataType] = data; + return data; + } catch (error) { + console.error(`Error loading ${dataType}:`, error); + throw error; + } +}; + +// Load all data types +const loadAllData = async (version = '') => { + const [travelCities, regionalCities, stationInfo] = await Promise.all([ + loadData('travelcities', version), + loadData('regionalcities', version), + loadData('stations', version), + ]); + + // Set global variables for backward compatibility + window.TravelCities = travelCities; + window.RegionalCities = regionalCities; + window.StationInfo = stationInfo; + + return { travelCities, regionalCities, stationInfo }; +}; + +// Clear cache (useful for development) +const clearDataCache = () => { + dataCache = {}; +}; + +export { + loadData, + loadAllData, + clearDataCache, +}; diff --git a/views/index.ejs b/views/index.ejs index 29e0385..43bc0d0 100644 --- a/views/index.ejs +++ b/views/index.ejs @@ -28,12 +28,6 @@ <% if (production) { %> - - @@ -63,12 +57,6 @@ - - <% } %>