Merge upstream/main: integrate radar worker removal and improvements

- Remove radar-worker.mjs and integrate functionality into radar-processor.mjs
- Update package.json and gulpfile.mjs with upstream changes

gulp/publish-frontend.mjs

@@ -26,7 +26,7 @@ const cloudfront = new CloudFrontClient({ region: 'us-east-1' });
 
 const RESOURCES_PATH = './dist/resources';
 
-// Data is now injected server-side, no need to bundle separate data files
+// Data is now served as JSON files to avoid redundancy
 
 const webpackOptions = {
 	mode: 'production',
@@ -96,29 +96,6 @@ const buildJs = () => src(mjsSources)
 	.pipe(webpack(webpackOptions))
 	.pipe(dest(RESOURCES_PATH));
 
-const workerSources = [
-	'./server/scripts/modules/radar-worker.mjs',
-];
-
-const buildWorkers = () => {
-	// update the file name in the webpack options
-	const output = {
-		chunkFilename: '[id].mjs',
-		chunkFormat: 'module',
-		filename: '[name].mjs',
-	};
-	const workerWebpackOptions = {
-		...webpackOptions,
-		output,
-		entry: {
-			'radar-worker': workerSources[0],
-		},
-	};
-	return src(workerSources)
-		.pipe(webpack(workerWebpackOptions))
-		.pipe(dest(RESOURCES_PATH));
-};
-
 const cssSources = [
 	'server/styles/main.css',
 ];
@@ -172,9 +149,10 @@ const uploadSources = [
 	'!dist/images/**/*',
 	'!dist/fonts/**/*',
 ];
-const upload = () => src(uploadSources, { base: './dist', encoding: false })
+
+const uploadCreator = (bucket) => () => src(uploadSources, { base: './dist', encoding: false })
 	.pipe(s3({
-		Bucket: process.env.BUCKET,
+		Bucket: bucket,
 		StorageClass: 'STANDARD',
 		maps: {
 			CacheControl: (keyname) => {
@@ -190,10 +168,14 @@ const imageSources = [
 	'server/images/**',
 	'!server/images/gimp/**',
 ];
-const uploadImages = () => src(imageSources, { base: './server', encoding: false })
+
+const upload = uploadCreator(process.env.BUCKET);
+const uploadPreview = uploadCreator(process.env.BUCKET_PREVIEW);
+
+const uploadImagesCreator = (bucket) => () => src(imageSources, { base: './server', encoding: false })
 	.pipe(
 		s3({
-			Bucket: process.env.BUCKET,
+			Bucket: bucket,
 			StorageClass: 'STANDARD',
 			maps: {
 				CacheControl: () => 'max-age=31536000',
@@ -201,11 +183,14 @@ const uploadImages = () => src(imageSources, { base: './server', encoding: false
 		}),
 	);
 
+const uploadImages = uploadImagesCreator(process.env.BUCKET);
+const uploadImagesPreview = uploadImagesCreator(process.env.BUCKET_PREVIEW);
+
 const copyImageSources = () => src(imageSources, { base: './server', encoding: false })
 	.pipe(dest('./dist'));
 
-const invalidate = () => cloudfront.send(new CreateInvalidationCommand({
-	DistributionId: process.env.DISTRIBUTION_ID,
+const invalidateCreator = (distributionId) => () => cloudfront.send(new CreateInvalidationCommand({
+	DistributionId: distributionId,
 	InvalidationBatch: {
 		CallerReference: (new Date()).toLocaleString(),
 		Paths: {
@@ -215,21 +200,26 @@ const invalidate = () => cloudfront.send(new CreateInvalidationCommand({
 	},
 }));
 
+const invalidate = invalidateCreator(process.env.DISTRIBUTION_ID);
+const invalidatePreview = invalidateCreator(process.env.DISTRIBUTION_ID_PREVIEW);
+
 const buildPlaylist = async () => {
 	const availableFiles = await reader();
 	const playlist = { availableFiles };
 	return file('playlist.json', JSON.stringify(playlist)).pipe(dest('./dist'));
 };
 
-const buildDist = series(clean, parallel(buildJs, buildWorkers, compressJsVendor, copyMetarVendor, copyCss, compressHtml, copyOtherFiles, copyDataFiles, copyImageSources, buildPlaylist));
+const buildDist = series(clean, parallel(buildJs, compressJsVendor, copyMetarVendor, copyCss, compressHtml, copyOtherFiles, copyDataFiles, copyImageSources, buildPlaylist));
 
 // upload_images could be in parallel with upload, but _images logs a lot and has little changes
 // by running upload last the majority of the changes will be at the bottom of the log for easy viewing
 const publishFrontend = series(buildDist, uploadImages, upload, invalidate);
+const stageFrontend = series(buildDist, uploadImagesPreview, uploadPreview, invalidatePreview);
 
 export default publishFrontend;
 
 export {
 	buildDist,
 	invalidate,
+	stageFrontend,
 };

gulpfile.mjs

@@ -1,9 +1,10 @@
 import updateVendor from './gulp/update-vendor.mjs';
-import publishFrontend, { buildDist, invalidate } from './gulp/publish-frontend.mjs';
+import publishFrontend, { buildDist, invalidate, stageFrontend } from './gulp/publish-frontend.mjs';
 
 export {
 	updateVendor,
 	publishFrontend,
 	buildDist,
 	invalidate,
+	stageFrontend,
 };

package-lock.json

@@ -1,12 +1,12 @@
 {
 	"name": "ws4kp",
-	"version": "5.25.3",
+	"version": "5.26.0",
 	"lockfileVersion": 3,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "ws4kp",
-			"version": "5.25.3",
+			"version": "5.26.0",
 			"license": "MIT",
 			"dependencies": {
 				"dotenv": "^16.5.0",

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "ws4kp",
-	"version": "5.25.3",
+	"version": "5.26.0",
 	"description": "Welcome to the WeatherStar 4000+ project page!",
 	"main": "index.mjs",
 	"type": "module",

server/scripts/modules/radar-processor.mjs

@@ -0,0 +1,64 @@
+import { removeDopplerRadarImageNoise } from './radar-utils.mjs';
+import { RADAR_FULL_SIZE, RADAR_FINAL_SIZE } from './radar-constants.mjs';
+
+// process a single radar image and place it on the provided canvas
+const processRadar = async (data) => {
+	const {
+		url, RADAR_HOST, OVERRIDES, radarSourceXY,
+	} = data;
+
+	// get the image
+	const modifiedRadarUrl = OVERRIDES.RADAR_HOST ? url.replace(RADAR_HOST, OVERRIDES.RADAR_HOST) : url;
+	const radarResponsePromise = fetch(modifiedRadarUrl);
+
+	// calculate offsets and sizes
+	const radarSource = {
+		width: 240,
+		height: 163,
+		x: Math.round(radarSourceXY.x / 2),
+		y: Math.round(radarSourceXY.y / 2),
+	};
+
+	// create radar context for manipulation
+	const radarCanvas = document.createElement('canvas');
+	radarCanvas.width = RADAR_FULL_SIZE.width;
+	radarCanvas.height = RADAR_FULL_SIZE.height;
+	const radarContext = radarCanvas.getContext('2d');
+	radarContext.imageSmoothingEnabled = false;
+
+	// test response
+	const radarResponse = await radarResponsePromise;
+	if (!radarResponse.ok) throw new Error(`Unable to fetch radar error ${radarResponse.status} ${radarResponse.statusText} from ${radarResponse.url}`);
+
+	// get the blob
+	const radarImgBlob = await radarResponse.blob();
+
+	// assign to an html image element
+	const radarImgElement = await createImageBitmap(radarImgBlob);
+	// draw the entire image
+	radarContext.clearRect(0, 0, RADAR_FULL_SIZE.width, RADAR_FULL_SIZE.height);
+	radarContext.drawImage(radarImgElement, 0, 0, RADAR_FULL_SIZE.width, RADAR_FULL_SIZE.height);
+
+	// crop the radar image without scaling
+	const croppedRadarCanvas = document.createElement('canvas');
+	croppedRadarCanvas.width = radarSource.width;
+	croppedRadarCanvas.height = radarSource.height;
+	const croppedRadarContext = croppedRadarCanvas.getContext('2d');
+	croppedRadarContext.imageSmoothingEnabled = false;
+	croppedRadarContext.drawImage(radarCanvas, radarSource.x, radarSource.y, croppedRadarCanvas.width, croppedRadarCanvas.height, 0, 0, croppedRadarCanvas.width, croppedRadarCanvas.height);
+
+	// clean the image
+	removeDopplerRadarImageNoise(croppedRadarContext);
+
+	// stretch the radar image
+	const stretchCanvas = document.createElement('canvas');
+	stretchCanvas.width = RADAR_FINAL_SIZE.width;
+	stretchCanvas.height = RADAR_FINAL_SIZE.height;
+	const stretchContext = stretchCanvas.getContext('2d', { willReadFrequently: true });
+	stretchContext.imageSmoothingEnabled = false;
+	stretchContext.drawImage(croppedRadarCanvas, 0, 0, radarSource.width, radarSource.height, 0, 0, RADAR_FINAL_SIZE.width, RADAR_FINAL_SIZE.height);
+
+	return stretchCanvas.toDataURL();
+};
+
+export default processRadar;

server/scripts/modules/radar-tiles.mjs

@@ -39,80 +39,35 @@ const setTiles = (data) => {
 	// the tiles are arranged as follows, with the horizontal axis as x, and correlating with the second set of digits in the image file number
 	// T[0] T[1]
 	// T[2] T[3]
-	// tile 0 gets special treatment, it's placement is the basis for all downstream calculations
-	const t0Source = modTile(sourceXY.x, sourceXY.y);
-	const t0Width = TILE_SIZE.x - t0Source.x;
-	const t0Height = TILE_SIZE.y - t0Source.y;
-	const t0FinalSize = { x: t0Width, y: t0Height };
 
-	// these will all be used again for the overlay, calculate them once here
-	const mapCoordinates = [];
-	// t[0]
-	mapCoordinates.push({
-		sx: t0Source.x,
-		sw: t0Width,
-		dx: 0,
-		dw: t0FinalSize.x,
-
-		sy: t0Source.y,
-		sh: t0Height,
-		dy: 0,
-		dh: t0FinalSize.y,
-	});
-	// t[1]
-	mapCoordinates.push({
-		sx: 0,
-		sw: TILE_SIZE.x - t0Width,
-		dx: t0FinalSize.x,
-		dw: TILE_SIZE.x - t0Width,
-
-		sy: t0Source.y,
-		sh: t0Height,
-		dy: 0,
-		dh: t0FinalSize.y,
-	});
-	// t[2]
-	mapCoordinates.push({
-		sx: t0Source.x,
-		sw: t0Width,
-		dx: 0,
-		dw: t0FinalSize.x,
-
-		sy: 0,
-		sh: TILE_SIZE.y - t0Height,
-		dy: t0FinalSize.y,
-		dh: TILE_SIZE.y - t0Height,
-	});
-	// t[3]
-	mapCoordinates.push({
-		sx: 0,
-		sw: TILE_SIZE.x - t0Width,
-		dx: t0FinalSize.x,
-		dw: TILE_SIZE.x - t0Width,
-
-		sy: 0,
-		sh: TILE_SIZE.y - t0Height,
-		dy: t0FinalSize.y,
-		dh: TILE_SIZE.y - t0Height,
-	});
+	// calculate the shift of tile 0 (upper left)
+	const tileShift = modTile(sourceXY.x, sourceXY.y);
 
 	// determine which tiles are used
 	const usedTiles = [
 		true,
-		mapCoordinates[1].dx < RADAR_FINAL_SIZE.width,
-		mapCoordinates[2].dy < RADAR_FINAL_SIZE.height,
-		mapCoordinates[2].dy < RADAR_FINAL_SIZE.height && mapCoordinates[1].dx < RADAR_FINAL_SIZE.width,
+		TILE_SIZE.x - tileShift.x < RADAR_FINAL_SIZE.width,
+		TILE_SIZE.y - tileShift.y < RADAR_FINAL_SIZE.width,
 	];
+	// if we need t[1] and t[2] then we also need t[3]
+	usedTiles.push(usedTiles[1] && usedTiles[2]);
 
 	// helper function for populating tiles
 	const populateTile = (tileName) => (elem, index) => {
-		// check if the tile is used
-		if (!usedTiles[index]) return;
-
-		// set the image source and size
-		elem.src = `/images/maps/radar/${tileName}-${baseMapTiles[index]}.webp`;
+		// always set the size to flow the images correctly
 		elem.width = TILE_SIZE.x;
 		elem.height = TILE_SIZE.y;
+
+		// check if the tile is used
+		if (!usedTiles[index]) {
+			elem.src = '';
+			return;
+		}
+
+		// set the image source and size
+		const newSource = `/images/maps/radar/${tileName}-${baseMapTiles[index]}.webp`;
+		if (elem.src === newSource) return;
+		elem.src = newSource;
 	};
 
 	// populate the map and overlay tiles
@@ -122,7 +77,6 @@ const setTiles = (data) => {
 
 	// fill the tiles with the overlay
 	// shift the map tile containers
-	const tileShift = modTile(sourceXY.x, sourceXY.y);
 	const mapTileContainer = document.querySelector(`#${elemIdFull} .map-tiles`);
 	mapTileContainer.style.top = `${-tileShift.y}px`;
 	mapTileContainer.style.left = `${-tileShift.x}px`;
@@ -130,12 +84,6 @@ const setTiles = (data) => {
 	const overlayTileContainer = document.querySelector(`#${elemIdFull} .overlay-tiles`);
 	overlayTileContainer.style.top = `${-tileShift.y}px`;
 	overlayTileContainer.style.left = `${-tileShift.x}px`;
-
-	// return some useful data
-	return {
-		usedTiles,
-		baseMapTiles,
-	};
 };
 
 export default setTiles;

server/scripts/modules/radar-worker.mjs

@@ -1,83 +0,0 @@
-import { removeDopplerRadarImageNoise } from './radar-utils.mjs';
-import { RADAR_FULL_SIZE, RADAR_FINAL_SIZE } from './radar-constants.mjs';
-
-onmessage = async (e) => {
-	try {
-		const {
-			url, radarSourceXY, debug,
-		} = e.data;
-
-		if (debug) {
-			console.log('[RADAR-WORKER] Message received at:', new Date().toISOString(), 'File:', url.split('/').pop());
-		}
-
-		// get the image (URL is already rewritten for caching by radar.mjs)
-		const radarResponsePromise = fetch(url);
-
-		// calculate offsets and sizes
-		const radarSource = {
-			width: 240,
-			height: 163,
-			x: Math.round(radarSourceXY.x / 2),
-			y: Math.round(radarSourceXY.y / 2),
-		};
-
-		// create radar context for manipulation
-		const radarCanvas = new OffscreenCanvas(RADAR_FULL_SIZE.width, RADAR_FULL_SIZE.height);
-		const radarContext = radarCanvas.getContext('2d');
-		if (!radarContext) {
-			throw new Error('Failed to get radar canvas context');
-		}
-
-		radarContext.imageSmoothingEnabled = false;
-
-		// test response
-		const radarResponse = await radarResponsePromise;
-		if (!radarResponse.ok) throw new Error(`Unable to fetch radar image: got ${radarResponse.status} ${radarResponse.statusText} from ${radarResponse.url}`);
-
-		// get the blob
-		const radarImgBlob = await radarResponse.blob();
-
-		// assign to an html image element
-		const radarImgElement = await createImageBitmap(radarImgBlob);
-		// draw the entire image
-		radarContext.clearRect(0, 0, RADAR_FULL_SIZE.width, RADAR_FULL_SIZE.height);
-		radarContext.drawImage(radarImgElement, 0, 0, RADAR_FULL_SIZE.width, RADAR_FULL_SIZE.height);
-
-		// crop the radar image without scaling
-		const croppedRadarCanvas = new OffscreenCanvas(radarSource.width, radarSource.height);
-		const croppedRadarContext = croppedRadarCanvas.getContext('2d');
-		if (!croppedRadarContext) {
-			throw new Error('Failed to get cropped radar canvas context');
-		}
-
-		croppedRadarContext.imageSmoothingEnabled = false;
-		croppedRadarContext.drawImage(radarCanvas, radarSource.x, radarSource.y, croppedRadarCanvas.width, croppedRadarCanvas.height, 0, 0, croppedRadarCanvas.width, croppedRadarCanvas.height);
-
-		// clean the image
-		removeDopplerRadarImageNoise(croppedRadarContext);
-
-		// stretch the radar image
-		const stretchCanvas = new OffscreenCanvas(RADAR_FINAL_SIZE.width, RADAR_FINAL_SIZE.height);
-		const stretchContext = stretchCanvas.getContext('2d', { willReadFrequently: true });
-		if (!stretchContext) {
-			throw new Error('Failed to get stretch canvas context');
-		}
-
-		stretchContext.imageSmoothingEnabled = false;
-		stretchContext.drawImage(croppedRadarCanvas, 0, 0, radarSource.width, radarSource.height, 0, 0, RADAR_FINAL_SIZE.width, RADAR_FINAL_SIZE.height);
-
-		const stretchedRadar = stretchCanvas.transferToImageBitmap();
-
-		if (debug) {
-			console.log('[RADAR-WORKER] Sending processed radar at:', new Date().toISOString(), 'Canvas size:', stretchCanvas.width, 'x', stretchCanvas.height, 'File:', url.split('/').pop());
-		}
-
-		postMessage(stretchedRadar, [stretchedRadar]);
-	} catch (error) {
-		console.warn('[RADAR-WORKER] Error at:', new Date().toISOString(), 'Error:', error.message);
-		// Handle radar fetch errors by indicating failure to the main thread
-		// This allows the radar display to set totalScreens = 0 and skip in animation
-		postMessage({ error: true, message: error.message });
-	}
-};

server/scripts/modules/radar.mjs

@@ -5,33 +5,14 @@ import { safeText } from './utils/fetch.mjs';
 import WeatherDisplay from './weatherdisplay.mjs';
 import { registerDisplay, timeZone } from './navigation.mjs';
 import * as utils from './radar-utils.mjs';
-import { rewriteUrl } from './utils/url-rewrite.mjs';
-import { debugFlag } from './utils/debug.mjs';
-import { version } from './progress.mjs';
 import setTiles from './radar-tiles.mjs';
-
-// TEMPORARY fix to disable radar on ios safari. The same engine (webkit) is
-// used for all ios browers (chrome, brave, firefox, etc) so it's safe to skip
-// any subsequent narrowing of the user-agent.
-const isIos = /iP(ad|od|hone)/i.test(window.navigator.userAgent);
-// NOTE: iMessages/Messages preview is provided by an Apple scraper that uses a
-// user-agent similar to: `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1)
-// AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4
-// facebookexternalhit/1.1 Facebot Twitterbot/1.0`. There is currently a bug in
-// Messages macos/ios where a constantly crashing website seems to cause an
-// entire Messages thread to permanently lockup until the individual website
-// preview is deleted! Messages ios will judder but allows the message to be
-// deleted eventually. Messages macos beachballs forever and prevents the
-// successful deletion. See
-// https://github.com/netbymatt/ws4kp/issues/74#issuecomment-2921154962 for more
-// context.
-const isBot = /twitterbot|Facebot/i.test(window.navigator.userAgent);
+import processRadar from './radar-processor.mjs';
 
 // Use OVERRIDE_RADAR_HOST if provided, otherwise default to mesonet
 const RADAR_HOST = (typeof OVERRIDES !== 'undefined' ? OVERRIDES?.RADAR_HOST : undefined) || 'mesonet.agron.iastate.edu';
 class Radar extends WeatherDisplay {
 	constructor(navId, elemId) {
-		super(navId, elemId, 'Local Radar', !isIos && !isBot);
+		super(navId, elemId, 'Local Radar');
 
 		this.okToDrawCurrentConditions = false;
 		this.okToDrawCurrentDateTime = false;
@@ -72,12 +53,6 @@ class Radar extends WeatherDisplay {
 			return;
 		}
 
-		// get the workers started
-		if (!this.workers) {
-			// get some web workers started
-			this.workers = (new Array(this.dopplerRadarImageMax)).fill(null).map(() => radarWorker());
-		}
-
 		const baseUrl = `https://${RADAR_HOST}/archive/data/`;
 		const baseUrlEnd = '/GIS/uscomp/?F=0&P=n0r*.png'; // This URL returns an index of .png files for the given date
 
@@ -163,11 +138,12 @@ class Radar extends WeatherDisplay {
 
 		// Load the most recent doppler radar images.
 		try {
-			const radarInfo = await Promise.all(urls.map(async (url, index) => {
-				const processedRadar = await this.workers[index].processRadar({
-					url: rewriteUrl(url).toString(), // Apply URL rewriting for caching; convert to string for worker
+			const radarInfo = await Promise.all(urls.map(async (url) => {
+				const processedRadar = await processRadar({
+					url,
+					RADAR_HOST,
+					OVERRIDES,
 					radarSourceXY,
-					debug: debugFlag('radar'),
 				});
 
 				// store the time
@@ -184,15 +160,7 @@ class Radar extends WeatherDisplay {
 					zone: 'UTC',
 				}).setZone(timeZone());
 
-				const onscreenCanvas = document.createElement('canvas');
-				onscreenCanvas.width = processedRadar.width;
-				onscreenCanvas.height = processedRadar.height;
-				const onscreenContext = onscreenCanvas.getContext('bitmaprenderer');
-				onscreenContext.transferFromImageBitmap(processedRadar);
-
-				const dataUrl = onscreenCanvas.toDataURL();
-
-				const elem = this.fillTemplate('frame', { map: { type: 'img', src: dataUrl } });
+				const elem = this.fillTemplate('frame', { map: { type: 'img', src: processedRadar } });
 				return {
 					time,
 					elem,
@@ -233,50 +201,5 @@ class Radar extends WeatherDisplay {
 	}
 }
 
-// create a radar worker with helper functions
-const radarWorker = () => {
-	// create the worker
-	const worker = new Worker(`/resources/radar-worker.mjs?_=${version()}`, { type: 'module' });
-
-	const processRadar = (data) => new Promise((resolve, reject) => {
-		if (debugFlag('radar')) {
-			console.log('[RADAR-MAIN] Posting to worker at:', new Date().toISOString(), 'File:', data.url.split('/').pop());
-		}
-		// prepare for done message
-		worker.onmessage = (e) => {
-			if (debugFlag('radar')) {
-				console.log('[RADAR-MAIN] Received from worker at:', new Date().toISOString(), 'Data type:', e?.data?.constructor?.name);
-			}
-			if (e?.data?.error) {
-				console.warn('[RADAR-MAIN] Worker error:', e.data.message);
-				// Worker encountered an error
-				reject(new Error(e.data.message));
-			} else if (e?.data instanceof Error) {
-				console.warn('[RADAR-MAIN] Worker exception:', e.data);
-				reject(e.data);
-			} else if (e?.data instanceof ImageBitmap) {
-				if (debugFlag('radar')) {
-					console.log('[RADAR-MAIN] Successfully received ImageBitmap, size:', e.data.width, 'x', e.data.height);
-				}
-				resolve(e.data);
-			}
-		};
-
-		// start up the worker
-		worker.postMessage({
-			...data,
-			debug: debugFlag('radar'),
-		});
-	});
-
-	// return the object
-	return {
-		processRadar,
-	};
-};
-
 // register display
-// TEMPORARY: except on IOS and bots
-if (!isIos && !isBot) {
-	registerDisplay(new Radar(11, 'radar'));
-}
+registerDisplay(new Radar(11, 'radar'));