6.1.5

.vscode/launch.json

@@ -26,6 +26,18 @@
 			"skipFiles": [
 				"<node_internals>/**"
 			],
+			"args": [
+				"--use-cache"
+			],
+			"type": "node"
+		},
+		{
+			"name": "Data:stations-api",
+			"program": "${workspaceFolder}/datagenerators/stations.mjs",
+			"request": "launch",
+			"skipFiles": [
+				"<node_internals>/**"
+			],
 			"type": "node"
 		},
 		{

datagenerators/stations-postprocessor.mjs

@@ -2,6 +2,8 @@
 
 import { readFileSync, writeFileSync } from 'fs';
 
+import * as url from 'node:url';
+
 // Load station data
 const stationInfo = JSON.parse(readFileSync('./datagenerators/output/stations-raw.json', 'utf8'));
 // const regionalCities = JSON.parse(readFileSync('./datagenerators/output/regionalcities.json', 'utf8'));
@@ -1109,26 +1111,43 @@ or where the fallback to the ICAO airport code occurred:
 	jq -c '.[] | select(.name | test("^[A-Z]{3}$")) | {state, city, simple, name}'
 */
 
-const diffMode = process.argv.includes('--diff');
+const readArguments = () => ({
-const onlyProblems = process.argv.includes('--only-problems');
+	diffMode: process.argv.includes('--diff'),
-const noProblems = process.argv.includes('--no-problems');
+	onlyProblems: process.argv.includes('--only-problems'),
-const onlyDuplicates = process.argv.includes('--only-dupes');
+	noProblems: process.argv.includes('--no-problems'),
-const noPriority = process.argv.includes('--no-priority');
+	onlyDuplicates: process.argv.includes('--only-dupes'),
-const noSimple = process.argv.includes('--no-simple');
+	noPriority: process.argv.includes('--no-priority'),
-const noCoordinates = process.argv.includes('--no-coords');
+	noSimple: process.argv.includes('--no-simple'),
-const writeFile = process.argv.includes('--write');
+	noCoordinates: process.argv.includes('--no-coords'),
+	writeFile: process.argv.includes('--write'),
+});
 
-// Process ALL stations at once to get the display name map
+const DEFAULT_OPTIONS = {
-let displayNameMap = processAllStations(stationInfo);
+	diffMode: false,
+	onlyProblems: false,
+	noProblems: false,
+	onlyDuplicates: false,
+	noPriority: false,
+	noSimple: false,
+	noCoordinates: false,
+	writeFile: false,
+};
 
-// Apply priority-based deduplication
+const postProcessor = (_options) => {
-displayNameMap = resolveDuplicatesByPriority(displayNameMap, stationInfo);
+	// combine default and provided options
+	const options = { ...DEFAULT_OPTIONS, ..._options };
 
-const results = [];
+	// Process ALL stations at once to get the display name map
+	let displayNameMap = processAllStations(stationInfo);
 
-// Now iterate through stations and use the pre-computed display names
+	// Apply priority-based deduplication
-const stations = Object.values(stationInfo);
+	displayNameMap = resolveDuplicatesByPriority(displayNameMap, stationInfo);
-stations.forEach((station) => {
+
+	const results = [];
+
+	// Now iterate through stations and use the pre-computed display names
+	const stations = Object.values(stationInfo);
+	stations.forEach((station) => {
 		const originalName = station.city;
 		const processedName = processingUtils.finalCleanup(displayNameMap[station.id]); // Look up by station ID
 
@@ -1164,12 +1183,12 @@ stations.forEach((station) => {
 			priority,
 			potentialIssues,
 		});
-});
+	});
 
-// Check for duplicates by state
+	// Check for duplicates by state
-const cleanedMapByState = new Map();
+	const cleanedMapByState = new Map();
 
-results.forEach((result) => {
+	results.forEach((result) => {
 		const { state } = result;
 		if (!cleanedMapByState.has(state)) {
 			cleanedMapByState.set(state, new Map());
@@ -1180,9 +1199,9 @@ results.forEach((result) => {
 		} else {
 			stateMap.set(result.city, [result]);
 		}
-});
+	});
 
-cleanedMapByState.forEach((stateMap, _state) => {
+	cleanedMapByState.forEach((stateMap, _state) => {
 		stateMap.forEach((originals, _cleaned) => {
 			if (originals.length > 1) {
 				originals.forEach((original) => {
@@ -1192,22 +1211,22 @@ cleanedMapByState.forEach((stateMap, _state) => {
 				});
 			}
 		});
-});
+	});
 
-// Filter results if requested
+	// Filter results if requested
-let finalResults = results;
+	let finalResults = results;
-if (onlyProblems) {
+	if (options.onlyProblems) {
 		finalResults = results.filter((r) => r.potentialIssues.length > 0);
-}
+	}
-if (onlyDuplicates) {
+	if (options.onlyDuplicates) {
 		finalResults = finalResults.filter((r) => r.potentialIssues.includes('duplicate'));
-}
+	}
 
-const outputResult = finalResults.map((result) => {
+	const outputResult = finalResults.map((result) => {
 		let outputItem = result;
 
 		// Don't include lat or long in diff mode
-	if (noCoordinates || diffMode) {
+		if (options.noCoordinates || options.diffMode) {
 			const {
 				lat: _lat, lon: _lon, ...resultWithoutLocation
 			} = result;
@@ -1215,33 +1234,61 @@ const outputResult = finalResults.map((result) => {
 		}
 
 		// Don't include potentialIssues when --no-problems is specified
-	if (noProblems || diffMode) {
+		if (options.noProblems || options.diffMode) {
 			const { potentialIssues: _potentialIssues, ...resultWithoutIssues } = outputItem;
 			outputItem = resultWithoutIssues;
 		}
 
 		// Remove type and priority if --no-priority is specified
-	if (noPriority || diffMode) {
+		if (options.noPriority || options.diffMode) {
 			const { type: _type, priority: _priority, ...resultWithoutPriority } = outputItem;
 			outputItem = resultWithoutPriority;
 		}
 
 		// remove simple field if --no-simple is specified
-	if (noSimple || diffMode) {
+		if (options.noSimple || options.diffMode) {
 			const { simple: _simple, ...resultWithoutSimple } = outputItem;
 			outputItem = resultWithoutSimple;
 		}
 
 		return outputItem;
-});
+	});
 
-if (writeFile) {
 	const fileResults = results.map(({
-		simple: _simple, type: _type, potentialIssues: _potentialIssues, ...rest
+		simple: _simple, type: _type, potentialIssues: _potentialIssues, location: _location, ...rest
 	}) => rest);
 
+	if (options.writeFile) {
 		writeFileSync('./datagenerators/output/stations.json', compactStringifyToObject(fileResults));
 		console.log(`Wrote ${fileResults.length} processed stations to datagenerators/output/stations.json`);
-} else {
+	} else {
 		console.log(compactStringifyToArray(outputResult));
+	}
+
+	// array to output object
+	const returnObject = {};
+	fileResults.forEach((item) => {
+		returnObject[item.id] = item;
+	});
+
+	return returnObject;
+};
+
+// determine if running from command line or module
+const commandLine = (() => {
+	if (import.meta.url.startsWith('file:')) { // (A)
+		const modulePath = url.fileURLToPath(import.meta.url);
+		if (process.argv[1] === modulePath) { // (B)
+			return true;
+		}
+	}
+	return false;
 }
+)();
+
+// run post processor if called from command line
+if (commandLine) {
+	postProcessor(readArguments());
+}
+
+export default postProcessor;

datagenerators/stations.mjs

@@ -1,3 +1,4 @@
+/* eslint-disable no-loop-func */
 // list all stations in a single file
 // only find stations with 4 letter codes
 
@@ -6,18 +7,25 @@ import https from './https.mjs';
 import states from './stations-states.mjs';
 import chunk from './chunk.mjs';
 import overrides from './stations-overrides.mjs';
+import postProcessor from './stations-postprocessor.mjs';
+
+// check for cached flag
+const USE_CACHE = process.argv.includes('--use-cache');
 
 // skip stations starting with these letters
 const skipStations = ['U', 'C', 'H', 'W', 'Y', 'T', 'S', 'M', 'O', 'L', 'A', 'F', 'B', 'N', 'V', 'R', 'D', 'E', 'I', 'G', 'J'];
 
 // chunk the list of states
-const chunkStates = chunk(states, 1);
+const chunkStates = chunk(states, 3);
 
 // store output
 const output = {};
+let completed = 0;
 
-// process all chunks
+// get data from api if desired
-for (let i = 0; i < chunkStates.length; i += 1) {
+if (!USE_CACHE) {
+	// process all chunks
+	for (let i = 0; i < chunkStates.length; i += 1) {
 		const stateChunk = chunkStates[i];
 		// loop through states
 
@@ -44,16 +52,12 @@ for (let i = 0; i < chunkStates.length; i += 1) {
 							console.log(`Duplicate station: ${state}-${id}`);
 							return;
 						}
-					// get any overrides if available
-					const override = overrides[id] ?? {};
 						output[id] = {
 							id,
 							city: station.properties.name,
 							state,
 							lat: station.geometry.coordinates[1],
 							lon: station.geometry.coordinates[0],
-						// finally add the overrides
-						...override,
 						};
 					});
 					next = stations?.pagination?.next;
@@ -62,11 +66,32 @@ for (let i = 0; i < chunkStates.length; i += 1) {
 					writeFileSync('./datagenerators/output/stations-raw.json', JSON.stringify(output, null, 2));
 				}
 				while (next && stations.features.length > 0);
-			console.log(`Complete: ${state}`);
+				completed += 1;
+				console.log(`Complete: ${state} ${completed}/${states.length}`);
 				return true;
 			} catch {
 				console.error(`Unable to get state: ${state}`);
 				return false;
 			}
 		}));
+	}
 }
+
+// run the post processor
+// data is passed through the file stations-raw.json
+const postProcessed = postProcessor();
+
+// apply any overrides
+Object.entries(overrides).forEach(([id, values]) => {
+	// check for existing value
+	if (postProcessed[id]) {
+		// apply the overrides
+		postProcessed[id] = {
+			...postProcessed[id],
+			...values,
+		};
+	}
+});
+
+// write final file to disk
+writeFileSync('./datagenerators/output/stations.json', JSON.stringify(postProcessed, null, 2));

package-lock.json

@@ -1,12 +1,12 @@
 {
 	"name": "ws4kp",
-	"version": "6.1.4",
+	"version": "6.1.5",
 	"lockfileVersion": 3,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "ws4kp",
-			"version": "6.1.4",
+			"version": "6.1.5",
 			"license": "MIT",
 			"dependencies": {
 				"dotenv": "^17.0.1",

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "ws4kp",
-	"version": "6.1.4",
+	"version": "6.1.5",
 	"description": "Welcome to the WeatherStar 4000+ project page!",
 	"main": "index.mjs",
 	"type": "module",