Store followers amount + Delete 35 days old data to prevent inaccurac…

…y with refunds / steam data changes
L-U-M-E-N · Jan 13, 2024 · 3566ce2 · 3566ce2
1 parent 636493f
commit 3566ce2
Show file tree

Hide file tree

Showing 2 changed files with 90 additions and 33 deletions.
diff --git a/SteamCSV.js b/SteamCSV.js
@@ -91,3 +91,24 @@ export async function QueryWishlistActionsForCSV({ dateStart = DATE_START, dateE
 		cookie
 	);
 }
+
+export async function QueryFollowers({ id }) {
+	const res = await fetch(`https://steamcommunity.com/games/${id}/memberslistxml/?xml=1`);
+	const textBody = await res.text();
+
+	// We can't parse XML in NodeJS without modules ? Okay, let's do it manually !
+
+	// 1. Take groupDetails
+	const relevantPart = textBody.split('<groupDetails>')[1].split('</groupDetails>')[0];
+
+	// 2. Check Invalid format
+	if(!relevantPart.includes('<memberCount>')) {
+		throw new Error('Invalid XML !');
+	}
+
+	// 3. Extract data
+	return parseInt( 
+		relevantPart.split('<memberCount>')[1].split('</memberCount>')[0],
+		10
+	);
+}
diff --git a/main.server.js b/main.server.js
@@ -1,4 +1,4 @@
-import { QueryPackageSalesForCSV, QueryWishlistActionsForCSV } from './SteamCSV.js';
+import { QueryPackageSalesForCSV, QueryWishlistActionsForCSV, QueryFollowers } from './SteamCSV.js';
 
 const COOKIE_FORMAT = (runAs) => config.COOKIE_FORMAT.replace('${runAs}', runAs);
 
@@ -36,15 +36,16 @@ export default class SteamSoldWishlist {
 			await SteamSoldWishlist.UpdateSoldAmount();
 		} catch(e) {
 			log(' ' + e.message, 'error');
-			SteamSoldWishlist.running = false;
-			return;
 		}
 		try {
 			await SteamSoldWishlist.UpdateWishlistAmount();
 		} catch(e) {
 			log(' ' + e.message, 'error');
-			SteamSoldWishlist.running = false;
-			return;
+		}
+		try {
+			await SteamSoldWishlist.UpdateFollowers();
+		} catch(e) {
+			log(' ' + e.message, 'error');
 		}
 		SteamSoldWishlist.running = false;
 	}
@@ -63,39 +64,55 @@ export default class SteamSoldWishlist {
 			)
 		`;
 
-		const dbData = (await Database.execQuery('SELECT * FROM steam_sold')).rows;
+		await Database.execQuery('BEGIN');
 
-		for(const currentPackage of config.soldPackages) {
-			const data = await QueryPackageSalesForCSV({ ...currentPackage, cookie: COOKIE_FORMAT(currentPackage.runAs) });
-
-			for(const line of data) {
-				const currentDate = (new Date(line['Date'] + 'T00:00:00.000Z')).getTime();
+		await Database.execQuery(`
+			DELETE FROM steam_sold
+			WHERE date > (
+				SELECT MAX(date) FROM steam_sold
+			) - interval '35d'
+		`);
 
-				if(dbData.find((data) =>
-					data.bundle_id == line['Bundle(ID#)'] &&
-					data.product_id == line['Product(ID#)'] &&
-					(data.date.getTime() == currentDate) &&
-					data.country_code == line['Country Code'] &&
-					data.type == line["Type"] &&
-					data.gross_units_sold == line["Gross Units Sold"] &&
-					data.net_units_sold == line["Net Units Sold"] &&
-					data.gross_steam_sale_usd == line["Gross Steam Sales (USD)"] &&
-					data.net_steam_sale_usd == line["Net Steam Sales (USD)"]
-				)) {
-					continue;
-				}
-
-				try {
-					await Database.execQuery(
-						query,
-						Object.values(line)
-					);
-				} catch(e) {
-					// Do nothing
+		try {
+			const dbData = (await Database.execQuery('SELECT * FROM steam_sold')).rows;
+
+			for(const currentPackage of config.soldPackages) {
+				const data = await QueryPackageSalesForCSV({ ...currentPackage, cookie: COOKIE_FORMAT(currentPackage.runAs) });
+
+				for(const line of data) {
+					const currentDate = (new Date(line['Date'] + 'T00:00:00.000Z')).getTime();
+
+					if(dbData.find((data) =>
+						data.bundle_id == line['Bundle(ID#)'] &&
+						data.product_id == line['Product(ID#)'] &&
+						(data.date.getTime() == currentDate) &&
+						data.country_code == line['Country Code'] &&
+						data.type == line["Type"] &&
+						data.gross_units_sold == line["Gross Units Sold"] &&
+						data.net_units_sold == line["Net Units Sold"] &&
+						data.gross_steam_sale_usd == line["Gross Steam Sales (USD)"] &&
+						data.net_steam_sale_usd == line["Net Steam Sales (USD)"]
+					)) {
+						continue;
+					}
+
+					try {
+						await Database.execQuery(
+							query,
+							Object.values(line)
+						);
+					} catch(e) {
+						log(' ' + e.message, 'error');
+					}
 				}
 			}
+		} catch(e) {
+			await Database.execQuery('ROLLBACK');
+			throw e;
 		}
 
+		await Database.execQuery('COMMIT');
+
 		log('Saved Steam sold status', 'info');
 	}
 
@@ -126,13 +143,32 @@ export default class SteamSoldWishlist {
 						Object.values(line)
 					);
 				} catch(e) {
-					// Do nothing
+					log(' ' + e.message, 'error');
 				}
 			}
 		}
 
 		log('Saved Steam wishlist status', 'info');
 	}
+
+	static async UpdateFollowers() {
+		const query = 'INSERT INTO public.steam_followers(datelocal, game, amount) VALUES ($1, $2, $3)'
+
+		for(const currentPackage of config.wishlistApps) {
+			const amount = await QueryFollowers({ id: currentPackage.id });
+
+			try {
+				await Database.execQuery(
+					query,
+					[new Date(), currentPackage.name, amount]
+				);
+			} catch(e) {
+				log(' ' + e.message, 'error');
+			}
+		}
+
+		log('Saved Steam followers status', 'info');
+	}
 }
 
 SteamSoldWishlist.init();