Support for large feeds and other improvements

index.html

@@ -183,6 +183,10 @@
 			text-align: right;
 		}
 
+		.raw {
+			margin: 10px;
+		}
+
 		.error {
 			flex-grow: 1;
 			margin: 1em;

lib/render.js

@@ -1,7 +1,32 @@
 import {h} from 'preact'
 import ms from 'ms'
+import { useState, useRef } from 'preact/hooks';
+
+export const MAX_ITEMS = 1000;
 
 export const renderDelay = (delay) => {
 	if ('number' !== typeof delay) return '?'
 	return delay === 0 ? '0s' : ms(delay * 1000)
 }
+
+export const maxItemsMsg = (itemCount) => {
+	if (itemCount > MAX_ITEMS) {
+		return (
+			<p>
+			Only the first {MAX_ITEMS} items are rendered. Use the filter above.
+			</p>
+		)
+	}
+}
+
+export const filterInput = (placeholder) => {
+	let inputValue = useRef('')
+	let [query, setQuery] = useState('')
+	return [
+		list => list.filter(e => !query || JSON.stringify(e).toLowerCase().includes(query)).slice(0, MAX_ITEMS),
+		<span>
+			<input onChange={e => inputValue.current = e.target.value} placeholder={placeholder || 'filter...'} />
+			<input type="button" value="filter" onClick={_ => setQuery(inputValue.current.toLowerCase())}/>
+		</span>
+	]
+}

stores/feed-log.js

@@ -16,17 +16,18 @@ const feedLogStore = (state, bus) => {
 		.flatMap(tu => (tu.trip_update.stop_time_update || []).map(stu => ({
 			entityId: tu.id,
 			timestamp: tu.trip_update.timestamp || header.timestamp || null,
-			routeId: tu.trip_update.trip.route_id,
-			directionId: tu.trip_update.trip.direction_id,
-			tripId: tu.trip_update.trip.trip_id,
-			vehicleId: tu.trip_update.vehicle.id,
-			vehicleLabel: tu.trip_update.vehicle.label,
+			routeId: tu.trip_update.trip?.route_id,
+			directionId: tu.trip_update.trip?.direction_id,
+			tripId: tu.trip_update.trip?.trip_id,
+			vehicleId: tu.trip_update.vehicle?.id,
+			vehicleLabel: tu.trip_update.vehicle?.label,
 			stopId: stu.stop_id,
-			tArrival: stu.arrival.time,
-			delay: stu.arrival.delay,
+			scheduleRelationship: stu.schedule_relationship,
+			tArrival: stu.arrival?.time,
+			delay: stu.arrival?.delay || stu.departure?.delay,
 		})))
 		// keep all with a future/current arrival
-		.filter(({tArrival}) => tArrival && tArrival >= now)
+		.filter(({tArrival}) => !tArrival || tArrival >= now)
 		// sooner arrival first, delayed first
 		.sort((a, b) => {
 			const timeUntilA = a.tArrival - now

stores/feed.js

@@ -4,7 +4,7 @@ import {Buffer} from 'buffer/' // trailing slash is intentional
 import {FeedMessage} from 'gtfs-rt-bindings'
 import syncViaPeriodicFetch from 'fetch-periodic-sync'
 
-const MAX_FEED_SIZE = 5 * 1024 * 1024 // 5mb
+const MAX_FEED_SIZE = 128 * 1024 * 1024 // 5mb
 
 const CONTENT_TYPES = [
 	'application/octet-stream', // generic "binary" blob
@@ -40,7 +40,7 @@ const feedStore = (state, bus) => {
 		}
 
 		const buf = Buffer.from(await res.arrayBuffer())
-		const data = FeedMessage.decode(buf)
+		const data = FeedMessage.toObject(FeedMessage.decode(buf))
 		if (!data || !data.header || !Array.isArray(data.entity)) {
 			const err = new Error(`couldn't parse feed`)
 			err.rawFeed = buf

views/bar.js

@@ -20,6 +20,7 @@ const barView = ({state, emit}) => {
 			{renderNavItem('log', state.view === 'log')}
 			{renderNavItem('inspector', state.view === 'inspector')}
 			{renderNavItem('map', state.view === 'map')}
+			{renderNavItem('raw', state.view === 'raw')}
 			<li><a href={pkg.homepage} target="_parent" title={name + ' homepage'}>?</a></li>
 		</ul>
 	)

views/inspector.js

@@ -1,7 +1,7 @@
 import {h} from 'preact'
-import {TripUpdate, VehiclePosition} from 'gtfs-rt-bindings'
+import {TripDescriptor, VehiclePosition} from 'gtfs-rt-bindings'
 import ms from 'ms'
-import {renderDelay} from '../lib/render'
+import {renderDelay, maxItemsMsg, filterInput} from '../lib/render'
 
 const renderStartDate = (sD) => {
 	if ('string' !== typeof sD || !sD) return '?'
@@ -14,13 +14,19 @@ const renderStartTime = (sT) => {
 
 const {
 	SCHEDULED,
-	SKIPPED,
-	NO_DATA,
-} = TripUpdate.StopTimeUpdate.ScheduleRelationship
-const renderScheduleRelationship = (sR) => {
+	ADDED,
+	UNSCHEDULED,
+	CANCELED,
+	DUPLICATED,
+	DELETED
+} = TripDescriptor.ScheduleRelationship
+const renderTripScheduleRelationship = (sR) => {
 	if (sR === SCHEDULED) return <code><abbr title="SCHEDULED">SCHED</abbr></code>
-	if (sR === SKIPPED) return <code><abbr title="SKIPPED">SKIP</abbr></code>
-	if (sR === NO_DATA) return <code><abbr title="NO_DATA">NO_DATA</abbr></code>
+	if (sR === ADDED) return <code><abbr title="ADDED">ADD</abbr></code>
+	if (sR === UNSCHEDULED) return <code><abbr title="UNSCHEDULED">UNSCHED</abbr></code>
+	if (sR === CANCELED) return <code><abbr title="CANCELED">CNCL</abbr></code>
+	if (sR === DUPLICATED) return <code><abbr title="DUPLICATED">DUPL</abbr></code>
+	if (sR === DELETED) return <code><abbr title="DELETED">DEL</abbr></code>
 	return '?'
 }
 
@@ -115,7 +121,7 @@ const renderTripUpdate = (entity, emit) => {
 			<td><code>{t.trip_id}</code></td>
 			<td>{renderStartDate(t.start_date)}</td>
 			<td>{renderStartTime(t.start_time)}</td>
-			<td>{renderScheduleRelationship(t.schedule_relationship)}</td>
+			<td>{renderTripScheduleRelationship(t.schedule_relationship)}</td>
 			<td><code>{v.id}</code></td>
 			<td><code>{v.label}</code></td>
 			<td><code>{v.license_plate}</code></td>
@@ -126,10 +132,12 @@ const renderTripUpdate = (entity, emit) => {
 const renderTripUpdates = (feed, emit) => {
 	const tripUpdates = feed.entity
 	.filter(entity => !!entity.trip_update)
-
+	const [filter, inputBox] = filterInput('search in feed JSON...')
+
 	return (
 		<div>
-			<h2><code>TripUpdate</code>s ({tripUpdates.length})</h2>
+			<h2><code>TripUpdate</code>s ({tripUpdates.length}) {inputBox}</h2>
+			{maxItemsMsg(tripUpdates.length)}
 			<table class="trip-updates">
 				<thead>
 					<tr>
@@ -151,7 +159,7 @@ const renderTripUpdates = (feed, emit) => {
 					</tr>
 				</thead>
 				<tbody>
-					{tripUpdates.map(renderTripUpdate)}
+					{filter(tripUpdates).map(renderTripUpdate)}
 				</tbody>
 			</table>
 		</div>
@@ -186,10 +194,12 @@ const renderVehiclePosition = (entity, emit) => {
 const renderVehiclePositions = (feed, emit) => {
 	const vehiclePositions = feed.entity
 	.filter(entity => !!entity.vehicle)
+	const [filter, inputBox] = filterInput()
 
 	return (
 		<div>
-			<h2><code>VehiclePosition</code>s ({vehiclePositions.length})</h2>
+			<h2><code>VehiclePosition</code>s ({vehiclePositions.length}) {inputBox}</h2>
+			{maxItemsMsg(vehiclePositions.length)}
 			<table class="vehicle-positions">
 				<thead>
 					<tr>
@@ -215,7 +225,7 @@ const renderVehiclePositions = (feed, emit) => {
 					</tr>
 				</thead>
 				<tbody>
-					{vehiclePositions.map(vehPos => renderVehiclePosition(vehPos, emit))}
+					{filter(vehiclePositions).map(vehPos => renderVehiclePosition(vehPos, emit))}
 				</tbody>
 			</table>
 		</div>

views/log.js

@@ -1,18 +1,35 @@
 import {h} from 'preact'
-import {renderDelay} from '../lib/render'
+import {renderDelay, maxItemsMsg, filterInput} from '../lib/render'
+import {TripUpdate} from 'gtfs-rt-bindings'
 
 const renderTimestamp = (t) => {
 	// todo: correct tz
 	// todo: abbreviate?
+	if (!t) {
+		return '?';
+	}
 	return new Date(t * 1000).toISOString()
 }
 
+const {
+	SCHEDULED,
+	SKIPPED,
+	NO_DATA,
+} = TripUpdate.StopTimeUpdate.ScheduleRelationship
+const renderStopTimeUpdateScheduleRelationship = (sR) => {
+	if (sR === SCHEDULED) return <code><abbr title="SCHEDULED">SCHED</abbr></code>
+	if (sR === SKIPPED) return <code><abbr title="SKIPPED">SKIP</abbr></code>
+	if (sR === NO_DATA) return <code><abbr title="NO_DATA">NO_DATA</abbr></code>
+	return '?'
+}
+
 const renderFeedLogEntry = (entry) => {
 	const {
 		entityId, timestamp,
 		routeId, directionId, tripId,
 		vehicleId, vehicleLabel,
 		stopId,
+		scheduleRelationship,
 		tArrival, delay,
 	} = entry
 	return (
@@ -25,27 +42,28 @@ const renderFeedLogEntry = (entry) => {
 			<td><code>{vehicleId}</code></td>
 			<td><code>{vehicleLabel}</code></td>
 			<td><code>{stopId}</code></td>
+			<td>{renderStopTimeUpdateScheduleRelationship(scheduleRelationship)}</td>
 			<td>{renderTimestamp(tArrival)}</td>
 			<td>{renderDelay(delay)}</td>
 		</tr>
 	)
 }
 
 const logView = ({state, emit}) => {
-	const totalEntries = state.feedLog.length
-	const feedLog = state.feedLog.slice(0, 1000)
-
+	const feedLog = state.feedLog
+	const [filter, inputBox] = filterInput()
 	return (
 		<div class="log">
-			<p>Showing {feedLog.length}{totalEntries > feedLog.length ? ' out of ' + totalEntries : null} <code>StopTimeUpdate</code>s.</p>
+			<h2>Upcoming <code>StopTimeUpdate</code>s ({feedLog.length}) {inputBox}</h2>
+			{maxItemsMsg(feedLog.length)}
 			<table>
 				<thead>
 					<tr>
 						<th rowspan="2"><code><abbr title="entity.id">e_id</abbr></code></th>
 						<th rowspan="2"><abbr title="stop_time_update[].timestamp or feed timestamp"><code>t</code></abbr></th>
 						<th colspan="2"><code>trip</code></th>
 						<th colspan="3"><code>vehicle</code></th>
-						<th colspan="3"><code>stop_time_update</code></th>
+						<th colspan="4"><code>stop_time_update</code></th>
 					</tr>
 					<tr>
 						<th><code>route_id</code></th>
@@ -54,12 +72,13 @@ const logView = ({state, emit}) => {
 						<th><code>id</code></th>
 						<th><code>label</code></th>
 						<th><code>stop_id</code></th>
+						<th><abbr title="stop_time_update[].schedule_relationship">s_rel</abbr></th>
 						<th><code><abbr title="stop_time_update[].arrival.time">arr</abbr></code></th>
 						<th><code><abbr title="stop_time_update[].arrival.delay">delay</abbr></code></th>
 					</tr>
 				</thead>
 				<tbody>
-					{feedLog.map(renderFeedLogEntry)}
+					{filter(feedLog).map(renderFeedLogEntry)}
 				</tbody>
 			</table>
 		</div>

views/main.js

@@ -2,6 +2,7 @@ import {h} from 'preact'
 import Bar from './bar'
 import Log from './log'
 import Inspector from './inspector'
+import Raw from './raw'
 import _Map from './map'
 import Error from './error'
 
@@ -10,6 +11,7 @@ const mainView = ({state, emit}) => {
 		'log': Log,
 		'inspector': Inspector,
 		'map': _Map,
+		'raw': Raw
 	})[state.view] || Inspector
 	return (
 		<div class="app">

views/raw.js

@@ -0,0 +1,18 @@
+import {h} from 'preact'
+
+const MAX_LEN = 1024*1024;
+
+const rawView = ({state, emit}) => {
+	const feed = state.feedData
+	const raw = JSON.stringify(feed, null, 2);
+	return (
+		<div class="raw">
+			<p>{raw.length > MAX_LEN ? 'Truncated to '+MAX_LEN+' characters.' : ''}</p>
+			<pre>
+				{raw.substring(0, MAX_LEN)}
+			</pre>
+		</div>
+	)
+}
+
+export default rawView