diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..09bd002 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,23 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +Steps/data to reproduce the behavior, e.g. +- url to the zip file +- route ID +- stop ID +- outward/return + +**Release used** +Which gtfs2 release and HA type (HAOS/Container) + +**Additional** +Please add logs if helpfull diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..f26a893 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,17 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: "[FEATURE]: " +labels: '' +assignees: '' + +--- + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/README.md b/README.md index db99fc2..d3b334e 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Core GTFS uses start + stop, it then determines every option between them and pr ***Solution/workaround in GTFS2***: attribute added: next_departure_line shows all next departues with their line/means-of-transport. So even if you select a route first and then two stops, the attibutes will still show alternatives between those 2 stops, if applicable. ## Updates + 20231126 - realtime vehile tracking with geojson output - workflow tweaks @@ -32,6 +33,7 @@ Core GTFS uses start + stop, it then determines every option between them and pr 20231104: initial version + ## ToDo's / In Development / Known Issues - Issue when updating the source db: pygtfs error: at the moment unclear as errors fluctuate, posisbly a lack of resources (mem/cpu) - get realtime data for sources that donot base on routes, e.g. France's TER realtime source only uses trip_id @@ -68,13 +70,25 @@ Data can be updated at your own discretion by a service, e.g. you can have a wee ![image](https://github.com/vingerha/gtfs2/assets/44190435/2d639afa-376b-4956-8223-2c982dc537cb) - or via yaml ![image](https://github.com/vingerha/gtfs2/assets/44190435/0d50bb87-c081-4cd6-8dc5-9603a44c21a4) +======= +## Known issues/challenges with source data +Static gtfs: +- not complying to the pygtfs unpacking library, examples: missing dates in feed_info > manual fix +- calendar not showing if a service is run on a specific day > fix via adding calendar_dates to filter, only works if (!) calendar_dates is used alternatively for the same purpose +- missing routes/stops/times, transport runs but gtfs does nto show it > report issue with your gtfs data provider +- routes show A > B (outward) but stop selection shows inversed B > A, within one gtfs source both good as incorrect start/end can show up > report issue with your gtfs data provider + +Realtime gtfs +- few realtiem providers also add vehicle positions with lat/lon, these are not always up to date > report issue with your gtfs data provider +- format incorrect of incomming json/feed > report issue with your gtfs data provider, they should adhere to standards +- realtime data not always available, few refreshes are fine then nothing then fine again, often related to timeout from provider > report issue with your gtfs data provider ## Thank you - @joostlek ... massive thanks to help me through many (!) tech aspects and getting this to the inital version - @mxbssn for initiating, bringing ideas, helping with testing - @mark1foley for his gtfs real time integration which was enhanced with its integration in GTFS2 + diff --git a/custom_components/gtfs2/__init__.py b/custom_components/gtfs2/__init__.py index 7aee183..de1aeeb 100644 --- a/custom_components/gtfs2/__init__.py +++ b/custom_components/gtfs2/__init__.py @@ -8,7 +8,9 @@ from datetime import timedelta from .const import DOMAIN, PLATFORMS, DEFAULT_PATH, DEFAULT_REFRESH_INTERVAL -from .coordinator import GTFSUpdateCoordinator + +from .coordinator import GTFSUpdateCoordinator, GTFSRealtimeUpdateCoordinator + import voluptuous as vol from .gtfs_helper import get_gtfs @@ -16,6 +18,7 @@ async def async_migrate_entry(hass, config_entry: ConfigEntry) -> bool: """Migrate old entry.""" + _LOGGER.warning("Migrating from version %s", config_entry.version) if config_entry.version == 1: @@ -87,9 +90,10 @@ async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: #await coordinator.async_config_entry_first_refresh() + if not coordinator.last_update_success: raise ConfigEntryNotReady - + hass.data[DOMAIN][entry.entry_id] = { "coordinator": coordinator, } @@ -125,5 +129,4 @@ def update_gtfs(call): async def update_listener(hass: HomeAssistant, entry: ConfigEntry): """Handle options update.""" hass.data[DOMAIN][entry.entry_id]['coordinator'].update_interval = timedelta(minutes=1) - return True \ No newline at end of file diff --git a/custom_components/gtfs2/config_flow.py b/custom_components/gtfs2/config_flow.py index 47c3cfc..1bea2bc 100644 --- a/custom_components/gtfs2/config_flow.py +++ b/custom_components/gtfs2/config_flow.py @@ -9,6 +9,7 @@ from homeassistant.data_entry_flow import FlowResult import homeassistant.helpers.config_validation as cv from homeassistant.core import HomeAssistant, callback + from homeassistant.helpers import selector from .const import ( @@ -22,6 +23,7 @@ CONF_TRIP_UPDATE_URL ) + from .gtfs_helper import ( get_gtfs, get_next_departure, @@ -46,6 +48,7 @@ class ConfigFlow(config_entries.ConfigFlow, domain=DOMAIN): VERSION = 5 + def __init__(self) -> None: """Init ConfigFlow.""" self._pygtfs = "" @@ -86,6 +89,7 @@ async def async_step_user(self, user_input: dict | None = None) -> FlowResult: async def async_step_source(self, user_input: dict | None = None) -> FlowResult: """Handle a flow initialized by the user.""" errors: dict[str, str] = {} + if user_input is None: return self.async_show_form( step_id="source", @@ -108,6 +112,7 @@ async def async_step_source(self, user_input: dict | None = None) -> FlowResult: _LOGGER.debug(f"UserInputs Data: {self._user_inputs}") return await self.async_step_route() + async def async_step_remove(self, user_input: dict | None = None) -> FlowResult: """Handle a flow initialized by the user.""" errors: dict[str, str] = {} @@ -181,6 +186,7 @@ async def async_step_stops(self, user_input: dict | None = None) -> FlowResult: vol.Required("destination", default=last_stop): vol.In(stops), vol.Required("name"): str, vol.Optional("include_tomorrow", default = False): selector.BooleanSelector(), + }, ), errors=errors, @@ -260,6 +266,7 @@ async def async_step_init( ) -> FlowResult: """Manage the options.""" if user_input is not None: + if user_input['real_time']: self._user_inputs.update(user_input) return await self.async_step_real_time() diff --git a/custom_components/gtfs2/const.py b/custom_components/gtfs2/const.py index 2f0f914..50f0ebd 100644 --- a/custom_components/gtfs2/const.py +++ b/custom_components/gtfs2/const.py @@ -239,6 +239,7 @@ WHEELCHAIR_BOARDING_DEFAULT = STATE_UNKNOWN WHEELCHAIR_BOARDING_OPTIONS = {1: True, 2: False} + #gtfs_rt ATTR_STOP_ID = "Stop ID" ATTR_ROUTE = "Route" @@ -273,4 +274,3 @@ TIME_STR_FORMAT = "%H:%M" - diff --git a/custom_components/gtfs2/coordinator.py b/custom_components/gtfs2/coordinator.py index eda229c..bd82adb 100644 --- a/custom_components/gtfs2/coordinator.py +++ b/custom_components/gtfs2/coordinator.py @@ -24,6 +24,7 @@ from .gtfs_helper import get_gtfs, get_next_departure, check_datasource_index, create_trip_geojson, check_extracting from .gtfs_rt_helper import get_rt_route_statuses, get_rt_trip_statuses, get_next_services + _LOGGER = logging.getLogger(__name__) @@ -74,9 +75,8 @@ async def _async_update_data(self) -> dict[str, str]: _LOGGER.warning("Cannot update this sensor as still unpacking: %s", self._data["file"]) previous_data["extracting"] = True return previous_data - - # determin static + rt or only static (refresh schedule depending) + # determinestatic + rt or only static (refresh schedule depending) #1. sensor exists with data but refresh interval not yet reached, use existing data if previous_data is not None and (datetime.datetime.strptime(previous_data["gtfs_updated_at"],'%Y-%m-%dT%H:%M:%S.%f%z') + timedelta(minutes=options.get("refresh_interval", DEFAULT_REFRESH_INTERVAL))) > dt_util.utcnow() + timedelta(seconds=1) : run_static = False @@ -144,4 +144,3 @@ async def _async_update_data(self) -> dict[str, str]: _LOGGER.debug("GTFS RT: RealTime not selected in entity options") return self._data - diff --git a/custom_components/gtfs2/gtfs_helper.py b/custom_components/gtfs2/gtfs_helper.py index b7eaa7b..4dbce7f 100644 --- a/custom_components/gtfs2/gtfs_helper.py +++ b/custom_components/gtfs2/gtfs_helper.py @@ -20,6 +20,7 @@ def get_next_departure(self): _LOGGER.debug("Get next departure with data: %s", self._data) + if check_extracting(self): _LOGGER.warning("Cannot get next depurtures on this datasource as still unpacking: %s", self._data["file"]) return {} @@ -279,6 +280,7 @@ def get_next_departure(self): dest_depart_time = ( f"{dest_depart.strftime(dt_util.DATE_STR_FORMAT)} {item['dest_depart_time']}" ) + # align on timezone depart_time = dt_util.parse_datetime(origin_depart_time).replace(tzinfo=timezone) arrival_time = dt_util.parse_datetime(dest_arrival_time).replace(tzinfo=timezone) @@ -286,7 +288,7 @@ def get_next_departure(self): origin_depart_time = dt_util.as_utc(datetime.datetime.strptime(origin_depart_time, "%Y-%m-%d %H:%M:%S")).isoformat() dest_arrival_time = dt_util.as_utc(datetime.datetime.strptime(dest_arrival_time, "%Y-%m-%d %H:%M:%S")).isoformat() dest_depart_time = dt_util.as_utc(datetime.datetime.strptime(dest_depart_time, "%Y-%m-%d %H:%M:%S")).isoformat() - + origin_stop_time = { "Arrival Time": origin_arrival_time, "Departure Time": origin_depart_time, @@ -342,6 +344,7 @@ def get_gtfs(hass, path, data, update=False): return "extracting" if update and data["extract_from"] == "url" and os.path.exists(os.path.join(gtfs_dir, file)): remove_datasource(hass, path, filename) + if update and data["extract_from"] == "zip" and os.path.exists(os.path.join(gtfs_dir, file)) and os.path.exists(os.path.join(gtfs_dir, sqlite)): os.remove(os.path.join(gtfs_dir, sqlite)) if data["extract_from"] == "zip": diff --git a/custom_components/gtfs2/gtfs_rt_helper.py b/custom_components/gtfs2/gtfs_rt_helper.py index 3a3853b..7d2e824 100644 --- a/custom_components/gtfs2/gtfs_rt_helper.py +++ b/custom_components/gtfs2/gtfs_rt_helper.py @@ -1,8 +1,10 @@ import logging from datetime import datetime, timedelta + import json import os + import homeassistant.helpers.config_validation as cv import homeassistant.util.dt as dt_util import requests @@ -50,6 +52,7 @@ DEFAULT_PATH_GEOJSON, TIME_STR_FORMAT + ) def due_in_minutes(timestamp): @@ -79,6 +82,7 @@ def log_debug(data: list, indent_level: int) -> None: def get_gtfs_feed_entities(url: str, headers, label: str): _LOGGER.debug(f"GTFS RT get_feed_entities for url: {url} , headers: {headers}, label: {label}") + feed = gtfs_realtime_pb2.FeedMessage() # type: ignore # TODO add timeout to requests call @@ -126,6 +130,7 @@ def get_next_services(self): timezone=dt_util.get_time_zone(self.hass.config.time_zone) if self._relative : + due_in = ( due_in_minutes(next_services[0].arrival_time) if len(next_services) > 0 @@ -182,6 +187,7 @@ def get_rt_route_statuses(self): if self._vehicle_position_url != "" : vehicle_positions = get_rt_vehicle_positions(self) + class StopDetails: def __init__(self, arrival_time, position): self.arrival_time = arrival_time @@ -195,7 +201,7 @@ def __init__(self, arrival_time, position): for entity in feed_entities: if entity.HasField("trip_update"): - # OUTCOMMENTED as spamming even debig log + # OUTCOMMENTED as spamming even debug log # If delimiter specified split the route ID in the gtfs rt feed #log_debug( #[ @@ -220,7 +226,7 @@ def __init__(self, arrival_time, position): route_id = entity.trip_update.trip.route_id else: route_id = route_id_split[0] - # OUTCOMMENTED as spamming even debig log + # OUTCOMMENTED as spamming even debug log #log_debug( # [ # "Feed Route ID", @@ -231,12 +237,13 @@ def __init__(self, arrival_time, position): # 1, #) + else: route_id = entity.trip_update.trip.route_id if route_id not in departure_times: - departure_times[route_id] = {} - + departure_times[route_id] = {} + if entity.trip_update.trip.direction_id is not None: direction_id = str(entity.trip_update.trip.direction_id) else: @@ -302,6 +309,7 @@ def __init__(self, arrival_time, position): ) self.info = departure_times + #_LOGGER.debug("Departure times: %s", departure_times) return departure_times @@ -411,7 +419,6 @@ def get_rt_vehicle_positions(self): geojson_element = {"geometry": {"coordinates":[],"type": "Point"}, "properties": {"id": "", "title": "", "trip_id": "", "route_id": "", "direction_id": "", "vehicle_id": "", "vehicle_label": ""}, "type": "Feature"} for entity in feed_entities: vehicle = entity.vehicle - if not vehicle.trip.trip_id: # Vehicle is not in service continue @@ -462,6 +469,4 @@ def update_geojson(self): _LOGGER.debug("GTFS RT geojson file: %s", file) with open(file, "w") as outfile: json.dump(self.geojson, outfile) - - diff --git a/custom_components/gtfs2/sensor.py b/custom_components/gtfs2/sensor.py index a2ddd2d..7a79572 100644 --- a/custom_components/gtfs2/sensor.py +++ b/custom_components/gtfs2/sensor.py @@ -12,6 +12,8 @@ from homeassistant.util import slugify import homeassistant.util.dt as dt_util +from .coordinator import GTFSRealtimeUpdateCoordinator + from .const import ( ATTR_ARRIVAL, ATTR_BICYCLE, @@ -402,7 +404,6 @@ def _update_attrs(self): # noqa: C901 PLR0911 self._attributes[ATTR_INFO_RT] = ( "No realtime information" ) - self._attr_extra_state_attributes = self._attributes return self._attr_extra_state_attributes @@ -430,5 +431,3 @@ def remove_keys(self, prefix: str) -> None: self._attributes = { k: v for k, v in self._attributes.items() if not k.startswith(prefix) } - - diff --git a/example.md b/example.md index ee1305c..dd266c6 100644 --- a/example.md +++ b/example.md @@ -11,11 +11,14 @@ In this case that is: https://data.lemet.fr/documents/LEMET-gtfs.zip ![image](https://github.com/vingerha/gtfs2/assets/44190435/7dd77425-07f8-45d0-8d0c-d9948fca6fbb) -![image](https://github.com/vingerha/gtfs2/assets/44190435/3688925f-63cd-451a-9db1-313a028c2188) +### Two options, either extract from a file you placed in the gtfs2 folder Or use a url -NOTE: this will download and unpack the zip-file to a sqlite database, which can take (many) minutes, **please be patient** +![image](https://github.com/vingerha/gtfs2/assets/44190435/e64cb7d9-7b68-4169-9cc4-e216a303f7d3) + +NOTE: this will download and unpack the zip-file to a sqlite database, which can take time (examples from 10mins to 2hrs), **please be patient** + +![image](https://github.com/vingerha/gtfs2/assets/44190435/dd26f517-1cd9-4386-b4ea-c605d02a0ac7) -![image](https://github.com/vingerha/gtfs2/assets/44190435/02ab24ed-c10d-43e5-8c3e-f221044a1a9e) ## Select the route