-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathdl_data.py
179 lines (141 loc) · 6.6 KB
/
dl_data.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
#!/usr/bin/env python3
"""Download and manipulate BOM Data
This script allows the user to download the most recently available
weather conditions for a specific location from the Bureau of
Meteorology (BOM) website, manipulate that data including adding
a base64 encoded image of the location at that time, and then converts
that data back into json format.
Currently being tested on data from Toowoomba Wellcamp Airport.
`http://www.bom.gov.au/fwo/IDQ60801/IDQ60801.99435.json`
This script requires that several pypi modules be installed within the
Python environment you are running this script in. These modules are
`requests` to access online content, `json` to convert data to and from
json format, and `datetime` to change how dates & times are displayed.
This script also requires two accompanying custom modules be installed.
These modules are: `dl_img_conv_b64` (to download an image and convert
to base64) and `wellcamp_urls` (containing the links to the most recent
data and image from the target location, Wellcamp Airport).
The dl_data python file can be imported as a module and contains the
following functions:
* __init__ - to construct the main function
* dl_weather - returns the data as a dictionary in JSON format
* dl_time - extracts a human-readable version of the local time
from the BOM data
"""
import requests
import json
import datetime
import dl_img_conv_b64
from wellcamp_urls import image_url, data_url
class DownloadData:
"""
A class to download the most recent weather conditions at a
specific location. Output in json format.
...
Attributes
----------
'data_url' : str
The URL of the data to be downloaded from the BOM website
Defined outside of the scope of the class.
'image_url' : str
The URL of the image to be downloaded from AirServiceAus site
Defined outside of the scope of the class.
Methods
-------
'dl_weather(data_url, image_url)'
Downloads the most recent weather conditions at a set location
as a dictionary, adds to that data the base64 representation of
an image that is the current view of that location.
'dl_time(data_url)'
Extracts the time that the most recent weather conditions data
was uploaded and returns it in a neat, human readable format.
"""
def __init__(self):
"""
The constructor for DownloadData class.
Only used to construct the class currently.
"""
pass
def dl_weather(self, data_url, image_url):
"""
Method called upon to create a dictionary of current weather
conditions at a specific location.
This method downloads the data from the BOM website, extracts
the required data and places it into a dictionary, downloads
an image from AirServices Australia, converts that image into
a base64 encoded string, and then adds that string to the
dictionary.
...
Parameters
----------
'data_url' : str
The URL of the data to be downloaded from the BOM website
Defined outside of the scope of the class.
'image_url' : str
The URL of the image to be downloaded from AirServiceAus site
Defined outside of the scope of the class.
`returned_object` : file-like object
Object containing the image URL server's response to the HTTP
GET request. `raw` is used to create a file-like object
representation of the response. Use of raw requires that
stream=True be set on the request. stream=True guarantees no
interruptions while downloading the image, avoiding corruption
of the object being downloaded. Setting decode_content=True
forces the decompression of the response.raw file-like object.
`result` : str [in utf-8 format]
The string representation of the base64 encoded image.
Encoded into utf-8 format to return string without the [b' '].
Raises
------
status.code if/else statement:
Checks if image was retrieved successfully (HTTP Status = 200).
If image download fails, (HTTP status code != 200), an error
message is printed in the console.
"""
x = requests.get(data_url)
# Gets information from website and turns it into a usable format
# [0] is the position of the most recent dataset inserted into the list of datasets ["data"]
if x.status_code == 200:
y = json.loads(x.text)["observations"]["data"][0]
# adds base64 image to data
y.update(local_image_b64=dl_img_conv_b64.DownloadConvert().conv_img_to_b64(image_url))
# Removes the unnecessary "sort order" value;
y.pop("sort_order", None)
result = str(y)
else:
result = str('Data Couldn\'t be retrieved')
return result
def dl_time(self, data_url):
# defined in previous function
x = requests.get(data_url)
y = json.loads(x.text)["observations"]["data"][0]["local_date_time_full"]
# Creates a heading for displaying the current dataset with it's time as part of the heading
t1 = datetime.datetime.strptime(y, "%Y%m%d%H%M%S")
t2 = datetime.datetime.strftime(t1, '%A, %d %B %Y %H:%M:%S')
# prints the local time that the data was uploaded to the BOM website
return t2
if __name__ == "__main__":
"""
Function to test the functions in this module
Uses the method conv_img_to_b64 from the class DownloadConvert to
downloads and convert a sample image into a utf-8 encoded base64
string. This string is then printed to the console/terminal.
This code will only run if the module is run directly (not imported
by another module). It can be used as a unit test for this module.
...
Attributes
----------
image_url : str
URL of the sample image to be downloaded.
output: str
The string representation of the base64 encoded sample image.
Encoded into utf-8 format to return string without the [b' '].
Methods
-------
dl_weather(data_url)
Downloads an online image, then converts it to Base64 and then
returns the result.
"""
output_weather = DownloadData().dl_weather(data_url, image_url)
output_time = DownloadData().dl_time(data_url)
print(output_time + "\n" + output_weather)