diff --git a/Data_collector.py b/Data_collector.py new file mode 100755 index 0000000..76e150b --- /dev/null +++ b/Data_collector.py @@ -0,0 +1,109 @@ +#!/usr/local/bin/python +from firebase_admin import credentials, firestore, initialize_app +from requests import get as api_get +from time import time as timestamp +from time import sleep as wait +from datetime import datetime +import logging + +logging.basicConfig(level=logging.INFO) + + +class DataCollector: + __CERT_PATH = "secrets/icl-iot-weather-firebase-adminsdk.json" + __API_KEY_PATH = "secrets/weather_api_key.txt" + __CITY = "London" + __API_ENDPOINT = "api.openweathermap.org/data/2.5/weather" + __TESTING = False + + def __init__(self): + self.init_firebase() + self.init_api() + + def init_firebase(self): + cred = credentials.Certificate(self.__CERT_PATH) + initialize_app(cred) + self.firestore_db = firestore.client() + + def init_api(self): + with open(self.__API_KEY_PATH, "r") as api_key_file: + self.api_key = api_key_file.read() + + self.request_url = (f"https://{self.__API_ENDPOINT}?" + f"q={self.__CITY} &" + f"appid={str(self.api_key)}".split('\n')[0]) + + def get_weather_data(self): + api_rsp = api_get(self.request_url) + return api_rsp + + def process_api_data(self, api_rsp): + api_data = api_rsp + if api_data.status_code != 200: + logging.warning("Big fuckup, " + f"expected 200 but got {api_data.status_code}") + logging.warning(f"Response: {api_data.text}") + # Email me with this data + # generate obvious outlier data and stil push + # or push previous data + return {"data": "TODO"} + + weather_data = api_data.json() + station_data = weather_data.get('main', False) + if station_data: + current_temp_kelvin = station_data.get('temp', {}) + current_temp_celcius = current_temp_kelvin - 273.15 + current_humidity_pct = station_data.get('humidity', 0) + cloud_data = weather_data.get('clouds', {}) + current_cloud_pct = cloud_data.get('all', 0) + wind_data = weather_data.get('wind', {}) + current_wind_ms = wind_data.get('speed', 0) + rain_data = weather_data.get('rain', {}) + rain_mm_1h = rain_data.get('1h', 0) + + relevant_data = { + "timestamp": timestamp(), + "datetime": datetime.now(), + "temp": current_temp_celcius, + "humidity": current_humidity_pct, + "cloud": current_cloud_pct, + "wind": current_wind_ms, + "rain_1h": rain_mm_1h, + "is_test": self.__TESTING + } + return relevant_data + else: + logging.warning("COULD NOT GET STATION DATA") + relevant_data = { + "timestamp": timestamp(), + "datetime": datetime.now() + } + return relevant_data + + def upload_to_firebase(self, data): + try: + self.firestore_db.collection(u'weather_data').add(data) + except Exception as e: + logging.warning(f"FAILED TO UPLOAD TO FIREBASE: {e}") + # Email me + + def collect_data(self): + try: + logging.info(f"Running collection at {datetime.now()}") + weather_data = self.get_weather_data() + processed_data = self.process_api_data(weather_data) + self.upload_to_firebase(processed_data) + except Exception as e: + logging.warning(f"ERROR: {e}") + # Email me + + def run_time_loop(self): + while True: + self.collect_data() + wait(60*60*1) # wait 1h + # wait(10*1*1) # wait 1h + + +if __name__ == "__main__": + weather_logger = DataCollector() + weather_logger.run_time_loop() diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..21bc676 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.8 +WORKDIR /code +COPY . . +RUN pip install -r requirements.txt +EXPOSE 80 +EXPOSE 443 +CMD "/code/sleep.py" diff --git a/playground.py b/playground.py index fc389e1..baa6001 100644 --- a/playground.py +++ b/playground.py @@ -1,5 +1,8 @@ +#!/usr/local/bin/python from firebase_admin import credentials, firestore, initialize_app from requests import get as api_get +from time import time as timestamp +from datetime import datetime cert_path = "secrets/icl-iot-weather-firebase-adminsdk-d2a8f-577125a0ff.json" cred = credentials.Certificate(cert_path) @@ -7,20 +10,30 @@ initialize_app(cred) firestore_db = firestore.client() - -#firestore_db.collection(u'songs').add({'song': 'Imagine', 'artist': 'John Lennon'} with open("secrets/weather_api_key.txt", "r") as api_key_file: api_key = api_key_file.read() request_endpoint = "api.openweathermap.org/data/2.5/weather" request_city = "London" -request_url = f"https://{request_endpoint}?q={request_city}&appid={api_key}" -print(request_url) +request_url = f"https://{str(request_endpoint)}?q={str(request_city)}&appid={str(api_key)}".split('\n')[0] api_rsp = api_get(request_url) print(api_rsp.text) -# snapshots = list(firestore_db.collection(u'weather_data').get()) -# for snapshot in snapshots: -# print(snapshot.to_dict()) +current_weather_data = { + timestamp": timestamp.now(), + "datetime": datetime.now(), + "temp": current_temp_celcius, + "humidity": current_humidity_pct, + "cloud": current_cloud_pct, + "wind": current_wind_ms, + "rain_1h": rain_mm_1h, + "is_test": is_test" +} + +snapshots = list(firestore_db.collection(u'weather_data').get()) +for snapshot in snapshots: + print(snapshot.to_dict()) + +#firestore_db.collection(u'songs').add({'song': 'Imagine', 'artist': 'John Lennon'} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f0a4d05 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,46 @@ +astroid==2.4.2 +autopep8==1.5.4 +CacheControl==0.12.6 +cachetools==4.1.1 +certifi==2020.6.20 +cffi==1.14.3 +chardet==3.0.4 +firebase-admin==4.4.0 +firestore==0.0.8 +flake8==3.8.4 +google-api-core==1.22.4 +google-api-python-client==1.12.3 +google-auth==1.22.1 +google-auth-httplib2==0.0.4 +google-cloud-core==1.4.3 +google-cloud-firestore==1.9.0 +google-cloud-storage==1.31.2 +google-crc32c==1.0.0 +google-resumable-media==1.1.0 +googleapis-common-protos==1.52.0 +grpcio==1.32.0 +httplib2==0.18.1 +idna==2.10 +importlib-metadata==2.0.0 +iso8601==0.1.13 +isort==5.6.4 +lazy-object-proxy==1.4.3 +mccabe==0.6.1 +msgpack==1.0.0 +protobuf==3.13.0 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycodestyle==2.6.0 +pycparser==2.20 +pyflakes==2.2.0 +pylint==2.6.0 +pytz==2020.1 +requests==2.24.0 +rsa==4.6 +six==1.15.0 +toml==0.10.1 +typed-ast==1.4.1 +uritemplate==3.0.1 +urllib3==1.25.10 +wrapt==1.12.1 +zipp==3.3.1 diff --git a/sleep.py b/sleep.py new file mode 100755 index 0000000..9e2599a --- /dev/null +++ b/sleep.py @@ -0,0 +1,7 @@ +#!/usr/local/bin/python +import time +import logging +logging.warning('RUNNING.>.>.') +while True: + logging.info('Running...') + time.sleep(5)