Update

Data_collector.py

@@ -0,0 +1,109 @@
+#!/usr/local/bin/python
+from firebase_admin import credentials, firestore, initialize_app
+from requests import get as api_get
+from time import time as timestamp
+from time import sleep as wait
+from datetime import datetime
+import logging
+
+logging.basicConfig(level=logging.INFO)
+
+
+class DataCollector:
+    __CERT_PATH = "secrets/icl-iot-weather-firebase-adminsdk.json"
+    __API_KEY_PATH = "secrets/weather_api_key.txt"
+    __CITY = "London"
+    __API_ENDPOINT = "api.openweathermap.org/data/2.5/weather"
+    __TESTING = False
+
+    def __init__(self):
+        self.init_firebase()
+        self.init_api()
+
+    def init_firebase(self):
+        cred = credentials.Certificate(self.__CERT_PATH)
+        initialize_app(cred)
+        self.firestore_db = firestore.client()
+
+    def init_api(self):
+        with open(self.__API_KEY_PATH, "r") as api_key_file:
+            self.api_key = api_key_file.read()
+
+        self.request_url = (f"https://{self.__API_ENDPOINT}?"
+                            f"q={self.__CITY} &"
+                            f"appid={str(self.api_key)}".split('\n')[0])
+
+    def get_weather_data(self):
+        api_rsp = api_get(self.request_url)
+        return api_rsp
+
+    def process_api_data(self, api_rsp):
+        api_data = api_rsp
+        if api_data.status_code != 200:
+            logging.warning("Big fuckup, "
+                            f"expected 200 but got {api_data.status_code}")
+            logging.warning(f"Response: {api_data.text}")
+            # Email me with this data
+            # generate obvious outlier data and stil push
+            # or push previous data
+            return {"data": "TODO"}
+
+        weather_data = api_data.json()
+        station_data = weather_data.get('main', False)
+        if station_data:
+            current_temp_kelvin = station_data.get('temp', {})
+            current_temp_celcius = current_temp_kelvin - 273.15
+            current_humidity_pct = station_data.get('humidity', 0)
+            cloud_data = weather_data.get('clouds', {})
+            current_cloud_pct = cloud_data.get('all', 0)
+            wind_data = weather_data.get('wind', {})
+            current_wind_ms = wind_data.get('speed', 0)
+            rain_data = weather_data.get('rain', {})
+            rain_mm_1h = rain_data.get('1h', 0)
+
+            relevant_data = {
+                "timestamp": timestamp(),
+                "datetime": datetime.now(),
+                "temp": current_temp_celcius,
+                "humidity": current_humidity_pct,
+                "cloud": current_cloud_pct,
+                "wind": current_wind_ms,
+                "rain_1h": rain_mm_1h,
+                "is_test": self.__TESTING
+            }
+            return relevant_data
+        else:
+            logging.warning("COULD NOT GET STATION DATA")
+            relevant_data = {
+                "timestamp": timestamp(),
+                "datetime": datetime.now()
+            }
+            return relevant_data
+
+    def upload_to_firebase(self, data):
+        try:
+            self.firestore_db.collection(u'weather_data').add(data)
+        except Exception as e:
+            logging.warning(f"FAILED TO UPLOAD TO FIREBASE: {e}")
+            # Email me
+
+    def collect_data(self):
+        try:
+            logging.info(f"Running collection at {datetime.now()}")
+            weather_data = self.get_weather_data()
+            processed_data = self.process_api_data(weather_data)
+            self.upload_to_firebase(processed_data)
+        except Exception as e:
+            logging.warning(f"ERROR: {e}")
+            # Email me
+
+    def run_time_loop(self):
+        while True:
+            self.collect_data()
+            wait(60*60*1)  # wait 1h
+            # wait(10*1*1)  # wait 1h
+
+
+if __name__ == "__main__":
+    weather_logger = DataCollector()
+    weather_logger.run_time_loop()

Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3.8
+WORKDIR /code
+COPY . .
+RUN pip install -r requirements.txt
+EXPOSE 80
+EXPOSE 443
+CMD "/code/sleep.py"

playground.py

@@ -1,5 +1,8 @@
+#!/usr/local/bin/python
 from firebase_admin import credentials, firestore, initialize_app
 from requests import get as api_get
+from time import time as timestamp
+from datetime import datetime
 
 cert_path = "secrets/icl-iot-weather-firebase-adminsdk-d2a8f-577125a0ff.json"
 cred = credentials.Certificate(cert_path)
@@ -7,20 +10,30 @@ initialize_app(cred)
 
 firestore_db = firestore.client()
 
-
-#firestore_db.collection(u'songs').add({'song': 'Imagine', 'artist': 'John Lennon'}
 with open("secrets/weather_api_key.txt", "r") as api_key_file:
     api_key = api_key_file.read()
 
 request_endpoint = "api.openweathermap.org/data/2.5/weather"
 request_city = "London"
 
-request_url = f"https://{request_endpoint}?q={request_city}&appid={api_key}"
-print(request_url)
+request_url = f"https://{str(request_endpoint)}?q={str(request_city)}&appid={str(api_key)}".split('\n')[0]
 
 api_rsp = api_get(request_url)
 print(api_rsp.text)
 
-# snapshots = list(firestore_db.collection(u'weather_data').get())
-# for snapshot in snapshots:
-#     print(snapshot.to_dict())
+current_weather_data = {
+    timestamp": timestamp.now(),
+    "datetime": datetime.now(),
+    "temp": current_temp_celcius,
+    "humidity": current_humidity_pct,
+    "cloud": current_cloud_pct,
+    "wind": current_wind_ms,
+    "rain_1h": rain_mm_1h,
+    "is_test": is_test"
+}
+
+snapshots = list(firestore_db.collection(u'weather_data').get())
+for snapshot in snapshots:
+    print(snapshot.to_dict())
+
+#firestore_db.collection(u'songs').add({'song': 'Imagine', 'artist': 'John Lennon'}

requirements.txt

@@ -0,0 +1,46 @@
+astroid==2.4.2
+autopep8==1.5.4
+CacheControl==0.12.6
+cachetools==4.1.1
+certifi==2020.6.20
+cffi==1.14.3
+chardet==3.0.4
+firebase-admin==4.4.0
+firestore==0.0.8
+flake8==3.8.4
+google-api-core==1.22.4
+google-api-python-client==1.12.3
+google-auth==1.22.1
+google-auth-httplib2==0.0.4
+google-cloud-core==1.4.3
+google-cloud-firestore==1.9.0
+google-cloud-storage==1.31.2
+google-crc32c==1.0.0
+google-resumable-media==1.1.0
+googleapis-common-protos==1.52.0
+grpcio==1.32.0
+httplib2==0.18.1
+idna==2.10
+importlib-metadata==2.0.0
+iso8601==0.1.13
+isort==5.6.4
+lazy-object-proxy==1.4.3
+mccabe==0.6.1
+msgpack==1.0.0
+protobuf==3.13.0
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycodestyle==2.6.0
+pycparser==2.20
+pyflakes==2.2.0
+pylint==2.6.0
+pytz==2020.1
+requests==2.24.0
+rsa==4.6
+six==1.15.0
+toml==0.10.1
+typed-ast==1.4.1
+uritemplate==3.0.1
+urllib3==1.25.10
+wrapt==1.12.1
+zipp==3.3.1

sleep.py

@@ -0,0 +1,7 @@
+#!/usr/local/bin/python
+import time
+import logging
+logging.warning('RUNNING.>.>.')
+while True:
+    logging.info('Running...')
+    time.sleep(5)