Merge pull request #263 from dwreeves/main

Fix email exports issue with manual pagination, and fix timestamp representation in frontend

Author: dwreeves

app/blueprints/frontend.py

@@ -4,6 +4,7 @@
 from typing import Dict
 
 import pandas as pd
+import pytz
 from flask import Blueprint
 from flask import current_app
 from flask import flash
@@ -67,7 +68,9 @@ def flag_widget_params(force_display: bool = False) -> Dict[str, Any]:
     return dict(
         boathouses=boathouses,
         website_options=website_options,
-        model_last_updated_time=get_latest_prediction_time(),
+        model_last_updated_time=get_latest_prediction_time().astimezone(
+            pytz.timezone("US/Eastern")
+        ),
         boating_season=force_display or website_options.boating_season,
         flagging_message=website_options.rendered_flagging_message,
     )

app/data/processing/hobolink.py

@@ -48,42 +48,67 @@ def get_live_hobolink_data(
         end_date = datetime.now(tz=UTC)
     if start_date is None:
         start_date = end_date - timedelta(days=days_ago)
-    res = request_to_hobolink(start_date=start_date, end_date=end_date, loggers=loggers)
-    df = parse_hobolink_data(res.json(), exclude_sensors=exclude_sensors)
+    data = request_to_hobolink(start_date=start_date, end_date=end_date, loggers=loggers)
+    df = parse_hobolink_data(data, exclude_sensors=exclude_sensors)
     return df
 
 
 def request_to_hobolink(
     start_date: datetime, end_date: datetime, loggers: str = None, token: str | None = None
-) -> requests.models.Response:
+) -> list[dict[str, Any]]:
     """ """
     if loggers is None:
         loggers = current_app.config["HOBOLINK_LOGGERS"]
     if token is None:
         token = current_app.config["HOBOLINK_BEARER_TOKEN"]
 
-    res = requests.get(
-        urljoin(BASE_URL, "/v1/data"),
-        params={
-            "start_date_time": start_date.strftime("%Y-%m-%d %H:%M:%S"),
-            "end_date_time": end_date.strftime("%Y-%m-%d %H:%M:%S"),
-            "loggers": loggers,
-        },
-        headers={"Authorization": f"Bearer {token}", "accept": "application/json"},
-    )
-
-    if res.status_code >= 400:
-        error_msg = (
-            f"API request to the HOBOlink endpoint failed with status code {res.status_code}:"
-            + res.text
+    # HOBOLink API returns max of 100,000 results at a time.
+    # Additionally, there is no pagination.
+    # Therefore we must paginate ourselves.
+    #
+    # Note that API timestamps also pull full closed interval of data,
+    # so we need to be careful to not accidentally pull duplicates by timestamp.
+    pagination_delta = timedelta(days=10)
+    epsilon_delta = timedelta(seconds=1)
+    data: list[dict[str, Any]] = []
+
+    start_date_for_req = start_date
+    end_date_for_req = min(start_date + pagination_delta, end_date)
+    half_interval = False
+
+    while True:
+        res = requests.get(
+            urljoin(BASE_URL, "/v1/data"),
+            params={
+                "start_date_time": start_date_for_req.strftime("%Y-%m-%d %H:%M:%S"),
+                "end_date_time": end_date_for_req.strftime("%Y-%m-%d %H:%M:%S"),
+                "loggers": loggers,
+            },
+            headers={"Authorization": f"Bearer {token}", "accept": "application/json"},
         )
-        abort(500, error_msg)
+        if res.status_code >= 400:
+            error_msg = (
+                f"API request to the HOBOlink endpoint failed with status code {res.status_code}:"
+                + res.text
+            )
+            abort(500, error_msg)
 
-    return res
+        data.extend(res.json()["data"])
+
+        if end_date_for_req == end_date:
+            break
+
+        end_date_for_req += pagination_delta
+        start_date_for_req += pagination_delta
+        if not half_interval:
+            start_date_for_req += epsilon_delta
+            half_interval = True
+
+    return data
 
 
 def parse_hobolink_data(
-    data: dict[str, Any], exclude_sensors: list[str] | None = None
+    data: list[dict[str, Any]], exclude_sensors: list[str] | None = None
 ) -> pd.DataFrame:
     """
     Clean the response from the HOBOlink API.
@@ -98,7 +123,7 @@ def parse_hobolink_data(
         # This sensor is for internal temp of device.
         exclude_sensors = current_app.config["HOBOLINK_EXCLUDE_SENSORS"]
 
-    df = pd.DataFrame(data["data"])
+    df = pd.DataFrame(data)
     df = df.loc[~df["sensor_sn"].isin(exclude_sensors), :]
     df["time"] = pd.to_datetime(df["timestamp"])
     df["sensor_measurement_type"] = df["sensor_measurement_type"].str.lower().str.replace(" ", "_")

app/templates/boathouses.html

@@ -21,6 +21,6 @@ <h2>Map</h2>
     {%with flagging_message='' %}{% include get_widget_filename(version=2) %}{% endwith %}
     </div>
     <div class="last-updated">
-        ( Last updated: {{ model_last_updated_time | strftime }} )
+        ( Last updated: {{ model_last_updated_time | strftime("%B %-d, %Y at %-I:%M %p") }} )
     </div>
 {% endblock %}

app/templates/flags.html

@@ -36,7 +36,7 @@
     {% include get_widget_filename(version=version) %}
 
     <div class="last-updated">
-        ( Last updated: {{ model_last_updated_time | strftime }} )
+        ( Last updated: {{ model_last_updated_time | strftime("%B %-d, %Y at %-I:%M %p") }} )
     </div>
     <div class="info">
         <a href="{{ url_for('flagging.index', _external=True) }}" target="_blank">Click here for more information.</a>

app/templates/index.html

@@ -14,7 +14,7 @@ <h2>Water Quality Today</h2>
     {% include get_widget_filename() %}
 </div>
 <div class="last-updated">
-    ( Last updated: {{ model_last_updated_time | strftime }} )
+    ( Last updated: {{ model_last_updated_time | strftime("%B %-d, %Y at %-I:%M %p") }} )
 </div>
 <h2>What Do the Flags Mean?</h2>
 <ul class="flag-explanation">