A few odds and ends...some Socrata related fixes, and a unit test

Author: Sara Siegal

.gitignore

@@ -43,3 +43,6 @@ venv/
 
 .DS_Store
 
+# claude
+CLAUDE.md
+/.claude

src/features/Map/index.jsx

@@ -329,7 +329,7 @@ class MapContainer extends React.Component {
         endDate
       );
     } else {
-      requests = await getServiceRequestSocrata();
+      requests = await getServiceRequestSocrata(startDate, endDate);
     }
     return requests;
   }
@@ -373,8 +373,12 @@ class MapContainer extends React.Component {
       const typeId = getTypeIdFromTypeName(
         request.RequestType ?? request.requesttype
       );
-      const closedDate =
-        request.ClosedDate ?? moment(request.closeddate).valueOf();
+
+      // request.ClosedDate is undefined for Socrata data source (Socrata uses lowercase, PascalCase field doesn't exist)
+      // For open requests: request.closeddate is also undefined (absent from JSON when no close date)
+      // For closed requests: request.closeddate is a Date object (yup coerced it), so closedDate = moment(Date).valueOf() — also not null
+      const rawClosedDate = request.ClosedDate ?? request.closeddate;
+      const closedDate = rawClosedDate != null ? moment(rawClosedDate).valueOf() : null;
       const createdDateMs = moment(
         request.CreatedDate ?? request.createddate
       ).valueOf();

src/utils/DataService.js

@@ -60,17 +60,30 @@ const socrataServiceRequestSchema = object({
 
 const srArraySchema = array().of(socrataServiceRequestSchema);
 
-export async function getServiceRequestSocrata() {
+export async function getServiceRequestSocrata(startDate, endDate) {
   const dataLoadStartTime = performance.now();
 
   try {
-    // Fetch current year SR data through Socrata API
-    const currentYear = String(new Date().getFullYear());
-    const currentYearFilename = `https://data.lacity.org/resource/${dataResources[currentYear]}.json`
-    const response = await fetch(
-      currentYearFilename
+    // Build list of years covered by the date range
+    const startYear = moment(startDate).year();
+    const endYear = moment(endDate).year();
+    const years = [];
+    for (let year = startYear; year <= endYear; year++) {
+      years.push(String(year));
+    }
+
+    // Fetch data for each year filtered by the requested date range.
+    // Without a $where clause, Socrata returns only 1000 records in internal-ID
+    // order (i.e. the oldest records first), which would all fail the client-side
+    // Mapbox date filter. We also raise $limit well above the default 1000 so that
+    // the full date range is covered.
+    const unvalidatedByYear = await Promise.all(
+      years.map((year) => {
+        const where = `createddate >= '${startDate}T00:00:00.000' AND createddate <= '${endDate}T23:59:59.999'`;
+        const url = `https://data.lacity.org/resource/${dataResources[year]}.json?$where=${encodeURIComponent(where)}&$limit=1000`;
+        return fetch(url).then((res) => res.json());
+      })
     );
-    const unvalidatedSrs = await response.json();
 
     const dataLoadEndTime = performance.now();
     console.log(
@@ -80,7 +93,12 @@ export async function getServiceRequestSocrata() {
     );
 
     const mapLoadStartTime = performance.now();
-    const validatedSrs = await srArraySchema.validate(unvalidatedSrs);
+    const validatedByYear = await Promise.all(
+      unvalidatedByYear.map((unvalidatedSrs) =>
+        srArraySchema.validate(unvalidatedSrs)
+      )
+    );
+    const validatedSrs = validatedByYear.flat();
     const mapLoadEndTime = performance.now();
     console.log(
       `Socrata map preparation time: ${Math.floor(