fix postgis to return results, filter by states from build process

Author: reukiodo

uwpostgis/Dockerfile

@@ -65,6 +65,8 @@ RUN pip install --break-system-packages --no-cache-dir -r /requirements.txt
 # copy database from db_creator
 COPY --chmod=700 --from=db_creator /pgdata /pgdata
 RUN chown -R 5001:5001 /tmp /var/run/postgresql /pgdata
+ARG state_var
+RUN echo "$state_var" > /pgdata/states
 
 WORKDIR /opt/palantir/sidecars/shared-volumes/shared/
 USER 5001

uwpostgis/process_csv.py

@@ -4,25 +4,39 @@
 import os
 import warnings
 import psycopg
+import subprocess
+import sys
+import time
 warnings.filterwarnings('ignore')
 
-# tested 10/4/24 - JTL
+# load states
+with open('/pgdata/states', 'r') as f:
+    states = f.read().split()
+print(states)
 
-import_csv = '/opt/palantir/sidecars/shared-volumes/shared/infile.csv'
-output_csv = '/opt/palantir/sidecars/shared-volumes/shared/outfile.csv'
+# import_csv = '/opt/palantir/sidecars/shared-volumes/shared/infile.csv'
+import_csv = sys.argv[1]
+print("reading in " + import_csv)
+# output_csv = '/opt/palantir/sidecars/shared-volumes/shared/outfile.csv'
+output_csv = os.path.dirname(os.path.abspath(import_csv)) + "/outfile.csv"
+print("writing to " + output_csv)
+
+## start postgres
+subprocess.Popen("postgres")
+time.sleep(10)
 
 ## import infile
 address_df = pd.read_csv(import_csv)
 
 ## prepare for output
 address_df[['rating','stno','street',
-            'styp','city','st','zip','lat','lon']] = [None, None, None, None, 
-                                                      None,None, None, 
+            'styp','city','st','zip','lat','lon']] = [None, None, None, None,
+                                                      None, None, None,
                                                       None, None]
 address_df = address_df.reset_index(drop=True)
 
-#setup the sql connection
-#POSTGRES Connection goes here.
+# setup the sql connection
+# POSTGRES Connection goes here.
 ## destination is a Postgres database
 dest_user = 'clad_svc'
 dest_pw = 'not_on_gitlab'
@@ -36,40 +50,39 @@
 
 ## run the loop
 for address_var in range(address_df.shape[0]):
+    if any(state.upper() in str(address_df.address[address_var]).upper() for state in states):
+        try:
+            print("processing " + str(address_df.address[address_var]))
+            string_to_geocode = '''
+                SELECT
+                    g.rating
+                , ST_AsText(ST_SnapToGrid(g.geomout,0.00001)) As wktlonlat
+                , (addy).address As stno
+                , (addy).streetname As street
+                , (addy).streettypeabbrev As styp
+                , (addy).location As city
+                , (addy).stateabbrev As st
+                , (addy).zip FROM geocode(''' + "'" + address_df.address[address_var] + "'" +  ''') As g;'''
 
-    try:
-
-        string_to_geocode = '''
-            SELECT 
-                g.rating
-            , ST_AsText(ST_SnapToGrid(g.geomout,0.00001)) As wktlonlat
-            , (addy).address As stno
-            , (addy).streetname As street
-            , (addy).streettypeabbrev As styp
-            , (addy).location As city
-            , (addy).stateabbrev As st
-            , (addy).zip FROM geocode(''' + "'" + address_df.address[address_var] + "'" +  ''') As g;'''
-
-        all_results = pd.read_sql(string_to_geocode, cnxn)
+            all_results = pd.read_sql(string_to_geocode, cnxn)
 
-        # grab first result which will be one with best rating 
-        temp_results = all_results.head(1)
-        latlong = temp_results['wktlonlat'][0].replace('POINT(','').replace(')','',).split(" ")
-        print(latlong)
-        
-        address_df.rating[address_var] = temp_results.rating[0]
-        address_df.stno[address_var] = temp_results.stno[0]
-        address_df.street[address_var] = temp_results.street[0]
-        address_df.styp[address_var] = temp_results.styp[0]
-        address_df.city[address_var] = temp_results.city[0]
-        address_df.st[address_var] = temp_results.st[0]
-        address_df.zip[address_var] = temp_results.zip[0]
-        address_df.lat[address_var] = latlong[1]
-        address_df.lon[address_var] = latlong[0]
+            # grab first result which will be one with best rating
+            temp_results = all_results.head(1)
+            latlong = temp_results['wktlonlat'][0].replace('POINT(','').replace(')','',).split(" ")
+            print(latlong)
 
-    except:
-        pass
+            address_df.rating[address_var] = temp_results.rating[0]
+            address_df.stno[address_var] = temp_results.stno[0]
+            address_df.street[address_var] = temp_results.street[0]
+            address_df.styp[address_var] = temp_results.styp[0]
+            address_df.city[address_var] = temp_results.city[0]
+            address_df.st[address_var] = temp_results.st[0]
+            address_df.zip[address_var] = temp_results.zip[0]
+            address_df.lat[address_var] = latlong[1]
+            address_df.lon[address_var] = latlong[0]
 
+        except:
+            pass
 
 ## write outout
 address_df.to_csv(output_csv)