FODMET_step5-main_get_final_csv_and_shp.py

# import libraries
import os
import pandas as pd
import geopandas as gpd
import shutil
from FODMET_config_settings import Settings
from FODMET_utils import Util

config_path = 'FODMET_config.ini'
# instance of classes
config_manager = Settings(config_path)
settings = config_manager.get_config()
util = Util()


# get easily changed variables from configuration file
date_range = settings.get('RUN', 'date_range')
FOD_csv_file_path = settings.get('Data', 'FOD_csv_file_path')
FOD_shp_full_path = settings.get('Data', 'FOD_shp_path')
fenz_dataset_path = settings.get('FOD', 'fenz_dataset_path')
weather_metric_file_path = settings.get('Data', 'weather_metric_file_path')
weather_station_file_path = settings.get('Data', 'suitable_weather_station_file_path')
soi_metric_file_path = settings.get('Data', 'soi_metric_file_path')
fenz_point_shp_path = settings.get('FOD', 'fenz_point_shp_path')
fenz_polygon_shp_path = settings.get('FOD', 'fenz_polygon_shp_path')


# get date range. We only retrieve data whtin this period
tmp = date_range.split('~')
start_date_str = tmp[0]
end_date_str = tmp[1]
start_date = pd.to_datetime(start_date_str)
end_date = pd.to_datetime(end_date_str)


######### step 1: backup 1 csv files and 1 shapefile directory
# step 1.1 csv file
dot_index = FOD_csv_file_path.rfind(".")
FOD_csv_file_path = f'{FOD_csv_file_path[:dot_index]}_{date_range}{FOD_csv_file_path[dot_index:]}'
# print
if os.path.exists(FOD_csv_file_path):
    log_info = f'[INFO] Please backup the the csv file: {FOD_csv_file_path}, because we will delete it and then renegerate it.'
    print(log_info)
    
    input('[INFO] Press Enter to continue...')
# delete
try:
    if os.path.exists(FOD_csv_file_path):
        os.remove(FOD_csv_file_path)
        print(f"[INFO] Deleted {FOD_csv_file_path}")
except Exception as e:
    error_message = f"[ERROR] Failed to delete {FOD_csv_file_path}, the error message is: {str(e)}"
    print(error_message)
    raise RuntimeError(error_message)
    
print()

# step 1.2 shapefile directory
(FOD_shp_dir, _, FOD_shp_pure_name, FOD_shp_pure_ext) = util.get_file_full_path_info(FOD_shp_full_path)
FOD_shp_dir = f'{FOD_shp_dir}_{date_range}'
# print
if os.path.exists(FOD_shp_dir):
    log_info = f'[INFO] Please backup the the shapefile directory: {FOD_shp_dir}, because we will delete it and then renegerate it.'
    print(log_info)
    
    input('[INFO] Press Enter to continue...')
# delete
try:
    if os.path.exists(FOD_shp_dir):
        shutil.rmtree(FOD_shp_dir)
        print(f"[INFO] Removed the dir {FOD_shp_dir}")
except Exception as e:
    error_message = f"[ERROR] Failed to remove {FOD_shp_dir}, the error is: {str(e)}"
    print(error_message)
    raise RuntimeError(error_message)
    
# create dir because I justed deleted it
try:
    os.makedirs(FOD_shp_dir)
    print(f"[INFO] Created the dir {FOD_shp_dir}")
except Exception as e:
    error_message = f"[ERROR] Failed to create the dir, the error is: {str(e)}"
    print(error_message)
    raise RuntimeError(error_message)


########## step 2: Merging FENZ csv and Weather Station csv and Weather Metrics csv
print()
print('[INFO] Merging FENZ csv and Weather Station csv and Weather Metrics csv.')

dot_index = weather_station_file_path.rfind(".")
weather_station_file_path = f'{weather_station_file_path[:dot_index]}_{date_range}{weather_station_file_path[dot_index:]}'
weather_station_df = pd.read_csv(weather_station_file_path, na_values=[-999, -9999, "-999", "-9999"], 
                 usecols=['ID', 'ClimZnName', 'WxName', 'WxID'],
                 dtype={'WxID':str})

dot_index = weather_metric_file_path.rfind(".")
weather_metric_file_path = f'{weather_metric_file_path[:dot_index]}_{date_range}{weather_metric_file_path[dot_index:]}'
weather_metric_df = pd.read_csv(weather_metric_file_path, na_values=[-999, -9999, "-999", "-9999"])

weather_df = pd.merge(weather_station_df, weather_metric_df, on='ID', how='left')

dot_index = soi_metric_file_path.rfind(".")
soi_metric_file_path = f'{soi_metric_file_path[:dot_index]}_{date_range}{soi_metric_file_path[dot_index:]}'
soi_metric_df = pd.read_csv(soi_metric_file_path, na_values=[-999, -9999, "-999", "-9999"])

all_weather_df = pd.merge(weather_df, soi_metric_df, on='ID', how='left')
print(all_weather_df)

fenz_df = pd.read_csv(fenz_dataset_path, na_values=[-999, -9999, "-999", "-9999"], 
                      dtype={'Time': str, 'IgnSource': str, 'SlopeDir2': str, 'SlopeDir3': str, 
                             'LandCvr2': str, 'LandCvr3': str, 'Landform2': str, 
                             'Landform3': str}) # if not use dtype param, then these columns will become int64/float64 

fenz_df['Date'] = pd.to_datetime(fenz_df[['Year', 'Month', 'Day']])
fenz_df = fenz_df[(fenz_df['Date'] >= start_date) & (fenz_df['Date'] <= end_date)]
fenz_df = fenz_df.drop(columns=['Date'])
fenz_df = fenz_df.reset_index(drop=True)

merged_df = pd.merge(fenz_df, all_weather_df, on='ID', how='left')
print('[Check]', 'Length of: FODENV csv file',len(fenz_df),', Weather variables', len(weather_df), ', SOI variables',len(soi_metric_df), ', FODMET variables', len(all_weather_df), ', Merged data', len(merged_df))

# replace none with -9999
for column in merged_df.columns:
    merged_df[column] = merged_df[column].fillna("-9999")

# merge and save the final dataframe into a csv file
merged_df.to_csv(FOD_csv_file_path, index=False)
print(f'[INFO] FOD and Weather csv is saved in {FOD_csv_file_path}')


########### step 3: save weather data into shapefile
print()
print('[INFO] Saving to shapefile. It may take a few minutes. Please wait...')

## step 3.1: save point and polygon shp
fenz_polygon_df = gpd.read_file(fenz_polygon_shp_path)
fenz_polygon_df['Date'] = pd.to_datetime(fenz_polygon_df[['Year', 'Month', 'Day']])
fenz_polygon_df = fenz_polygon_df[(fenz_polygon_df['Date'] >= start_date) & (fenz_polygon_df['Date'] <= end_date)]
fenz_polygon_df = fenz_polygon_df.drop(columns=['Date'])
fenz_polygon_df = fenz_polygon_df.reset_index(drop=True)

merged_polygon_df = fenz_polygon_df.merge(all_weather_df, on='ID', how='left')
FOD_polygon_shp_path = f'{FOD_shp_dir}/{FOD_shp_pure_name}_polygons.{FOD_shp_pure_ext}'
for column in merged_polygon_df.columns:
    merged_polygon_df[column] = merged_polygon_df[column].fillna(-9999)
merged_polygon_df.to_file(FOD_polygon_shp_path, driver='ESRI Shapefile')
print(f"[INFO] Polygon shp saved in {FOD_polygon_shp_path}", ". There are", len(merged_polygon_df), "points" )

fenz_point_df = gpd.read_file(fenz_point_shp_path)
fenz_point_df['Date'] = pd.to_datetime(fenz_point_df[['Year', 'Month', 'Day']])
fenz_point_df = fenz_point_df[(fenz_point_df['Date'] >= start_date) & (fenz_point_df['Date'] <= end_date)]
fenz_point_df = fenz_point_df.drop(columns=['Date'])
fenz_point_df = fenz_point_df.reset_index(drop=True)

merged_point_df = fenz_point_df.merge(all_weather_df, on='ID', how='left')
FOD_point_shp_path = f'{FOD_shp_dir}/{FOD_shp_pure_name}_points.{FOD_shp_pure_ext}'
for column in merged_point_df.columns:
    merged_point_df[column] = merged_point_df[column].fillna(-9999)
merged_point_df.to_file(FOD_point_shp_path, driver='ESRI Shapefile')
print(f"[INFO] Point shp saved in {FOD_point_shp_path}", ". There are", len(merged_point_df), "points" )

#print(fenz_point_df.crs, fenz_polygon_df.crs)

print('[Check]','FODMET shapefiles length:',len(merged_point_df)+len(merged_polygon_df), 'FODMET csv length:',len(merged_df))


#fenz_df = gpd.GeoDataFrame(pd.concat((fenz_point_df, fenz_polygon_df), ignore_index=True), crs=fenz_point_df.crs)
#merged_df = fenz_df.merge(weather_df, on='ID', how='left')
#FOD_shp_path = f'{FOD_shp_dir}/{FOD_shp_pure_name}.{FOD_shp_pure_ext}'
#for column in merged_df.columns:
#    merged_df[column] = merged_df[column].fillna(-9999)
#merged_df.to_file(FOD_shp_path, driver='ESRI Shapefile')
#print(f"[INFO] Shp saved in {FOD_shp_path}")