Merge pull request #132 from rootcodelabs/deployment-est-gpu

nuwangeek · web-flow · commit 1359c0dbd52b · 2025-11-17T17:32:47.000+05:30
Get update from deployment-est-gpu into deployment-set-gpu-Bimsara
diff --git a/DSL/Resql/global-classifier/POST/get-agency-centops.sql b/DSL/Resql/global-classifier/POST/get-agency-centops.sql
@@ -0,0 +1,4 @@
+-- Check if agency exists in mock_centops table
+SELECT agency_id
+FROM public.mock_centops
+WHERE agency_id = :agencyId;
diff --git a/DSL/Resql/global-classifier/POST/get-all-dataset-versions.sql b/DSL/Resql/global-classifier/POST/get-all-dataset-versions.sql
@@ -1,3 +1,4 @@
 SELECT id, major, minor
 FROM public.dataset_versions
+WHERE generation_status = 'Generation_Success'
 ORDER BY id;
diff --git a/DSL/Resql/global-classifier/POST/get-datasets.sql b/DSL/Resql/global-classifier/POST/get-datasets.sql
@@ -1,27 +1,29 @@
 SELECT 
-    id,
-    major,
-    minor,
-    created_at,
-    generation_status,
-    last_model_trained,
-    last_trained,
+    dv.id,
+    dv.major,
+    dv.minor,
+    dv.created_at,
+    dv.generation_status,
+    COALESCE(dm.model_name, dv.last_model_trained) AS last_model_trained,
+    dv.last_trained,
     CEIL(COUNT(*) OVER() / :page_size::DECIMAL) AS total_pages
 FROM 
-    dataset_versions
+    dataset_versions dv
+LEFT JOIN 
+    data_models dm ON dv.last_model_trained = dm.model_id::text
 WHERE
-    (:generation_status = 'all' OR generation_status ILIKE '%' || :generation_status || '%')
+    (:generation_status = 'all' OR dv.generation_status ILIKE '%' || :generation_status || '%')
     AND (:dataset_name = 'all' 
-         OR POSITION(LOWER(:dataset_name) IN LOWER(CONCAT('v', major, '.', minor))) > 0
-         OR POSITION(LOWER(:dataset_name) IN LOWER(CONCAT(major, '.', minor))) > 0
-         OR POSITION(LOWER(:dataset_name) IN LOWER(major::text)) > 0
-         OR POSITION(LOWER(:dataset_name) IN LOWER(minor::text)) > 0)
+         OR POSITION(LOWER(:dataset_name) IN LOWER(CONCAT('v', dv.major, '.', dv.minor))) > 0
+         OR POSITION(LOWER(:dataset_name) IN LOWER(CONCAT(dv.major, '.', dv.minor))) > 0
+         OR POSITION(LOWER(:dataset_name) IN LOWER(dv.major::text)) > 0
+         OR POSITION(LOWER(:dataset_name) IN LOWER(dv.minor::text)) > 0)
 ORDER BY
-    CASE WHEN :sort_by = 'created_at' AND :sort_type = 'asc' THEN created_at END ASC,
-    CASE WHEN :sort_by = 'created_at' AND :sort_type = 'desc' THEN created_at END DESC,
-    -- CASE WHEN :sort_by = 'major' AND :sort_type = 'asc' THEN major END ASC,
-    -- CASE WHEN :sort_by = 'major' AND :sort_type = 'desc' THEN major END DESC,
-    -- CASE WHEN :sort_by = 'minor' AND :sort_type = 'asc' THEN minor END ASC,
-    -- CASE WHEN :sort_by = 'minor' AND :sort_type = 'desc' THEN minor END DESC,
-    CASE WHEN :sort_by IS NULL OR :sort_by = '' THEN created_at END DESC
+    CASE WHEN :sort_by = 'created_at' AND :sort_type = 'asc' THEN dv.created_at END ASC,
+    CASE WHEN :sort_by = 'created_at' AND :sort_type = 'desc' THEN dv.created_at END DESC,
+    -- CASE WHEN :sort_by = 'major' AND :sort_type = 'asc' THEN dv.major END ASC,
+    -- CASE WHEN :sort_by = 'major' AND :sort_type = 'desc' THEN dv.major END DESC,
+    -- CASE WHEN :sort_by = 'minor' AND :sort_type = 'asc' THEN dv.minor END ASC,
+    -- CASE WHEN :sort_by = 'minor' AND :sort_type = 'desc' THEN dv.minor END DESC,
+    CASE WHEN :sort_by IS NULL OR :sort_by = '' THEN dv.created_at END DESC
 OFFSET ((GREATEST(:page, 1) - 1) * :page_size) LIMIT :page_size;
diff --git a/DSL/Resql/global-classifier/POST/insert-agency-centops.sql b/DSL/Resql/global-classifier/POST/insert-agency-centops.sql
@@ -0,0 +1,3 @@
+-- Insert new agency into mock_centops table
+INSERT INTO public.mock_centops (agency_id, agency_name, created_at)
+VALUES (:agencyId, :agencyName, NOW());
diff --git a/DSL/Resql/global-classifier/POST/update-datasets-connected-models.sql b/DSL/Resql/global-classifier/POST/update-datasets-connected-models.sql
@@ -8,7 +8,8 @@ SET
         ELSE 
             connected_models
     END,
-    last_trained = CURRENT_TIMESTAMP
+    last_trained = CURRENT_TIMESTAMP,
+    last_model_trained = :modelId
 WHERE 
     id = :datasetId
 RETURNING 
diff --git a/DSL/Ruuter.public/global-classifier/POST/centops/add.yml b/DSL/Ruuter.public/global-classifier/POST/centops/add.yml
@@ -0,0 +1,76 @@
+declaration:
+  call: declare
+  version: 0.1
+  description: "Create a new data model record in data_models, update connected dataset and initiate training"
+  method: post
+  accepts: json
+  returns: json
+  namespace: global-classifier
+  # Input Validation Schema
+  allowlist:
+    body:
+      - field: agency_id
+        type: string
+        description: "ID of the agency (required)"
+      - field: agency_name
+        type: string
+        description: "Name of the agency (required)"
+        next: extractRequestData
+
+# Data Extraction
+extractRequestData:
+  assign:
+    agency_id: ${incoming.body.agency_id}
+    agency_name: ${incoming.body.agency_name}
+  next: checkAgencyExists
+
+# Check if Agency Already Exists
+checkAgencyExists:
+  call: http.post
+  args:
+    url: "[#GLOBAL_CLASSIFIER_RESQL]/get-agency-centops"
+    body:
+      agencyId: ${agency_id}
+  result: agency_check_res
+  next: evaluateAgencyExistence
+
+# Evaluate Agency Existence
+evaluateAgencyExistence:
+  switch:
+    - condition: "${agency_check_res.response.body.length > 0}"
+      next: return_agency_exists
+  next: insertAgencyData
+
+# Insert Agency Data
+insertAgencyData:
+  call: http.post
+  args:
+    url: "[#GLOBAL_CLASSIFIER_RESQL]/insert-agency-centops"
+    body:
+      agencyId: ${agency_id}
+      agencyName: ${agency_name}
+      createdAt: ${new Date().toISOString()}
+  result: insert_agency_res
+  next: checkInsertResult
+
+# Check Insert Result
+checkInsertResult:
+  switch:
+    - condition: ${200 <= insert_agency_res.response.statusCodeValue && insert_agency_res.response.statusCodeValue < 300}
+      next: return_success
+  next: return_insert_failed
+
+return_success:
+  return: "Agency created successfully in centops"
+  status: 200
+  next: end
+
+return_agency_exists:
+  return: "error: agency already exists in centops"
+  status: 400
+  next: end
+
+return_insert_failed:
+  return: "error: failed to add agency to centops"
+  status: 500
+  next: end
diff --git a/DSL/Ruuter.public/global-classifier/POST/data/callback.yml b/DSL/Ruuter.public/global-classifier/POST/data/callback.yml
@@ -25,7 +25,7 @@ declare:
         description: "List of agency IDs for which the dataset generation was completed"
 
 log_callback_received:
-  log: "Dataset generation callback received - Task ID: ${incoming.body.task_id}, Status: ${incoming.body.status}, File Path: ${incoming.body.filePath}"
+  log: "Dataset generation callback received - Task ID: ${incoming.body.task_id}, Status: ${incoming.body.status}, File Path: ${incoming.body.filePath}, Result: ${incoming.body.results}"
   next: extract_callback_data
 
 extract_callback_data:
diff --git a/GUI/src/components/molecules/DataModelForm/index.tsx b/GUI/src/components/molecules/DataModelForm/index.tsx
@@ -21,13 +21,15 @@ type DataModelFormType = {
   handleChange: (name: keyof DataModel, value: any) => void;
   errors?: Record<string, string>;
   type: string;
+  datasetVersions?: any;
 };
 
 const DataModelForm: FC<DataModelFormType> = ({
   dataModel,
   handleChange,
   errors,
   type,
+  datasetVersions: propDatasetVersions,
 }) => {
   const { t } = useTranslation();
   const [showTrainingResults, setShowTrainingResults] = useState(true);
@@ -39,8 +41,12 @@ const DataModelForm: FC<DataModelFormType> = ({
   const { data: datasetVersions } = useQuery({
     queryKey: dataModelsQueryKeys.DATA_MODEL_DEPLOYMENT_ENVIRONMENTS(),
     queryFn: () => getAllDatasetVersions(),
+    enabled: !propDatasetVersions, // Only fetch if not provided as prop
   });
 
+  // Use prop datasetVersions if provided, otherwise use the queried data
+  const finalDatasetVersions = propDatasetVersions || datasetVersions;
+
  let trainingResults = null;
   if (dataModel?.trainingResults?.value) {
     try {
@@ -91,10 +97,19 @@ const DataModelForm: FC<DataModelFormType> = ({
             }} >
               <FormSelect
                 name="datasetId"
-                options={toLabelValueArray(datasetVersions, 'id', 'version') ?? []}
+                options={toLabelValueArray(finalDatasetVersions, 'id', 'version') ?? []}
                 label=""
                 onSelectionChange={(selection) => {
                   handleChange('datasetId', selection?.value);
+                  // Update version when dataset is selected
+                  if (selection?.value && finalDatasetVersions) {
+                    const selectedDataset = finalDatasetVersions.find(
+                      (dataset: any) => dataset.id.toString() === selection.value
+                    );
+                    if (selectedDataset?.version) {
+                      handleChange('version', selectedDataset.version);
+                    }
+                  }
                 }}
                 value={dataModel?.datasetId === null && ""}
                 defaultValue={dataModel?.datasetId ? dataModel?.datasetId : ""}
diff --git a/GUI/src/pages/DataModels/ConfigureDataModel.tsx b/GUI/src/pages/DataModels/ConfigureDataModel.tsx
@@ -15,6 +15,7 @@ import { dataModelsQueryKeys } from 'utils/queryKeys';
 import { useTranslation } from 'react-i18next';
 import './DataModels.scss';
 import { configureDataModel, deleteDataModel, deployDataModel, getDataModelMetadata, getProductionDataModel } from 'services/datamodels';
+import { getAllDatasetVersions } from 'services/datasets';
 import { use } from 'i18next';
 import { set } from 'date-fns';
 import { areArraysEqual } from 'utils/commonUtilts';
@@ -41,6 +42,11 @@ const ConfigureDataModel: FC = () => {
     queryFn: () => getProductionDataModel(),
   });
 
+  const { data: datasetVersions } = useQuery({
+    queryKey: dataModelsQueryKeys.DATA_MODEL_DEPLOYMENT_ENVIRONMENTS(),
+    queryFn: () => getAllDatasetVersions(),
+  });
+
   const [initialData, setInitialData] = useState<Partial<DataModel>>({
     modelName: modelMetadata?.modelName,
     datasetId: modelMetadata?.connectedDsId,
@@ -85,6 +91,22 @@ const ConfigureDataModel: FC = () => {
     name: keyof DataModel,
     value: any
   ) => {
+    // Update version when dataset is changed
+    if (name === 'datasetId' && value && datasetVersions) {
+      const selectedDataset = datasetVersions.find(
+        (dataset: any) => dataset.id.toString() === value
+      );
+      if (selectedDataset?.version) {
+        setDataModel((prevDataModel) => ({
+          ...prevDataModel,
+          [name]: value,
+          version: selectedDataset.version,
+        }));
+        return; // Early return to avoid the second setDataModel call
+      }
+    }
+
+    // Default case - just update the field
     setDataModel((prevDataModel) => ({
       ...prevDataModel,
       [name]: value,
@@ -185,17 +207,24 @@ const ConfigureDataModel: FC = () => {
     return undefined;
   };
 
-  const buildUpdatedPayload = (updateType: string | undefined) => ({
-    modelGroupKey: modelMetadata.modelGroupKey ?? "",
-    modelName: dataModel.modelName ?? "",
-    connectedDsId: Number(dataModel.datasetId) ?? 0,
-    deploymentEnv: dataModel.deploymentEnvironment ?? "",
-    baseModels: dataModel.baseModels ?? [],
-    connectedDsMajorVersion: Number(dataModel.version?.split('.')[0]?.[1]) ?? 0,
-    connectedDsMinorVersion: Number(dataModel.version?.split('.')[1]) ?? 0,
-    updateType: updateType ?? "",
-    isTrainingNeeded: !areArraysEqual(initialData.baseModels as string[], dataModel.baseModels as string[])
-  });
+  const buildUpdatedPayload = (updateType: string | undefined) => {
+    // Parse version correctly - version format is "V1.0"
+    const versionParts = dataModel.version?.split('.');
+    const majorVersion = versionParts?.[0]?.substring(1); // Remove 'V' prefix
+    const minorVersion = versionParts?.[1];
+
+    return {
+      modelGroupKey: modelMetadata.modelGroupKey ?? "",
+      modelName: dataModel.modelName ?? "",
+      connectedDsId: Number(dataModel.datasetId) ?? 0,
+      deploymentEnv: dataModel.deploymentEnvironment ?? "",
+      baseModels: dataModel.baseModels ?? [],
+      connectedDsMajorVersion: Number(majorVersion) ?? 1,
+      connectedDsMinorVersion: Number(minorVersion) ?? 0,
+      updateType: updateType ?? "",
+      isTrainingNeeded: !areArraysEqual(initialData.baseModels as string[], dataModel.baseModels as string[])
+    };
+  };
 
   const deleteDataModelMutation = useMutation({
     mutationFn: deleteDataModel,
@@ -288,6 +317,7 @@ const ConfigureDataModel: FC = () => {
             }
             handleChange={handleDataModelAttributesChange}
             type="configure"
+            datasetVersions={datasetVersions}
           />
         )}
       </div>
diff --git a/GUI/src/pages/DataModels/CreateDataModel.tsx b/GUI/src/pages/DataModels/CreateDataModel.tsx
@@ -12,9 +12,9 @@ import {
   DataModel,
   ErrorsType,
 } from 'types/dataModels';
-import { da } from 'date-fns/locale';
 import { createDataModel, getProductionDataModel } from 'services/datamodels';
 import { dataModelsQueryKeys } from 'utils/queryKeys';
+import { getAllDatasetVersions } from 'services/datasets';
 
 const CreateDataModel: FC = () => {
   const { t } = useTranslation();
@@ -34,7 +34,35 @@ const CreateDataModel: FC = () => {
     queryFn: () => getProductionDataModel(),
   });
 
+  const { data: datasetVersions } = useQuery({
+    queryKey: dataModelsQueryKeys.DATA_MODEL_DEPLOYMENT_ENVIRONMENTS(),
+    queryFn: () => getAllDatasetVersions(),
+  });
+
   const handleDataModelAttributesChange = (name: string, value: string) => {
+    // Update version when dataset is changed
+    if (name === 'datasetId' && value && datasetVersions) {
+      const selectedDataset = datasetVersions.find(
+        (dataset: any) => dataset.id.toString() === value
+      );
+      if (selectedDataset?.version) {
+        setDataModel((prevDataModel) => ({
+          ...prevDataModel,
+          [name]: value,
+          version: selectedDataset.version,
+        }));
+        
+        // Clear datasetId error
+        setErrors((prevErrors) => {
+          const updatedErrors = { ...prevErrors };
+          delete updatedErrors.datasetId;
+          return updatedErrors;
+        });
+        return; // Early return to avoid the second setDataModel call
+      }
+    }
+
+    // Default case - just update the field
     setDataModel((prevFilters) => ({
       ...prevFilters,
       [name]: value,
@@ -86,14 +114,18 @@ const CreateDataModel: FC = () => {
   });
 
   const handleCreate = () => {
+    // Parse version correctly - version format is "V1.0"
+    const versionParts = dataModel?.version?.split('.');
+    const majorVersion = versionParts?.[0]?.substring(1); // Remove 'V' prefix
+    const minorVersion = versionParts?.[1];
 
     const paylod = {
       modelName: dataModel.modelName ?? "",
       deploymentEnv: dataModel.deploymentEnvironment ?? "",
       baseModels: dataModel.baseModels ?? [],
       connectedDsId: Number(dataModel.datasetId) ?? 0,
-      connectedDsMajorVersion: Number(dataModel?.version?.split('.')[0]?.[1]) ?? "",
-      connectedDsMinorVersion: Number(dataModel?.version?.split('.')[1]) ?? "",
+      connectedDsMajorVersion: Number(majorVersion) ?? 1,
+      connectedDsMinorVersion: Number(minorVersion) ?? 0,
     }
 
     if (prodDataModel && dataModel.deploymentEnvironment === "production") {
@@ -134,6 +166,7 @@ const CreateDataModel: FC = () => {
           dataModel={dataModel}
           handleChange={handleDataModelAttributesChange}
           type="create"
+          datasetVersions={datasetVersions}
         />
       </div>
       <div className="flex data-model-buttons">
diff --git a/docker-compose-ec2.yml b/docker-compose-ec2.yml
diff --git a/src/s3_dataset_processor/dataset_generation_callback_processor.py b/src/s3_dataset_processor/dataset_generation_callback_processor.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+-- Insert new agency into mock_centops table`
	`2`	`+INSERT INTO public.mock_centops (agency_id, agency_name, created_at)`
	`3`	`+VALUES (:agencyId, :agencyName, NOW());`