diff --git a/src/main/java/fr/insee/genesis/controller/dto/rawdata/ProcessingResultDto.java b/src/main/java/fr/insee/genesis/controller/dto/rawdata/ProcessingResultDto.java new file mode 100644 index 00000000..f5aa34c2 --- /dev/null +++ b/src/main/java/fr/insee/genesis/controller/dto/rawdata/ProcessingResultDto.java @@ -0,0 +1,4 @@ +package fr.insee.genesis.controller.dto.rawdata; + +public record ProcessingResultDto(int dataCount, int formattedDataCount) { +} diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java index f809e1ec..76003243 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java @@ -126,7 +126,7 @@ public ResponseEntity processRawResponsesByCollectionInstrumentId( @PathVariable("collectionInstrumentId") String collectionInstrumentId ) throws GenesisException{ log.info("Try to process raw responses for collectionInstrumentId {}", collectionInstrumentId); - DataProcessResult result = rawResponseApiPort.processRawResponsesByInterrogationIds(collectionInstrumentId); + DataProcessResult result = rawResponseApiPort.processRawResponsesByCollectionInstrumentId(collectionInstrumentId); return ResponseEntity.ok(result.message(collectionInstrumentId)); } diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java new file mode 100644 index 00000000..1504e84f --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java @@ -0,0 +1,216 @@ +package fr.insee.genesis.domain.converter.rawdata; + +import fr.insee.bpm.metadata.model.VariablesMap; +import fr.insee.genesis.Constants; +import fr.insee.genesis.domain.model.surveyunit.DataState; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.model.surveyunit.VariableModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; +import fr.insee.genesis.domain.utils.GroupUtils; +import fr.insee.genesis.domain.utils.JsonUtils; +import fr.insee.genesis.domain.parser.rawdata.LunaticJsonRawDataPayloadParser; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@Component +@Slf4j +@RequiredArgsConstructor +public class LunaticJsonRawDataConverter { + + private final LunaticJsonRawDataPayloadParser payloadParser; + + public List convertRawData( + List rawDataList, + VariablesMap variablesMap + ) { + return convertRawDataAndCollectEmptyModels(rawDataList, variablesMap, new ArrayList<>()); + } + + public List convertRawDataAndCollectEmptyModels( + List rawDataList, + VariablesMap variablesMap, + List emptySurveyUnitModels + ) { + List surveyUnitModels = new ArrayList<>(); + + for (DataState dataState : List.of(DataState.COLLECTED, DataState.EDITED)) { + for (LunaticJsonRawDataModel rawData : rawDataList) { + RawDataModelType rawDataModelType = getRawDataModelType(rawData); + + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(rawData.questionnaireId()) + .mode(rawData.mode()) + .interrogationId(rawData.interrogationId()) + .usualSurveyUnitId(rawData.idUE()) + .validationDate(payloadParser.getValidationDate(rawData)) + .isCapturedIndirectly(payloadParser.getIsCapturedIndirectly(rawData)) + .state(dataState) + .fileDate(rawData.recordDate()) + .recordDate(Instant.now()) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + convertRawDataCollectedVariables(rawData, surveyUnitModel, dataState, rawDataModelType, variablesMap); + + if (dataState == DataState.COLLECTED) { + convertRawDataExternalVariables(rawData, surveyUnitModel, rawDataModelType, variablesMap); + } + + boolean hasNoVariable = surveyUnitModel.getCollectedVariables().isEmpty() + && surveyUnitModel.getExternalVariables().isEmpty(); + + if (hasNoVariable) { + if (surveyUnitModel.getState() == DataState.COLLECTED) { + log.warn("No collected or external variable for interrogation {}, raw data is ignored.", + rawData.interrogationId()); + } + emptySurveyUnitModels.add(surveyUnitModel); + continue; + } + + surveyUnitModels.add(surveyUnitModel); + } + } + + return surveyUnitModels; + } + + private static RawDataModelType getRawDataModelType(LunaticJsonRawDataModel rawData) { + return rawData.data().containsKey("data") + ? RawDataModelType.FILIERE + : RawDataModelType.LEGACY; + } + + @SuppressWarnings("unchecked") + private void convertRawDataCollectedVariables( + LunaticJsonRawDataModel srcRawData, + SurveyUnitModel dstSurveyUnitModel, + DataState dataState, + RawDataModelType rawDataModelType, + VariablesMap variablesMap + ) { + Map dataMap = srcRawData.data(); + if (rawDataModelType == RawDataModelType.FILIERE) { + dataMap = (Map) dataMap.get("data"); + } + + dataMap = (Map) dataMap.get("COLLECTED"); + + Map collectedMap = JsonUtils.asMap(dataMap); + if (collectedMap == null || collectedMap.isEmpty()) { + if (dataState == DataState.COLLECTED) { + log.warn("No collected data for interrogation {}", srcRawData.interrogationId()); + } + return; + } + + String stateKey = dataState.toString(); + List destination = dstSurveyUnitModel.getCollectedVariables(); + + for (Map.Entry collectedVariable : collectedMap.entrySet()) { + RawResponseConverter.processCollectedVariable( + collectedVariable, + stateKey, + variablesMap, + dstSurveyUnitModel, + destination + ); + } + } + + @SuppressWarnings("unchecked") + private static void convertRawDataExternalVariables( + LunaticJsonRawDataModel srcRawData, + SurveyUnitModel dstSurveyUnitModel, + RawDataModelType rawDataModelType, + VariablesMap variablesMap + ) { + Map dataMap = srcRawData.data(); + if (rawDataModelType == RawDataModelType.FILIERE) { + dataMap = (Map) dataMap.get("data"); + } + + dataMap = (Map) dataMap.get("EXTERNAL"); + Map externalMap = JsonUtils.asMap(dataMap); + if (externalMap != null && !externalMap.isEmpty()) { + convertToExternalVar(dstSurveyUnitModel, variablesMap, externalMap); + } + } + + private static void convertToExternalVar( + SurveyUnitModel dstSurveyUnitModel, + VariablesMap variablesMap, + Map externalMap + ) { + for (Map.Entry externalVariableEntry : externalMap.entrySet()) { + Object valueObject = externalVariableEntry.getValue(); + + if (valueObject instanceof List) { + convertListVar(valueObject, externalVariableEntry, variablesMap, dstSurveyUnitModel.getExternalVariables()); + continue; + } + + if (valueObject != null) { + convertOneVar( + externalVariableEntry, + valueObject.toString(), + variablesMap, + 1, + dstSurveyUnitModel.getExternalVariables() + ); + } + } + } + + private static void convertListVar( + Object valuesForState, + Map.Entry collectedVariable, + VariablesMap variablesMap, + List destination + ) { + List values = JsonUtils.asStringList(valuesForState); + if (!values.isEmpty()) { + int iteration = 1; + for (String value : values) { + if (value != null && !value.isEmpty()) { + convertOneVar(collectedVariable, value, variablesMap, iteration, destination); + } + iteration++; + } + } + } + + private static void convertOneVar( + Map.Entry variableEntry, + String value, + VariablesMap variablesMap, + int iteration, + List destination + ) { + VariableModel variableModel = VariableModel.builder() + .varId(variableEntry.getKey()) + .value(value) + .scope(getIdLoop(variablesMap, variableEntry.getKey())) + .iteration(iteration) + .parentId(GroupUtils.getParentGroupName(variableEntry.getKey(), variablesMap)) + .build(); + + destination.add(variableModel); + } + + private static String getIdLoop(VariablesMap variablesMap, String variableName) { + if (variablesMap.getVariable(variableName) == null) { + log.warn("Variable {} not present in metadatas, assigning to {}", variableName, Constants.ROOT_GROUP_NAME); + return Constants.ROOT_GROUP_NAME; + } + return variablesMap.getVariable(variableName).getGroupName(); + } +} diff --git a/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseConverter.java b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseConverter.java new file mode 100644 index 00000000..28981448 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseConverter.java @@ -0,0 +1,304 @@ +package fr.insee.genesis.domain.converter.rawdata; + +import fr.insee.bpm.metadata.model.VariablesMap; +import fr.insee.genesis.Constants; +import fr.insee.genesis.domain.model.surveyunit.DataState; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.model.surveyunit.VariableModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; +import fr.insee.genesis.domain.utils.GroupUtils; +import fr.insee.genesis.domain.utils.JsonUtils; +import fr.insee.genesis.domain.parser.rawdata.RawResponsePayloadParser; +import fr.insee.modelefiliere.RawResponseDto; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService.getValueString; + +@Component +@Slf4j +@RequiredArgsConstructor +public class RawResponseConverter { + + private final RawResponsePayloadParser rawResponsePayloadParser; + + public List convertRawResponse( + List rawResponseModels, + VariablesMap variablesMap + ) { + return convertRawResponseAndCollectEmptyModels(rawResponseModels, variablesMap, new ArrayList<>()); + } + + public List convertRawResponseAndCollectEmptyModels( + List rawResponseModels, + VariablesMap variablesMap, + List emptySurveyUnitModels + ) { + List surveyUnitModels = new ArrayList<>(); + + for (DataState dataState : List.of(DataState.COLLECTED, DataState.EDITED)) { + for (RawResponseModel rawResponseModel : rawResponseModels) { + SurveyUnitModel surveyUnitModel = buildSurveyUnitModel(rawResponseModel, dataState); + + convertCollectedVariables(rawResponseModel, surveyUnitModel, dataState, variablesMap); + + if (dataState == DataState.COLLECTED) { + convertExternalVariables(rawResponseModel, surveyUnitModel, variablesMap); + } + + boolean hasNoVariable = surveyUnitModel.getCollectedVariables().isEmpty() + && surveyUnitModel.getExternalVariables().isEmpty(); + + if (hasNoVariable) { + if (surveyUnitModel.getState() == DataState.COLLECTED) { + log.warn( + "No collected or external variable for interrogation {}, raw data is ignored.", + rawResponseModel.interrogationId() + ); + } + emptySurveyUnitModels.add(surveyUnitModel); + continue; + } + + surveyUnitModels.add(surveyUnitModel); + } + } + + return surveyUnitModels; + } + + private SurveyUnitModel buildSurveyUnitModel(RawResponseModel rawResponseModel, DataState dataState) { + String questionnaireStateString = + rawResponsePayloadParser.getStringField(rawResponseModel, "questionnaireState"); + + RawResponseDto.QuestionnaireStateEnum questionnaireStateEnum = null; + try { + questionnaireStateEnum = RawResponseDto.QuestionnaireStateEnum.valueOf(questionnaireStateString); + } catch (IllegalArgumentException _) { + log.warn("'{}' is not a valid questionnaire state according to filiere model", questionnaireStateString); + } catch (NullPointerException _) { + } + + return SurveyUnitModel.builder() + .collectionInstrumentId(rawResponseModel.collectionInstrumentId()) + .majorModelVersion(rawResponsePayloadParser.getStringField(rawResponseModel, "majorModelVersion")) + .mode(rawResponseModel.mode()) + .interrogationId(rawResponseModel.interrogationId()) + .usualSurveyUnitId(rawResponsePayloadParser.getStringField(rawResponseModel, "usualSurveyUnitId")) + .questionnaireState(questionnaireStateEnum) + .validationDate(rawResponsePayloadParser.getValidationDate(rawResponseModel)) + .isCapturedIndirectly(rawResponsePayloadParser.getIsCapturedIndirectly(rawResponseModel)) + .state(dataState) + .fileDate(rawResponseModel.recordDate()) + .recordDate(Instant.now()) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + } + + @SuppressWarnings("unchecked") + private void convertCollectedVariables( + RawResponseModel rawResponseModel, + SurveyUnitModel dstSurveyUnitModel, + DataState dataState, + VariablesMap variablesMap + ) { + Map dataMap = rawResponseModel.payload(); + dataMap = (Map) dataMap.get("data"); + dataMap = (Map) dataMap.get("COLLECTED"); + + Map collectedMap = JsonUtils.asMap(dataMap); + if (collectedMap == null || collectedMap.isEmpty()) { + if (dataState == DataState.COLLECTED) { + log.warn("No collected data for interrogation {}", rawResponseModel.interrogationId()); + } + return; + } + + String stateKey = dataState.toString(); + List collectedVariables = dstSurveyUnitModel.getCollectedVariables(); + + for (Map.Entry entry : collectedMap.entrySet()) { + processCollectedVariable(entry, stateKey, variablesMap, dstSurveyUnitModel, collectedVariables); + } + } + + public static void processCollectedVariable( + Map.Entry entry, + String stateKey, + VariablesMap variablesMap, + SurveyUnitModel dstSurveyUnitModel, + List variableModelList + ) { + if (Constants.PAIRWISES.equals(entry.getKey())) { + handlePairwiseCollectedVariable(entry, DataState.valueOf(stateKey), variablesMap, dstSurveyUnitModel); + return; + } + + Map states = JsonUtils.asMap(entry.getValue()); + if (states == null) { + return; + } + + Object value = states.get(stateKey); + if (value == null) { + return; + } + + if (value instanceof List list) { + convertListVar(list, entry, variablesMap, variableModelList); + } else { + convertOneVar(entry, getValueString(value), variablesMap, 1, variableModelList); + } + } + + private static void convertListVar( + Object valuesForState, + Map.Entry variableEntry, + VariablesMap variablesMap, + List destination + ) { + List values = JsonUtils.asStringList(valuesForState); + + if (!values.isEmpty()) { + int iteration = 1; + for (String value : values) { + if (value != null && !value.isEmpty()) { + convertOneVar(variableEntry, value, variablesMap, iteration, destination); + } + iteration++; + } + } + } + + private static void convertOneVar( + Map.Entry variableEntry, + String value, + VariablesMap variablesMap, + int iteration, + List destination + ) { + VariableModel variableModel = VariableModel.builder() + .varId(variableEntry.getKey()) + .value(value) + .scope(getIdLoop(variablesMap, variableEntry.getKey())) + .iteration(iteration) + .parentId(GroupUtils.getParentGroupName(variableEntry.getKey(), variablesMap)) + .build(); + + destination.add(variableModel); + } + + private static String getIdLoop(VariablesMap variablesMap, String variableName) { + if (variablesMap.getVariable(variableName) == null) { + log.warn("Variable {} not present in metadata, assigning to {}", variableName, Constants.ROOT_GROUP_NAME); + return Constants.ROOT_GROUP_NAME; + } + return variablesMap.getVariable(variableName).getGroupName(); + } + + @SuppressWarnings("unchecked") + private void convertExternalVariables( + RawResponseModel rawResponseModel, + SurveyUnitModel dstSurveyUnitModel, + VariablesMap variablesMap + ) { + Map dataMap = rawResponseModel.payload(); + dataMap = (Map) dataMap.get("data"); + dataMap = (Map) dataMap.get("EXTERNAL"); + + Map externalMap = JsonUtils.asMap(dataMap); + if (externalMap != null && !externalMap.isEmpty()) { + for (Map.Entry externalVariableEntry : externalMap.entrySet()) { + Object valueObject = externalVariableEntry.getValue(); + + if (valueObject instanceof List) { + convertListVar( + valueObject, + externalVariableEntry, + variablesMap, + dstSurveyUnitModel.getExternalVariables() + ); + continue; + } + + if (valueObject != null) { + convertOneVar( + externalVariableEntry, + valueObject.toString(), + variablesMap, + 1, + dstSurveyUnitModel.getExternalVariables() + ); + } + } + } + } + + @SuppressWarnings("unchecked") + static void handlePairwiseCollectedVariable( + Map.Entry collectedVariable, + DataState dataState, + VariablesMap variablesMap, + SurveyUnitModel dstSurveyUnitModel + ) { + Object value = getValueForState(collectedVariable, dataState.toString()); + + if (isInvalidPairwiseVariable(value, variablesMap)) { + return; + } + + List individuals = (List) value; + String groupName = variablesMap.getVariable(Constants.PAIRWISE_PREFIX + 1).getGroupName(); + + for (int individualIndex = 0; individualIndex < individuals.size(); individualIndex++) { + List individualLinks = (List) individuals.get(individualIndex); + + for (int linkIndex = 1; linkIndex < Constants.MAX_LINKS_ALLOWED; linkIndex++) { + dstSurveyUnitModel.getCollectedVariables().add( + buildPairwiseVariable(individualLinks, linkIndex, individualIndex + 1, groupName) + ); + } + } + } + + private static VariableModel buildPairwiseVariable( + List individualLinks, + int linkIndex, + int iteration, + String groupName + ) { + String value = Constants.NO_PAIRWISE_VALUE; + + if (linkIndex <= individualLinks.size()) { + String v = individualLinks.get(linkIndex - 1); + value = (v == null || v.isBlank()) ? Constants.SAME_AXIS_VALUE : v; + } + + return VariableModel.builder() + .varId(Constants.PAIRWISE_PREFIX + linkIndex) + .value(value) + .scope(groupName) + .iteration(iteration) + .parentId(Constants.ROOT_GROUP_NAME) + .build(); + } + + private static Object getValueForState( + Map.Entry collectedVariable, + String stateKey + ) { + Map states = JsonUtils.asMap(collectedVariable.getValue()); + return states != null ? states.get(stateKey) : null; + } + + private static boolean isInvalidPairwiseVariable(Object value, VariablesMap variablesMap) { + return !(value instanceof List) || !variablesMap.hasVariable(Constants.PAIRWISE_PREFIX + 1); + } +} diff --git a/src/main/java/fr/insee/genesis/domain/parser/rawdata/LunaticJsonRawDataPayloadParser.java b/src/main/java/fr/insee/genesis/domain/parser/rawdata/LunaticJsonRawDataPayloadParser.java new file mode 100644 index 00000000..ab2d3fb9 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/parser/rawdata/LunaticJsonRawDataPayloadParser.java @@ -0,0 +1,38 @@ +package fr.insee.genesis.domain.parser.rawdata; + +import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +@Component +@Slf4j +public class LunaticJsonRawDataPayloadParser { + + public LocalDateTime getValidationDate(LunaticJsonRawDataModel rawData) { + try { + return rawData.data().get("validationDate") == null + ? null + : LocalDateTime.parse( + rawData.data().get("validationDate").toString(), + DateTimeFormatter.ISO_OFFSET_DATE_TIME + ); + } catch (Exception e) { + log.warn("Exception when parsing validation date : {}", e.toString()); + return null; + } + } + + public Boolean getIsCapturedIndirectly(LunaticJsonRawDataModel rawData) { + try { + return rawData.data().get("isCapturedIndirectly") == null + ? null + : Boolean.parseBoolean(rawData.data().get("isCapturedIndirectly").toString()); + } catch (Exception e) { + log.warn("Exception when parsing isCapturedIndirectly : {}", e.toString()); + return Boolean.FALSE; + } + } +} \ No newline at end of file diff --git a/src/main/java/fr/insee/genesis/domain/parser/rawdata/RawResponsePayloadParser.java b/src/main/java/fr/insee/genesis/domain/parser/rawdata/RawResponsePayloadParser.java new file mode 100644 index 00000000..64e2cb9c --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/parser/rawdata/RawResponsePayloadParser.java @@ -0,0 +1,47 @@ +package fr.insee.genesis.domain.parser.rawdata; + +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +@Component +@Slf4j +public class RawResponsePayloadParser { + + public Boolean getIsCapturedIndirectly(RawResponseModel rawResponseModel) { + try { + return rawResponseModel.payload().get("isCapturedIndirectly") == null + ? null + : Boolean.parseBoolean(rawResponseModel.payload().get("isCapturedIndirectly").toString()); + } catch (Exception e) { + log.warn("Exception when parsing isCapturedIndirectly : {}", e.toString()); + return Boolean.FALSE; + } + } + + public LocalDateTime getValidationDate(RawResponseModel rawResponseModel) { + try { + return rawResponseModel.payload().get("validationDate") == null + ? null + : LocalDateTime.parse( + rawResponseModel.payload().get("validationDate").toString(), + DateTimeFormatter.ISO_OFFSET_DATE_TIME + ); + } catch (Exception e) { + log.warn("Exception when parsing validation date : {}", e.toString()); + return null; + } + } + + public String getStringField(RawResponseModel rawResponseModel, String field) { + try { + return rawResponseModel.payload().get(field).toString(); + } catch (Exception e) { + log.warn("Exception when parsing {} : {}", field, e.toString()); + return null; + } + } +} diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java index 7771031d..6faaff06 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java @@ -19,9 +19,8 @@ public interface RawResponseApiPort { List getRawResponses(String collectionInstrumentId, Mode mode, List interrogationIdList); List getRawResponsesByInterrogationID(String interrogationId); DataProcessResult processRawResponsesByInterrogationIds(String collectionInstrumentId, List interrogationIdList, List errors) throws GenesisException; - DataProcessResult processRawResponsesByInterrogationIds(String collectionInstrumentId) throws GenesisException; + DataProcessResult processRawResponsesByCollectionInstrumentId(String collectionInstrumentId) throws GenesisException; - List convertRawResponse(List rawResponses, VariablesMap variablesMap); List getUnprocessedCollectionInstrumentIds(); void updateProcessDates(List surveyUnitModels); Page findRawResponseDataByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable); diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index 7cf627cf..69f13e8f 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -2,99 +2,68 @@ import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.Constants; import fr.insee.genesis.configuration.Config; import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; import fr.insee.genesis.controller.utils.ControllerUtils; +import fr.insee.genesis.domain.converter.rawdata.LunaticJsonRawDataConverter; import fr.insee.genesis.domain.model.context.DataProcessingContextModel; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.GroupedInterrogation; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; -import fr.insee.genesis.domain.model.surveyunit.VariableModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; -import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; import fr.insee.genesis.domain.ports.spi.DataProcessingContextPersistancePort; import fr.insee.genesis.domain.ports.spi.LunaticJsonRawDataPersistencePort; -import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; -import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityToolService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; -import fr.insee.genesis.domain.utils.GroupUtils; -import fr.insee.genesis.domain.utils.JsonUtils; +import fr.insee.genesis.controller.dto.rawdata.ProcessingResultDto; import fr.insee.genesis.exceptions.GenesisError; import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.infrastructure.utils.FileUtils; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; -import java.io.IOException; import java.math.BigDecimal; import java.time.Instant; import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import static fr.insee.genesis.domain.service.rawdata.RawResponseService.processCollectedVariable; @Service @Slf4j +@RequiredArgsConstructor public class LunaticJsonRawDataService implements LunaticJsonRawDataApiPort { private final ControllerUtils controllerUtils; private final QuestionnaireMetadataService metadataService; private final SurveyUnitService surveyUnitService; private final SurveyUnitQualityService surveyUnitQualityService; - private final SurveyUnitQualityToolPort surveyUnitQualityToolPort; - private final DataProcessingContextService dataProcessingContextService; + private final SurveyUnitQualityToolService surveyUnitQualityToolService; private final FileUtils fileUtils; private final Config config; + private final LunaticJsonRawDataConverter lunaticJsonRawDataConverter; @Qualifier("lunaticJsonMongoAdapterNew") private final LunaticJsonRawDataPersistencePort lunaticJsonRawDataPersistencePort; + @Qualifier("dataProcessingContextMongoAdapter") private final DataProcessingContextPersistancePort dataProcessingContextPersistancePort; - @Autowired - public LunaticJsonRawDataService(LunaticJsonRawDataPersistencePort lunaticJsonRawDataNewPersistencePort, - ControllerUtils controllerUtils, - QuestionnaireMetadataService metadataService, - SurveyUnitService surveyUnitService, - SurveyUnitQualityService surveyUnitQualityService, - FileUtils fileUtils, - DataProcessingContextService dataProcessingContextService, - SurveyUnitQualityToolPort surveyUnitQualityToolPort, - Config config, - DataProcessingContextPersistancePort dataProcessingContextPersistancePort - ) { - this.controllerUtils = controllerUtils; - this.metadataService = metadataService; - this.surveyUnitService = surveyUnitService; - this.surveyUnitQualityService = surveyUnitQualityService; - this.fileUtils = fileUtils; - this.lunaticJsonRawDataPersistencePort = lunaticJsonRawDataNewPersistencePort; - this.dataProcessingContextPersistancePort = dataProcessingContextPersistancePort; - this.surveyUnitQualityToolPort = surveyUnitQualityToolPort; - this.dataProcessingContextService = dataProcessingContextService; - this.config = config; - } - @Override public void save(LunaticJsonRawDataModel rawData) throws GenesisException { try { @@ -107,254 +76,179 @@ public void save(LunaticJsonRawDataModel rawData) throws GenesisException { } @Override - public List getRawDataByQuestionnaireId(String questionnaireId, Mode mode, List interrogationIdList) { - return lunaticJsonRawDataPersistencePort.findRawDataByQuestionnaireId(questionnaireId, mode, interrogationIdList); + public List getRawDataByQuestionnaireId( + String questionnaireId, + Mode mode, + List interrogationIdList + ) { + return lunaticJsonRawDataPersistencePort.findRawDataByQuestionnaireId( + questionnaireId, + mode, + interrogationIdList + ); } - @Override + @Override public List getRawDataByInterrogationId(String interrogationId) { return lunaticJsonRawDataPersistencePort.findRawDataByInterrogationId(interrogationId); } @Override - public DataProcessResult processRawDataByInterrogationIds(String questionnaireId, List interrogationIdList, List errors) throws GenesisException { - int dataCount=0; - int formattedDataCount=0; - DataProcessingContextModel dataProcessingContext = - dataProcessingContextService.getContextByCollectionInstrumentId(questionnaireId); - List modesList = controllerUtils.getModesList(questionnaireId, null); - for (Mode mode : modesList) { - //Load and save metadata into database, throw exception if none - VariablesMap variablesMap = getVariablesMap(questionnaireId, mode, errors); - int totalBatchs = Math.ceilDiv(interrogationIdList.size() , config.getRawDataProcessingBatchSize()); - int batchNumber = 1; - List interrogationIdListForMode = new ArrayList<>(interrogationIdList); - while(!interrogationIdListForMode.isEmpty()){ - log.info("Processing raw data batch {}/{}", batchNumber, totalBatchs); - int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize()); - List interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex); + public DataProcessResult processRawData(String questionnaireId) throws GenesisException { + List interrogationIds = lunaticJsonRawDataPersistencePort + .findUnprocessedInterrogationIdsByCollectionInstrumentId(questionnaireId) + .stream() + .toList(); - List rawData = getRawDataByQuestionnaireId(questionnaireId, mode, interrogationIdToProcess); + return processRawDataByInterrogationIds(questionnaireId, interrogationIds, new ArrayList<>()); + } - List surveyUnitModels = convertRawData( - rawData, - variablesMap - ); + @Override + public DataProcessResult processRawDataByInterrogationIds( + String questionnaireId, + List interrogationIdList, + List errors + ) throws GenesisException { - //Save converted data - surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap); - surveyUnitService.saveSurveyUnits(surveyUnitModels); + List modes = controllerUtils.getModesList(questionnaireId, null); + boolean shouldUseQualityTool = + surveyUnitQualityToolService.resolveWithReviewValue(questionnaireId); - //Update process dates - updateProcessDates(surveyUnitModels); + int batchSize = config.getRawDataProcessingBatchSize(); + int dataCount = 0; + int formattedDataCount = 0; - //Increment data count - dataCount += surveyUnitModels.size(); - formattedDataCount += surveyUnitModels.stream() - .filter(surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED)) - .toList() - .size(); + for (Mode mode : modes) { + VariablesMap variablesMap = getVariablesMap(questionnaireId, mode, errors); + List remainingInterrogationIds = new ArrayList<>(interrogationIdList); + int totalBatches = Math.ceilDiv(remainingInterrogationIds.size(), batchSize); + int batchNumber = 1; - //Send processed ids grouped by questionnaire (if review activated) - if(dataProcessingContext != null && dataProcessingContext.isWithReview()) { - sendProcessedIdsToQualityTool(surveyUnitModels); - } + while (!remainingInterrogationIds.isEmpty()) { + log.info( + "Processing raw data batch {}/{} for questionnaireId={} mode={}", + batchNumber, + totalBatches, + questionnaireId, + mode + ); - //Remove processed ids from list - interrogationIdListForMode = interrogationIdListForMode.subList(maxIndex, interrogationIdListForMode.size()); + int maxIndex = Math.min(remainingInterrogationIds.size(), batchSize); + List interrogationIdsToProcess = remainingInterrogationIds.subList(0, maxIndex); + ProcessingResultDto processingResultDto = processRawDataForMode( + questionnaireId, + mode, + interrogationIdsToProcess, + variablesMap, + shouldUseQualityTool + ); + + dataCount += processingResultDto.dataCount(); + formattedDataCount += processingResultDto.formattedDataCount(); + + remainingInterrogationIds = + remainingInterrogationIds.subList(maxIndex, remainingInterrogationIds.size()); batchNumber++; } } + return new DataProcessResult(dataCount, formattedDataCount, errors); } - @Override - public DataProcessResult processRawData(String questionnaireId) throws GenesisException { - int dataCount=0; - int formattedDataCount=0; - DataProcessingContextModel dataProcessingContext = - dataProcessingContextService.getContextByCollectionInstrumentId(questionnaireId); - List errors = new ArrayList<>(); - - List modesList = controllerUtils.getModesList(questionnaireId, null); - for (Mode mode : modesList) { - //Load and save metadata into database, throw exception if none - VariablesMap variablesMap = getVariablesMap(questionnaireId, mode, errors); - Set interrogationIds = - lunaticJsonRawDataPersistencePort.findUnprocessedInterrogationIdsByCollectionInstrumentId(questionnaireId); - - int totalBatchs = Math.ceilDiv(interrogationIds.size() , config.getRawDataProcessingBatchSize()); - int batchNumber = 1; - List interrogationIdListForMode = new ArrayList<>(interrogationIds); - while(!interrogationIdListForMode.isEmpty()){ - log.info("Processing raw data batch {}/{}", batchNumber, totalBatchs); - - int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize()); - List surveyUnitModels = getConvertedSurveyUnits( + private ProcessingResultDto processRawDataForMode( + String questionnaireId, + Mode mode, + List interrogationIds, + VariablesMap variablesMap, + boolean shouldUseQualityTool + ) { + List rawData = + lunaticJsonRawDataPersistencePort.findRawDataByQuestionnaireId( questionnaireId, mode, - interrogationIdListForMode, - maxIndex, - variablesMap + interrogationIds ); - //Save converted data - surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap); - surveyUnitService.saveSurveyUnits(surveyUnitModels); + List emptySurveyUnitModels = new ArrayList<>(); + List surveyUnitModels = + lunaticJsonRawDataConverter.convertRawDataAndCollectEmptyModels( + rawData, + variablesMap, + emptySurveyUnitModels + ); - //Update process dates - updateProcessDates(surveyUnitModels); + surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap); + surveyUnitService.saveSurveyUnits(surveyUnitModels); - //Increment data count - dataCount += surveyUnitModels.size(); - formattedDataCount += surveyUnitModels.stream() - .filter(surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED)) - .toList() - .size(); + updateProcessDates(surveyUnitModels); - //Send processed ids grouped by questionnaire (if review activated) - if(dataProcessingContext != null && dataProcessingContext.isWithReview()) { - sendProcessedIdsToQualityTool(surveyUnitModels); - } + if (!emptySurveyUnitModels.isEmpty()) { + updateProcessDates(emptySurveyUnitModels); + } - //Remove processed ids from list - interrogationIdListForMode = interrogationIdListForMode.subList(maxIndex, interrogationIdListForMode.size()); - batchNumber++; - } + if (shouldUseQualityTool) { + surveyUnitQualityToolService.sendProcessedIdsToQualityTool(surveyUnitModels); } - return new DataProcessResult(dataCount, formattedDataCount, errors); + + int formattedDataCount = (int) surveyUnitModels.stream() + .filter(surveyUnitModel -> surveyUnitModel.getState() == DataState.FORMATTED) + .count(); + + return new ProcessingResultDto(surveyUnitModels.size(), formattedDataCount); } + private VariablesMap getVariablesMap( + String questionnaireId, + Mode mode, + List errors + ) throws GenesisException { + VariablesMap variablesMap = metadataService.loadAndSaveIfNotExists( + questionnaireId, + questionnaireId, + mode, + fileUtils, + errors + ).getVariables(); - private VariablesMap getVariablesMap(String questionnaireId, Mode mode, List errors) throws GenesisException { - VariablesMap variablesMap = metadataService.loadAndSaveIfNotExists(questionnaireId, questionnaireId, mode, fileUtils, - errors).getVariables(); if (variablesMap == null) { - throw new GenesisException(HttpStatus.BAD_REQUEST, + throw new GenesisException( + HttpStatus.BAD_REQUEST, "Error during metadata parsing for mode %s :%n%s" .formatted(mode, errors.getLast().getMessage()) ); } - return variablesMap; - } - private List getConvertedSurveyUnits(String questionnaireId, Mode mode, List interrogationIdListForMode, int maxIndex, VariablesMap variablesMap) { - List interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex); - List rawData = getRawDataByQuestionnaireId(questionnaireId, mode, interrogationIdToProcess); - return convertRawData( - rawData, - variablesMap - ); - } - - private void sendProcessedIdsToQualityTool(List surveyUnitModels) { - try { - ResponseEntity response = - surveyUnitQualityToolPort.sendProcessedIds(getProcessedIdsMap(surveyUnitModels)); - - if (response.getStatusCode().is2xxSuccessful()) { - log.info("Successfully sent {} ids to quality tool", getProcessedIdsMap(surveyUnitModels).size()); - }else{ - log.warn("Survey unit quality tool responded non-2xx code {} and body {}", - response.getStatusCode(), response.getBody()); - } - }catch (IOException e){ - log.error("Error during Perret call request building : {}", e.toString()); - } + return variablesMap; } - private Map> getProcessedIdsMap(List surveyUnitModels) { - Map> processedInterrogationIdsPerQuestionnaire = new HashMap<>(); - surveyUnitModels.forEach(model -> - processedInterrogationIdsPerQuestionnaire - .computeIfAbsent(model.getCollectionInstrumentId(), k -> new HashSet<>()) - .add(model.getInterrogationId()) - ); - return processedInterrogationIdsPerQuestionnaire; + @Override + public List convertRawData( + List rawDataList, + VariablesMap variablesMap + ) { + return lunaticJsonRawDataConverter.convertRawData(rawDataList, variablesMap); } - @Override - public List convertRawData(List rawDataList, VariablesMap variablesMap) { - //Convert to genesis model - List surveyUnitModels = new ArrayList<>(); - List emptySurveyUnitModels = new ArrayList<>(); - //For each possible data state (we receive COLLECTED or EDITED) - for(DataState dataState : List.of(DataState.COLLECTED,DataState.EDITED)){ - for (LunaticJsonRawDataModel rawData : rawDataList) { - RawDataModelType rawDataModelType = getRawDataModelType(rawData); - - //Get optional fields - Boolean isCapturedIndirectly = getIsCapturedIndirectly(rawData); - LocalDateTime validationDate = getValidationDate(rawData); - - SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() - .collectionInstrumentId(rawData.questionnaireId()) - .mode(rawData.mode()) - .interrogationId(rawData.interrogationId()) - .usualSurveyUnitId(rawData.idUE()) - .validationDate(validationDate) - .isCapturedIndirectly(isCapturedIndirectly) - .state(dataState) - .fileDate(rawData.recordDate()) - .recordDate(Instant.now()) - .collectedVariables(new ArrayList<>()) - .externalVariables(new ArrayList<>()) - .build(); - - //Data collected variables conversion - convertRawDataCollectedVariables(rawData, surveyUnitModel, dataState, rawDataModelType, variablesMap); - - //External variables conversion into COLLECTED document - if(dataState == DataState.COLLECTED){ - convertRawDataExternalVariables(rawData, surveyUnitModel, rawDataModelType, variablesMap); - } - - boolean hasNoVariable = surveyUnitModel.getCollectedVariables().isEmpty() - && surveyUnitModel.getExternalVariables().isEmpty(); - - if(hasNoVariable){ - if(surveyUnitModel.getState() == DataState.COLLECTED){ - log.warn("No collected or external variable for interrogation {}, raw data is ignored.", rawData.interrogationId()); - } - emptySurveyUnitModels.add(surveyUnitModel); - continue;// don't add suModel but update processDate - } - surveyUnitModels.add(surveyUnitModel); + public void updateProcessDates(List surveyUnitModels) { + Set collectionInstrumentIds = new HashSet<>(); + for (SurveyUnitModel surveyUnitModel : surveyUnitModels) { + if (surveyUnitModel.getCollectionInstrumentId() != null) { + collectionInstrumentIds.add(surveyUnitModel.getCollectionInstrumentId()); } } - if(!emptySurveyUnitModels.isEmpty()){ - updateProcessDates(emptySurveyUnitModels); - } - return surveyUnitModels; - } - - private static RawDataModelType getRawDataModelType(LunaticJsonRawDataModel rawData) { - return rawData.data().containsKey("data") ? - RawDataModelType.FILIERE : - RawDataModelType.LEGACY; - } - private static LocalDateTime getValidationDate(LunaticJsonRawDataModel rawData) { - try{ - return rawData.data().get("validationDate") == null ? null : - LocalDateTime.parse(rawData.data().get("validationDate").toString(), DateTimeFormatter.ISO_OFFSET_DATE_TIME); - }catch(Exception e){ - log.warn("Exception when parsing validation date : {}}",e.toString()); - return null; - } - } - - private static Boolean getIsCapturedIndirectly(LunaticJsonRawDataModel rawData) { - try{ - return rawData.data().get("isCapturedIndirectly") == null ? null : - Boolean.parseBoolean(rawData.data().get("isCapturedIndirectly").toString()); - }catch(Exception e){ - log.warn("Exception when parsing isCapturedIndirectly : {}}",e.toString()); - return Boolean.FALSE; + for (String collectionInstrumentId : collectionInstrumentIds) { + Set interrogationIds = new HashSet<>(); + for (SurveyUnitModel surveyUnitModel : + surveyUnitModels.stream() + .filter(su -> su.getCollectionInstrumentId().equals(collectionInstrumentId)) + .toList()) { + interrogationIds.add(surveyUnitModel.getInterrogationId()); + } + lunaticJsonRawDataPersistencePort.updateProcessDates(collectionInstrumentId, interrogationIds); } - } @Override @@ -362,34 +256,37 @@ public List getUnprocessedDataIds() { List dtos = new ArrayList<>(); for (GroupedInterrogation groupedInterrogation : lunaticJsonRawDataPersistencePort.findUnprocessedIds()) { - for (String interrogationId : groupedInterrogation.interrogationIds()){ + for (String interrogationId : groupedInterrogation.interrogationIds()) { dtos.add(LunaticJsonRawDataUnprocessedDto.builder() .questionnaireId(groupedInterrogation.questionnaireId()) .interrogationId(interrogationId) - .build() - ); + .build()); } } + return dtos; } @Override public Set getUnprocessedDataQuestionnaireIds() { - Set unprocessedQuestionnaireIds = lunaticJsonRawDataPersistencePort.findDistinctQuestionnaireIdsByNullProcessDate(); + Set unprocessedQuestionnaireIds = + lunaticJsonRawDataPersistencePort.findDistinctQuestionnaireIdsByNullProcessDate(); Set unprocessedQuestionnaireIdsWithSpecs = new HashSet<>(); - for (String unprocessedQuestionnaireId : unprocessedQuestionnaireIds){ + + for (String unprocessedQuestionnaireId : unprocessedQuestionnaireIds) { Set modes = lunaticJsonRawDataPersistencePort.findModesByQuestionnaire(unprocessedQuestionnaireId); - if (modes.isEmpty()){ + if (modes.isEmpty()) { continue; } boolean areAllSpecsOK = true; - for(Mode mode : modes){ - if(!isSpecsPresentForQuestionnaireAndMode(unprocessedQuestionnaireId, mode)){ + for (Mode mode : modes) { + if (!isSpecsPresentForQuestionnaireAndMode(unprocessedQuestionnaireId, mode)) { areAllSpecsOK = false; } } - if(areAllSpecsOK){ + + if (areAllSpecsOK) { unprocessedQuestionnaireIdsWithSpecs.add(unprocessedQuestionnaireId); } } @@ -397,153 +294,30 @@ public Set getUnprocessedDataQuestionnaireIds() { return unprocessedQuestionnaireIdsWithSpecs; } - private boolean isSpecsPresentForQuestionnaireAndMode(String unprocessedQuestionnaireId, Mode mode) { + private boolean isSpecsPresentForQuestionnaireAndMode(String questionnaireId, Mode mode) { List genesisErrors = new ArrayList<>(); MetadataModel metadataModel; + try { metadataModel = metadataService.loadAndSaveIfNotExists( - unprocessedQuestionnaireId, - unprocessedQuestionnaireId, + questionnaireId, + questionnaireId, mode, fileUtils, genesisErrors ); } catch (GenesisException ge) { - log.warn("Genesis exception thrown for questionnaire %s and mode %s, excluding from get questionnaire ids...".formatted(unprocessedQuestionnaireId, mode)); + log.warn( + "Genesis exception thrown for questionnaire {} and mode {}, excluding from get questionnaire ids...", + questionnaireId, + mode + ); return false; } - return metadataModel != null && genesisErrors.isEmpty(); - } - - @SuppressWarnings("unchecked") - private static void convertRawDataExternalVariables( - LunaticJsonRawDataModel srcRawData, - SurveyUnitModel dstSurveyUnitModel, - RawDataModelType rawDataModelType, - VariablesMap variablesMap - ) { - Map dataMap = srcRawData.data(); - if (rawDataModelType.equals(RawDataModelType.FILIERE)) { - dataMap = (Map) dataMap.get("data"); - } - - dataMap = (Map)dataMap.get("EXTERNAL"); - Map externalMap = JsonUtils.asMap(dataMap); - if (externalMap != null && !externalMap.isEmpty()){ - convertToExternalVar(dstSurveyUnitModel, variablesMap, externalMap); - } - } - - private static void convertToExternalVar(SurveyUnitModel dstSurveyUnitModel, VariablesMap variablesMap, Map externalMap) { - for(Map.Entry externalVariableEntry : externalMap.entrySet()){ - Object valueObject = externalVariableEntry.getValue(); - if (valueObject instanceof List){ - //Array of values - convertListVar(valueObject, externalVariableEntry, variablesMap, dstSurveyUnitModel.getExternalVariables()); - continue; - } - //Value - if (valueObject != null) { - convertOneVar(externalVariableEntry, valueObject.toString(), variablesMap, 1, dstSurveyUnitModel.getExternalVariables()); - } - } - } - - private static void convertOneVar(Map.Entry externalVariableEntry, String valueObject, VariablesMap variablesMap, int iteration, List dstSurveyUnitModel) { - VariableModel externalVariableModel = VariableModel.builder() - .varId(externalVariableEntry.getKey()) - .value(valueObject) - .scope(getIdLoop(variablesMap, externalVariableEntry.getKey())) - .iteration(iteration) - .parentId(GroupUtils.getParentGroupName(externalVariableEntry.getKey(), variablesMap)) - .build(); - dstSurveyUnitModel.add(externalVariableModel); - } - - @SuppressWarnings("unchecked") - private void convertRawDataCollectedVariables( - LunaticJsonRawDataModel srcRawData, - SurveyUnitModel dstSurveyUnitModel, - DataState dataState, - RawDataModelType rawDataModelType, - VariablesMap variablesMap - ) { - Map dataMap = srcRawData.data(); - if (rawDataModelType.equals(RawDataModelType.FILIERE)) { - dataMap = (Map) dataMap.get("data"); - } - - dataMap = (Map)dataMap.get("COLLECTED"); - - - Map collectedMap = JsonUtils.asMap(dataMap); - if (collectedMap == null || collectedMap.isEmpty()){ - if(dataState.equals(DataState.COLLECTED)) { - log.warn("No collected data for interrogation {}", srcRawData.interrogationId()); - } - return; - } - convertToCollectedVar(dstSurveyUnitModel, dataState, variablesMap, collectedMap); - } - - private static void convertToCollectedVar( - SurveyUnitModel dstSurveyUnitModel, - DataState dataState, - VariablesMap variablesMap, - Map collectedMap - ) { - final String stateKey = dataState.toString(); - final var dest = dstSurveyUnitModel.getCollectedVariables(); - - for (Map.Entry collectedVariable : collectedMap.entrySet()) { - processCollectedVariable(collectedVariable, stateKey, variablesMap, dstSurveyUnitModel, dest); - } - } - - - private static void convertListVar(Object valuesForState, Map.Entry collectedVariable, VariablesMap variablesMap, List dstSurveyUnitModel) { - List values = JsonUtils.asStringList(valuesForState); - if (!values.isEmpty()) { - int iteration = 1; - for (String value : values) { - if (value != null && !value.isEmpty()) { - convertOneVar(collectedVariable, value, variablesMap, iteration, dstSurveyUnitModel); - } - iteration++; - } - } - } - - private static String getIdLoop(VariablesMap variablesMap, String variableName) { - if (variablesMap.getVariable(variableName) == null) { - log.warn("Variable {} not present in metadatas, assigning to {}", variableName, Constants.ROOT_GROUP_NAME); - return Constants.ROOT_GROUP_NAME; - } - return variablesMap.getVariable(variableName).getGroupName(); + return metadataModel != null && genesisErrors.isEmpty(); } - @Override - public void updateProcessDates(List surveyUnitModels) { - Set collectionInstrumentIds = new HashSet<>(); - for (SurveyUnitModel surveyUnitModel : surveyUnitModels) { - if(surveyUnitModel.getCollectionInstrumentId() != null) { - collectionInstrumentIds.add(surveyUnitModel.getCollectionInstrumentId()); - } - } - - for (String collectionInstrumentId : collectionInstrumentIds) { - Set interrogationIds = new HashSet<>(); - for (SurveyUnitModel surveyUnitModel : - surveyUnitModels.stream().filter( - surveyUnitModel -> surveyUnitModel.getCollectionInstrumentId().equals(collectionInstrumentId) - ).toList()) { - interrogationIds.add(surveyUnitModel.getInterrogationId()); - } - lunaticJsonRawDataPersistencePort.updateProcessDates(collectionInstrumentId, interrogationIds); - } - } - @Override public Set findDistinctQuestionnaireIds() { return lunaticJsonRawDataPersistencePort.findDistinctQuestionnaireIds(); @@ -561,15 +335,28 @@ public long countDistinctInterrogationIdsByQuestionnaireId(String questionnaireI @Override public Map> findProcessedIdsgroupedByQuestionnaireSince(LocalDateTime since) { - List idsByQuestionnaire = lunaticJsonRawDataPersistencePort.findProcessedIdsGroupedByQuestionnaireSince(since); - List collectionInstrumentIds = idsByQuestionnaire.stream().map(GroupedInterrogation::questionnaireId).toList(); - List contexts = dataProcessingContextPersistancePort.findByCollectionInstrumentIds(collectionInstrumentIds); - List collectionInstrumentIdsWithReview = contexts.stream().filter(DataProcessingContextModel::isWithReview).map(DataProcessingContextModel::getCollectionInstrumentId).toList(); - return idsByQuestionnaire.stream().filter(groupedInterrogation -> collectionInstrumentIdsWithReview.contains(groupedInterrogation.questionnaireId())) + List idsByQuestionnaire = + lunaticJsonRawDataPersistencePort.findProcessedIdsGroupedByQuestionnaireSince(since); + + List collectionInstrumentIds = idsByQuestionnaire.stream() + .map(GroupedInterrogation::questionnaireId) + .toList(); + + List contexts = + dataProcessingContextPersistancePort.findByCollectionInstrumentIds(collectionInstrumentIds); + + List collectionInstrumentIdsWithReview = contexts.stream() + .filter(DataProcessingContextModel::isWithReview) + .map(DataProcessingContextModel::getCollectionInstrumentId) + .toList(); + + return idsByQuestionnaire.stream() + .filter(groupedInterrogation -> + collectionInstrumentIdsWithReview.contains(groupedInterrogation.questionnaireId())) .collect(Collectors.toMap( - GroupedInterrogation::questionnaireId, - GroupedInterrogation::interrogationIds - )); + GroupedInterrogation::questionnaireId, + GroupedInterrogation::interrogationIds + )); } @Override @@ -583,12 +370,16 @@ public boolean existsByInterrogationId(String interrogationId) { } @Override - public Page findRawDataByCampaignIdAndDate(String campaignId, Instant startDt, Instant endDt, Pageable pageable){ - return lunaticJsonRawDataPersistencePort.findByCampaignIdAndDate(campaignId,startDt, endDt,pageable); + public Page findRawDataByCampaignIdAndDate( + String campaignId, + Instant startDt, + Instant endDt, + Pageable pageable + ) { + return lunaticJsonRawDataPersistencePort.findByCampaignIdAndDate(campaignId, startDt, endDt, pageable); } - //Utils - protected static String getValueString(Object value) { + public static String getValueString(Object value) { if (value instanceof Double || value instanceof Float) { BigDecimal bd = new BigDecimal(value.toString()); return bd.stripTrailingZeros().toPlainString(); @@ -598,4 +389,4 @@ protected static String getValueString(Object value) { } return String.valueOf(value); } -} +} \ No newline at end of file diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java index 48d9ec96..3d69665d 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java @@ -2,78 +2,64 @@ import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.Constants; import fr.insee.genesis.configuration.Config; import fr.insee.genesis.controller.utils.ControllerUtils; -import fr.insee.genesis.domain.model.context.DataProcessingContextModel; +import fr.insee.genesis.domain.converter.rawdata.RawResponseConverter; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; -import fr.insee.genesis.domain.model.surveyunit.VariableModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; import fr.insee.genesis.domain.ports.api.RawResponseApiPort; import fr.insee.genesis.domain.ports.spi.RawResponsePersistencePort; -import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; -import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityToolService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; -import fr.insee.genesis.domain.utils.GroupUtils; -import fr.insee.genesis.domain.utils.JsonUtils; +import fr.insee.genesis.controller.dto.rawdata.ProcessingResultDto; import fr.insee.genesis.exceptions.GenesisError; import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.modelefiliere.ModeDto; -import fr.insee.modelefiliere.RawResponseDto; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; -import java.io.IOException; import java.time.Instant; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.*; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import java.util.stream.Collectors; -import static fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService.getValueString; - @Service @Slf4j -public class RawResponseService implements RawResponseApiPort { +@RequiredArgsConstructor +public class RawResponseService implements RawResponseApiPort { private final ControllerUtils controllerUtils; private final QuestionnaireMetadataService metadataService; private final SurveyUnitService surveyUnitService; private final SurveyUnitQualityService surveyUnitQualityService; - private final SurveyUnitQualityToolPort surveyUnitQualityToolPort; - private final DataProcessingContextService dataProcessingContextService; + private final SurveyUnitQualityToolService surveyUnitQualityToolService; private final FileUtils fileUtils; private final Config config; + private final RawResponseConverter rawResponseConverter; @Qualifier("rawResponseMongoAdapter") private final RawResponsePersistencePort rawResponsePersistencePort; - public RawResponseService(ControllerUtils controllerUtils, QuestionnaireMetadataService metadataService, SurveyUnitService surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, SurveyUnitQualityToolPort surveyUnitQualityToolPort, DataProcessingContextService dataProcessingContextService, FileUtils fileUtils, Config config, RawResponsePersistencePort rawResponsePersistencePort) { - this.controllerUtils = controllerUtils; - this.metadataService = metadataService; - this.surveyUnitService = surveyUnitService; - this.surveyUnitQualityService = surveyUnitQualityService; - this.surveyUnitQualityToolPort = surveyUnitQualityToolPort; - this.dataProcessingContextService = dataProcessingContextService; - this.fileUtils = fileUtils; - this.config = config; - this.rawResponsePersistencePort = rawResponsePersistencePort; - } - @Override - public List getRawResponses(String collectionInstrumentId, Mode mode, List interrogationIdList) { - return rawResponsePersistencePort.findRawResponses(collectionInstrumentId,mode,interrogationIdList); + public List getRawResponses( + String collectionInstrumentId, + Mode mode, + List interrogationIdList + ) { + return rawResponsePersistencePort.findRawResponses(collectionInstrumentId, mode, interrogationIdList); } @Override @@ -81,239 +67,173 @@ public List getRawResponsesByInterrogationID(String interrogat return rawResponsePersistencePort.findRawResponsesByInterrogationID(interrogationId); } + public DataProcessResult processRawResponsesByCollectionInstrumentId(String collectionInstrumentId) throws GenesisException { + List interrogationIds = rawResponsePersistencePort + .findUnprocessedInterrogationIdsByCollectionInstrumentId(collectionInstrumentId) + .stream() + .toList(); + + return processRawResponsesByInterrogationIds( + collectionInstrumentId, + interrogationIds, + new ArrayList<>() + ); + } + @Override - public DataProcessResult processRawResponsesByInterrogationIds(String collectionInstrumentId, List interrogationIdList, List errors) throws GenesisException { - int dataCount=0; - int formattedDataCount=0; - DataProcessingContextModel dataProcessingContext = - dataProcessingContextService.getContextByCollectionInstrumentId(collectionInstrumentId); - List modesList = controllerUtils.getModesList(collectionInstrumentId, null); - for (Mode mode : modesList) { - //Load and save metadata into database, throw exception if none - VariablesMap variablesMap = getVariablesMap(collectionInstrumentId,mode,errors); - int totalBatchs = Math.ceilDiv(interrogationIdList.size() , config.getRawDataProcessingBatchSize()); + public DataProcessResult processRawResponsesByInterrogationIds( + String collectionInstrumentId, + List interrogationIdList, + List errors + ) throws GenesisException { + + List modes = controllerUtils.getModesList(collectionInstrumentId, null); + boolean shouldUseQualityTool = + surveyUnitQualityToolService.resolveWithReviewValue(collectionInstrumentId); + + int batchSize = config.getRawDataProcessingBatchSize(); + int dataCount = 0; + int formattedDataCount = 0; + + for (Mode mode : modes) { + VariablesMap variablesMap = getVariablesMap(collectionInstrumentId, mode, errors); + List remainingInterrogationIds = new ArrayList<>(interrogationIdList); + int totalBatches = Math.ceilDiv(remainingInterrogationIds.size(), batchSize); int batchNumber = 1; - List interrogationIdListForMode = new ArrayList<>(interrogationIdList); - while(!interrogationIdListForMode.isEmpty()){ - log.info("Processing raw data batch {}/{}", batchNumber, totalBatchs); - int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize()); - List interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex); - List rawResponseModels = getRawResponses(collectionInstrumentId, mode, interrogationIdToProcess); - - List surveyUnitModels = convertRawResponse( - rawResponseModels, - variablesMap + while (!remainingInterrogationIds.isEmpty()) { + log.info( + "Processing raw data batch {}/{} for collectionInstrumentId={} mode={}", + batchNumber, + totalBatches, + collectionInstrumentId, + mode ); - //Save converted data - surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap); - surveyUnitService.saveSurveyUnits(surveyUnitModels); - - //Update process dates - updateProcessDates(surveyUnitModels); - - //Increment data count - dataCount += surveyUnitModels.size(); - formattedDataCount += surveyUnitModels.stream() - .filter(surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED)) - .toList() - .size(); - - //Send processed ids grouped by questionnaire (if review activated) - if(dataProcessingContext != null && dataProcessingContext.isWithReview()) { - sendProcessedIdsToQualityTool(surveyUnitModels); - } else { - log.warn("Data processing context not found for collection instrument {}. Ids processed not send to quality tool.",collectionInstrumentId); - } + int maxIndex = Math.min(remainingInterrogationIds.size(), batchSize); + List batch = remainingInterrogationIds.subList(0, maxIndex); + + ProcessingResultDto result = processRawResponsesForMode( + collectionInstrumentId, + mode, + batch, + variablesMap, + shouldUseQualityTool + ); - //Remove processed ids from list - interrogationIdListForMode = interrogationIdListForMode.subList(maxIndex, interrogationIdListForMode.size()); + dataCount += result.dataCount(); + formattedDataCount += result.formattedDataCount(); + remainingInterrogationIds = + remainingInterrogationIds.subList(maxIndex, remainingInterrogationIds.size()); batchNumber++; } } + return new DataProcessResult(dataCount, formattedDataCount, errors); } - @Override - public DataProcessResult processRawResponsesByInterrogationIds(String collectionInstrumentId) throws GenesisException { - int dataCount=0; - int formattedDataCount=0; - DataProcessingContextModel dataProcessingContext = - dataProcessingContextService.getContextByCollectionInstrumentId(collectionInstrumentId); - List errors = new ArrayList<>(); - - List modesList = controllerUtils.getModesList(collectionInstrumentId, null); - for (Mode mode : modesList) { - //Load and save metadata into database, throw exception if none - VariablesMap variablesMap = getVariablesMap(collectionInstrumentId,mode,errors); - Set interrogationIds = - rawResponsePersistencePort.findUnprocessedInterrogationIdsByCollectionInstrumentId(collectionInstrumentId); - - int totalBatchs = Math.ceilDiv(interrogationIds.size() , config.getRawDataProcessingBatchSize()); - int batchNumber = 1; - List interrogationIdListForMode = new ArrayList<>(interrogationIds); - while(!interrogationIdListForMode.isEmpty()){ - log.info("Processing raw data batch {}/{}", batchNumber, totalBatchs); - int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize()); + private ProcessingResultDto processRawResponsesForMode( + String collectionInstrumentId, + Mode mode, + List interrogationIds, + VariablesMap variablesMap, + boolean shouldUseQualityTool + ) { + List rawResponseModels = + rawResponsePersistencePort.findRawResponses(collectionInstrumentId, mode, interrogationIds); - List surveyUnitModels = getConvertedSurveyUnits( - collectionInstrumentId, - mode, - interrogationIdListForMode, - maxIndex, - variablesMap); - - //Save converted data - surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap); - surveyUnitService.saveSurveyUnits(surveyUnitModels); - - //Update process dates - updateProcessDates(surveyUnitModels); - - //Increment data count - dataCount += surveyUnitModels.size(); - formattedDataCount += surveyUnitModels.stream() - .filter(surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED)) - .toList() - .size(); - - //Send processed ids grouped by questionnaire (if review activated) - if(dataProcessingContext != null && dataProcessingContext.isWithReview()) { - sendProcessedIdsToQualityTool(surveyUnitModels); - } + List emptySurveyUnitModels = new ArrayList<>(); + List surveyUnitModels = + rawResponseConverter.convertRawResponseAndCollectEmptyModels(rawResponseModels, variablesMap, emptySurveyUnitModels); - //Remove processed ids from list - interrogationIdListForMode = interrogationIdListForMode.subList(maxIndex, interrogationIdListForMode.size()); - batchNumber++; - } + surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap); + surveyUnitService.saveSurveyUnits(surveyUnitModels); + + updateProcessDates(surveyUnitModels); + + if (!emptySurveyUnitModels.isEmpty()) { + updateProcessDates(emptySurveyUnitModels); } - return new DataProcessResult(dataCount, formattedDataCount, errors); - } + if (shouldUseQualityTool) { + surveyUnitQualityToolService.sendProcessedIdsToQualityTool(surveyUnitModels); + } - private List getConvertedSurveyUnits(String collectionInstrumentId, Mode mode, List interrogationIdListForMode, int maxIndex, VariablesMap variablesMap) { - List interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex); - List rawResponseModels = getRawResponses(collectionInstrumentId, mode, interrogationIdToProcess); - return convertRawResponse( - rawResponseModels, - variablesMap - ); + int formattedDataCount = (int) surveyUnitModels.stream() + .filter(surveyUnitModel -> surveyUnitModel.getState() == DataState.FORMATTED) + .count(); + + return new ProcessingResultDto(surveyUnitModels.size(), formattedDataCount); } - private VariablesMap getVariablesMap(String collectionInstrumentId, Mode mode, List errors) throws GenesisException { - VariablesMap variablesMap = metadataService.loadAndSaveIfNotExists(collectionInstrumentId, collectionInstrumentId, mode, fileUtils, - errors).getVariables(); + private VariablesMap getVariablesMap( + String collectionInstrumentId, + Mode mode, + List errors + ) throws GenesisException { + VariablesMap variablesMap = metadataService.loadAndSaveIfNotExists( + collectionInstrumentId, + collectionInstrumentId, + mode, + fileUtils, + errors + ).getVariables(); + if (variablesMap == null) { - throw new GenesisException(HttpStatus.BAD_REQUEST, + throw new GenesisException( + HttpStatus.BAD_REQUEST, "Error during metadata parsing for mode %s :%n%s" .formatted(mode, errors.getLast().getMessage()) ); } - return variablesMap; - } - @Override - public List convertRawResponse(List rawResponseModels, VariablesMap variablesMap) { - //Convert to genesis model - List surveyUnitModels = new ArrayList<>(); - List emptySurveyUnitModels = new ArrayList<>(); - //For each possible data state (we receive COLLECTED or EDITED) - for(DataState dataState : List.of(DataState.COLLECTED,DataState.EDITED)){ - for (RawResponseModel rawResponseModel : rawResponseModels) { - //Get optional fields - Boolean isCapturedIndirectly = getIsCapturedIndirectly(rawResponseModel); - String questionnaireStateString = getStringFieldInPayload(rawResponseModel, "questionnaireState"); - RawResponseDto.QuestionnaireStateEnum questionnaireStateEnum = null; - try{ - questionnaireStateEnum = RawResponseDto.QuestionnaireStateEnum.valueOf(questionnaireStateString); - } catch (IllegalArgumentException iae){ - log.warn("'{}' is not a valid questionnaire state according to filiere model", questionnaireStateString); - } catch (NullPointerException ignored){ - //WARN already done in getStringFieldInPayload - } - LocalDateTime validationDate = getValidationDate(rawResponseModel); - String usualSurveyUnitId = getStringFieldInPayload(rawResponseModel,"usualSurveyUnitId"); - String majorModelVersion = getStringFieldInPayload(rawResponseModel, "majorModelVersion"); - - SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() - .collectionInstrumentId(rawResponseModel.collectionInstrumentId()) - .majorModelVersion(majorModelVersion) - .mode(rawResponseModel.mode()) - .interrogationId(rawResponseModel.interrogationId()) - .usualSurveyUnitId(usualSurveyUnitId) - .questionnaireState(questionnaireStateEnum) - .validationDate(validationDate) - .isCapturedIndirectly(isCapturedIndirectly) - .state(dataState) - .fileDate(rawResponseModel.recordDate()) - .recordDate(Instant.now()) - .collectedVariables(new ArrayList<>()) - .externalVariables(new ArrayList<>()) - .build(); - - //Data collected variables conversion - convertRawDataCollectedVariables(rawResponseModel, surveyUnitModel, dataState, variablesMap); - - //External variables conversion into COLLECTED document - if(dataState == DataState.COLLECTED){ - convertRawDataExternalVariables(rawResponseModel, surveyUnitModel, variablesMap); - } - - boolean hasNoVariable = surveyUnitModel.getCollectedVariables().isEmpty() - && surveyUnitModel.getExternalVariables().isEmpty(); - - if(hasNoVariable){ - if(surveyUnitModel.getState() == DataState.COLLECTED){ - log.warn("No collected or external variable for interrogation {}, raw data is ignored.", rawResponseModel.interrogationId()); - } - emptySurveyUnitModels.add(surveyUnitModel); - continue;// don't add suModel - } - surveyUnitModels.add(surveyUnitModel); - } - } - if(!emptySurveyUnitModels.isEmpty()){ - updateProcessDates(emptySurveyUnitModels); - } - return surveyUnitModels; + return variablesMap; } @Override public List getUnprocessedCollectionInstrumentIds() { List unprocessedCollectionInstrumentIds = rawResponsePersistencePort.getUnprocessedCollectionIds(); List unprocessedCollectionInstrumentIdsWithSpecs = new ArrayList<>(); - for (String unprocessedCollectionInstrumentId : unprocessedCollectionInstrumentIds){ - Set modes = new HashSet<>(rawResponsePersistencePort.findModesByCollectionInstrument(unprocessedCollectionInstrumentId)); - if (modes.isEmpty()){ + + for (String unprocessedCollectionInstrumentId : unprocessedCollectionInstrumentIds) { + Set modes = new HashSet<>( + rawResponsePersistencePort.findModesByCollectionInstrument(unprocessedCollectionInstrumentId) + ); + + if (modes.isEmpty()) { continue; } boolean areAllSpecsOK = true; - if(modes.contains(null) && modes.size() == 1){ + + if (modes.contains(null) && modes.size() == 1) { areAllSpecsOK = false; } - for(ModeDto modeDto : modes){ - if(modeDto == null){ + + for (ModeDto modeDto : modes) { + if (modeDto == null) { continue; } + Mode mode = Mode.getEnumFromJsonName(modeDto.toString()); - if(!isSpecsPresentForCollectionInstrumentAndMode(unprocessedCollectionInstrumentId, mode)){ + if (!isSpecsPresentForCollectionInstrumentAndMode(unprocessedCollectionInstrumentId, mode)) { areAllSpecsOK = false; } } - if(areAllSpecsOK){ - unprocessedCollectionInstrumentIdsWithSpecs.add(unprocessedCollectionInstrumentId); + + if (areAllSpecsOK) { + unprocessedCollectionInstrumentIdsWithSpecs .add(unprocessedCollectionInstrumentId); } } - return unprocessedCollectionInstrumentIdsWithSpecs; + return unprocessedCollectionInstrumentIdsWithSpecs ; } private boolean isSpecsPresentForCollectionInstrumentAndMode(String unprocessedCollectionInstrumentId, Mode mode) { List genesisErrors = new ArrayList<>(); MetadataModel metadataModel; + try { metadataModel = metadataService.loadAndSaveIfNotExists( unprocessedCollectionInstrumentId, @@ -323,10 +243,14 @@ private boolean isSpecsPresentForCollectionInstrumentAndMode(String unprocessedC genesisErrors ); } catch (GenesisException ge) { - log.warn("Genesis exception thrown for collection instrument %s and mode %s, excluding from get collection instrument ids..." - .formatted(unprocessedCollectionInstrumentId, mode)); + log.warn( + "Genesis exception thrown for collection instrument {} and mode {}, excluding from get collection instrument ids...", + unprocessedCollectionInstrumentId, + mode + ); return false; } + return metadataModel != null && genesisErrors.isEmpty(); } @@ -351,193 +275,14 @@ public boolean existsByInterrogationId(String interrogationId) { return rawResponsePersistencePort.existsByInterrogationId(interrogationId); } - private Map> getProcessedIdsMap(List surveyUnitModels) { - Map> processedInterrogationIdsPerQuestionnaire = new HashMap<>(); - surveyUnitModels.forEach(model -> - processedInterrogationIdsPerQuestionnaire - .computeIfAbsent(model.getCollectionInstrumentId(), k -> new HashSet<>()) - .add(model.getInterrogationId()) - ); - return processedInterrogationIdsPerQuestionnaire; - } - - private void sendProcessedIdsToQualityTool(List surveyUnitModels) { - try { - Map> processedIdsMap = getProcessedIdsMap(surveyUnitModels); - ResponseEntity response = - surveyUnitQualityToolPort.sendProcessedIds(processedIdsMap); - - if (response.getStatusCode().is2xxSuccessful()) { - log.info("Successfully sent {} ids to quality tool", processedIdsMap.size()); - }else{ - log.warn("Survey unit quality tool responded non-2xx code {} and body {}", - response.getStatusCode(), response.getBody()); - } - }catch (IOException e){ - log.error("Error during Perret call request building : {}", e.toString()); - } - } - - private static Boolean getIsCapturedIndirectly(RawResponseModel rawResponseModel) { - try{ - return rawResponseModel.payload().get("isCapturedIndirectly") == null ? null : - Boolean.parseBoolean(rawResponseModel.payload().get("isCapturedIndirectly").toString()); - }catch(Exception e){ - log.warn("Exception when parsing isCapturedIndirectly : {}",e.toString()); - return Boolean.FALSE; - } - } - - private static LocalDateTime getValidationDate(RawResponseModel rawResponseModel) { - try{ - return rawResponseModel.payload().get("validationDate") == null ? null : - LocalDateTime.parse(rawResponseModel.payload().get("validationDate").toString(), DateTimeFormatter.ISO_OFFSET_DATE_TIME); - }catch(Exception e){ - log.warn("Exception when parsing validation date : {}",e.toString()); - return null; - } - } - - private static String getStringFieldInPayload(RawResponseModel rawResponseModel, String field) { - try{ - return rawResponseModel.payload().get(field).toString(); - }catch(Exception e){ - log.warn("Exception when parsing {} : {}",field, e.toString()); - return null; - } - } - - @SuppressWarnings("unchecked") - private void convertRawDataCollectedVariables( - RawResponseModel rawResponseModel, - SurveyUnitModel dstSurveyUnitModel, - DataState dataState, - VariablesMap variablesMap - ) { - Map dataMap = rawResponseModel.payload(); - dataMap = (Map) dataMap.get("data"); - - dataMap = (Map)dataMap.get("COLLECTED"); - - - Map collectedMap = JsonUtils.asMap(dataMap); - if (collectedMap == null || collectedMap.isEmpty()){ - if(dataState.equals(DataState.COLLECTED)) { - log.warn("No collected data for interrogation {}", rawResponseModel.interrogationId()); - } - return; - } - convertToCollectedVar(dstSurveyUnitModel, dataState, variablesMap, collectedMap); - } - - private static void convertToCollectedVar( - SurveyUnitModel dstSurveyUnitModel, - DataState dataState, - VariablesMap variablesMap, - Map collectedMap - ) { - final String stateKey = dataState.toString(); - final var collectedVariables = dstSurveyUnitModel.getCollectedVariables(); - - for (Map.Entry collectedVariable : collectedMap.entrySet()) { - processCollectedVariable(collectedVariable, stateKey, variablesMap, dstSurveyUnitModel, collectedVariables); - } - } - - - static void processCollectedVariable( - Map.Entry entry, - String stateKey, - VariablesMap variablesMap, - SurveyUnitModel dstSurveyUnitModel, - List variableModelList - ) { - if (Constants.PAIRWISES.equals(entry.getKey())) { - handlePairwiseCollectedVariable(entry, DataState.valueOf(stateKey), variablesMap, dstSurveyUnitModel); - return; - } - - Map states = JsonUtils.asMap(entry.getValue()); - if (states == null) return; - - Object value = states.get(stateKey); - if (value == null) return; - - if (value instanceof List list) { - convertListVar(list, entry, variablesMap, variableModelList); - } else { - convertOneVar(entry, getValueString(value), variablesMap, 1, variableModelList); - } - } - - - private static void convertListVar(Object valuesForState, Map.Entry collectedVariable, VariablesMap variablesMap, List dstSurveyUnitModel) { - List values = JsonUtils.asStringList(valuesForState); - if (!values.isEmpty()) { - int iteration = 1; - for (String value : values) { - if (value != null && !value.isEmpty()) { - convertOneVar(collectedVariable, value, variablesMap, iteration, dstSurveyUnitModel); - } - iteration++; - } - } - } - - private static void convertOneVar(Map.Entry externalVariableEntry, String valueObject, VariablesMap variablesMap, int iteration, List dstSurveyUnitModel) { - VariableModel externalVariableModel = VariableModel.builder() - .varId(externalVariableEntry.getKey()) - .value(valueObject) - .scope(getIdLoop(variablesMap, externalVariableEntry.getKey())) - .iteration(iteration) - .parentId(GroupUtils.getParentGroupName(externalVariableEntry.getKey(), variablesMap)) - .build(); - dstSurveyUnitModel.add(externalVariableModel); - } - - private static String getIdLoop(VariablesMap variablesMap, String variableName) { - if (variablesMap.getVariable(variableName) == null) { - log.warn("Variable {} not present in metadata, assigning to {}", variableName, Constants.ROOT_GROUP_NAME); - return Constants.ROOT_GROUP_NAME; - } - return variablesMap.getVariable(variableName).getGroupName(); - } - - @SuppressWarnings("unchecked") - private static void convertRawDataExternalVariables( - RawResponseModel rawResponseModel, - SurveyUnitModel dstSurveyUnitModel, - VariablesMap variablesMap - ) { - Map dataMap = rawResponseModel.payload(); - dataMap = (Map) dataMap.get("data"); - - - dataMap = (Map)dataMap.get("EXTERNAL"); - Map externalMap = JsonUtils.asMap(dataMap); - if (externalMap != null && !externalMap.isEmpty()){ - convertToExternalVar(dstSurveyUnitModel, variablesMap, externalMap); - } - } - - private static void convertToExternalVar(SurveyUnitModel dstSurveyUnitModel, VariablesMap variablesMap, Map externalMap) { - for(Map.Entry externalVariableEntry : externalMap.entrySet()){ - Object valueObject = externalVariableEntry.getValue(); - if (valueObject instanceof List){ - //Array of values - convertListVar(valueObject, externalVariableEntry, variablesMap, dstSurveyUnitModel.getExternalVariables()); - continue; - } - //Value - if (valueObject != null) { - convertOneVar(externalVariableEntry, valueObject.toString(), variablesMap, 1, dstSurveyUnitModel.getExternalVariables()); - } - } - } - @Override - public Page findRawResponseDataByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable) { - return rawResponsePersistencePort.findByCampaignIdAndDate(campaignId,startDate, endDate,pageable); + public Page findRawResponseDataByCampaignIdAndDate( + String campaignId, + Instant startDate, + Instant endDate, + Pageable pageable + ) { + return rawResponsePersistencePort.findByCampaignIdAndDate(campaignId, startDate, endDate, pageable); } @Override @@ -556,77 +301,11 @@ public Set getDistinctCollectionInstrumentIds() { } @Override - public Page findRawResponseDataByCollectionInstrumentId(String collectionInstrumentId, Pageable pageable) { - return rawResponsePersistencePort.findByCollectionInstrumentId(collectionInstrumentId, pageable); - } - - - - @SuppressWarnings("unchecked") - static void handlePairwiseCollectedVariable( - Map.Entry collectedVariable, - DataState dataState, - VariablesMap variablesMap, - SurveyUnitModel dstSurveyUnitModel - ) { - Object value = getValueForState(collectedVariable, dataState.toString()); - - if (isInvalidPairwiseVariable(value, variablesMap)) { - return; - } - - List individuals = (List) value; - - String groupName = variablesMap - .getVariable(Constants.PAIRWISE_PREFIX + 1) - .getGroupName(); - - for (int individualIndex = 0; individualIndex < individuals.size(); individualIndex++) { - List individualLinks = (List) individuals.get(individualIndex); - - for (int linkIndex = 1; linkIndex < Constants.MAX_LINKS_ALLOWED; linkIndex++) { - dstSurveyUnitModel.getCollectedVariables().add( - buildPairwiseVariable(individualLinks, linkIndex, individualIndex+ 1, groupName) - ); - } - } - } - - private static VariableModel buildPairwiseVariable( - List individualLinks, - int linkIndex, - int iteration, - String groupName + public Page findRawResponseDataByCollectionInstrumentId( + String collectionInstrumentId, + Pageable pageable ) { - String value = Constants.NO_PAIRWISE_VALUE; - - if (linkIndex <= individualLinks.size()) { - String v = individualLinks.get(linkIndex - 1); - value = (v == null || v.isBlank()) - ? Constants.SAME_AXIS_VALUE - : v; - } - - return VariableModel.builder() - .varId(Constants.PAIRWISE_PREFIX + linkIndex) - .value(value) - .scope(groupName) - .iteration(iteration) - .parentId(Constants.ROOT_GROUP_NAME) - .build(); - } - - - private static Object getValueForState( - Map.Entry collectedVariable, - String stateKey - ) { - Map states = JsonUtils.asMap(collectedVariable.getValue()); - return states != null ? states.get(stateKey) : null; - } - - private static boolean isInvalidPairwiseVariable(Object value, VariablesMap variablesMap) { - return !(value instanceof List) || !variablesMap.hasVariable(Constants.PAIRWISE_PREFIX + 1); + return rawResponsePersistencePort.findByCollectionInstrumentId(collectionInstrumentId, pageable); } -} +} \ No newline at end of file diff --git a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitQualityToolService.java b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitQualityToolService.java new file mode 100644 index 00000000..db46194a --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitQualityToolService.java @@ -0,0 +1,72 @@ +package fr.insee.genesis.domain.service.surveyunit; + +import fr.insee.genesis.domain.model.context.DataProcessingContextModel; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; +import fr.insee.genesis.domain.service.context.DataProcessingContextService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@Service +@Slf4j +@RequiredArgsConstructor +public class SurveyUnitQualityToolService { + + private final SurveyUnitQualityToolPort surveyUnitQualityToolPort; + private final DataProcessingContextService dataProcessingContextService; + + public boolean resolveWithReviewValue(String collectionInstrumentId) { + DataProcessingContextModel dataProcessingContext = + dataProcessingContextService.getContextByCollectionInstrumentId(collectionInstrumentId); + + if (dataProcessingContext == null) { + log.warn( + "Data processing context not found for collection instrument {}. Ids processed not sent to quality tool.", + collectionInstrumentId + ); + return false; + } + + return dataProcessingContext.isWithReview(); + } + + public void sendProcessedIdsToQualityTool(List surveyUnitModels) { + try { + Map> processedIdsMap = getProcessedIdsMap(surveyUnitModels); + ResponseEntity response = surveyUnitQualityToolPort.sendProcessedIds(processedIdsMap); + + if (response.getStatusCode().is2xxSuccessful()) { + log.info("Successfully sent {} ids to quality tool", processedIdsMap.size()); + } else { + log.warn( + "Survey unit quality tool responded non-2xx code {} and body {}", + response.getStatusCode(), + response.getBody() + ); + } + } catch (IOException e) { + log.error("Error during Perret call request building : {}", e.toString()); + } + } + + private Map> getProcessedIdsMap(List surveyUnitModels) { + Map> processedInterrogationIdsPerQuestionnaire = new HashMap<>(); + + surveyUnitModels.forEach(model -> + processedInterrogationIdsPerQuestionnaire + .computeIfAbsent(model.getCollectionInstrumentId(), key -> new HashSet<>()) + .add(model.getInterrogationId()) + ); + + return processedInterrogationIdsPerQuestionnaire; + } +} \ No newline at end of file diff --git a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java index 8feafc73..d059d862 100644 --- a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java +++ b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java @@ -677,7 +677,6 @@ private void extractVariables(SurveyUnitModel surveyUnitModel, private Object getValueWithType(String variableName, String value, VariablesMap variablesMap) { if(!variablesMap.hasVariable(variableName)){ - log.debug("Variable {} not found in variableMap", variableName); return value; } if(value == null) return null; diff --git a/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessIT.java b/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessIT.java index 5448867a..aeb6345f 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessIT.java +++ b/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessIT.java @@ -5,6 +5,7 @@ import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; import fr.insee.genesis.domain.ports.api.RawResponseApiPort; import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityToolService; import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.DisplayName; @@ -51,7 +52,8 @@ class ControllerAccessIT extends IntegrationTestAbstract { private RawResponseApiPort rawResponseApiPort; @MockitoBean private RawResponseInputRepository rawRepository; - + @MockitoBean + private SurveyUnitQualityToolService surveyUnitQualityToolService; /** * Provides a stream of URIs that are allowed for reader. */ diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java index 662f7267..50dc7c36 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java @@ -238,7 +238,7 @@ class ProcessRawResponsesByCollectionInstrumentIdTests { @DisplayName("Should return 200 with count message") void processByCollectionInstrumentId_shouldReturn200() throws Exception { // GIVEN - when(rawResponseApiPort.processRawResponsesByInterrogationIds("QUEST01")) + when(rawResponseApiPort.processRawResponsesByCollectionInstrumentId("QUEST01")) .thenReturn(new DataProcessResult(5, 0, new ArrayList<>())); // WHEN / THEN @@ -252,7 +252,7 @@ void processByCollectionInstrumentId_shouldReturn200() throws Exception { @DisplayName("Should return GenesisException status when port throws") void processByCollectionInstrumentId_genesisException_shouldReturnExceptionStatus() throws Exception { // GIVEN - when(rawResponseApiPort.processRawResponsesByInterrogationIds(anyString())) + when(rawResponseApiPort.processRawResponsesByCollectionInstrumentId(anyString())) .thenThrow(new GenesisException(HttpStatus.UNPROCESSABLE_ENTITY, "Unprocessable")); // WHEN / THEN diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java index 36b405a5..5bab0fdf 100644 --- a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java @@ -5,6 +5,7 @@ import fr.insee.genesis.configuration.Config; import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; import fr.insee.genesis.controller.utils.ControllerUtils; +import fr.insee.genesis.domain.converter.rawdata.LunaticJsonRawDataConverter; import fr.insee.genesis.domain.model.context.DataProcessingContextModel; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.GroupedInterrogation; @@ -12,12 +13,14 @@ import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; +import fr.insee.genesis.domain.parser.rawdata.LunaticJsonRawDataPayloadParser; import fr.insee.genesis.domain.ports.spi.DataProcessingContextPersistancePort; import fr.insee.genesis.domain.ports.spi.LunaticJsonRawDataPersistencePort; import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityToolService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.exceptions.GenesisError; import fr.insee.genesis.exceptions.GenesisException; @@ -27,7 +30,6 @@ import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; @@ -52,8 +54,19 @@ import static fr.insee.genesis.TestConstants.DEFAULT_INTERROGATION_ID; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) @@ -72,6 +85,8 @@ class LunaticJsonRawDataServiceTest { @Mock private SurveyUnitQualityToolPort surveyUnitQualityToolPort; @Mock + private SurveyUnitQualityToolService surveyUnitQualityToolService; + @Mock private DataProcessingContextService dataProcessingContextService; @Mock private DataProcessingContextPersistancePort dataProcessingContextPersistancePort; @@ -80,12 +95,32 @@ class LunaticJsonRawDataServiceTest { @Mock private Config config; - @InjectMocks + private LunaticJsonRawDataConverter lunaticJsonRawDataConverter; + @Mock private LunaticJsonRawDataService service; private static final String QUESTIONNAIRE_ID = "questionnaire-1"; private static final String INTERROGATION_ID = "interrogation-1"; + + @BeforeEach + void init() { + lunaticJsonRawDataConverter = new LunaticJsonRawDataConverter(new LunaticJsonRawDataPayloadParser()); + + service = new LunaticJsonRawDataService( + controllerUtils, + metadataService, + surveyUnitService, + surveyUnitQualityService, + surveyUnitQualityToolService, + fileUtils, + config, + lunaticJsonRawDataConverter, + lunaticJsonRawDataPersistencePort, + dataProcessingContextPersistancePort + ); + } + @Nested @DisplayName("save()") class SaveTests { @@ -220,10 +255,8 @@ void missingMetadata_throwsGenesisException() throws GenesisException { void withReview_sendsProcessedIds() throws GenesisException, IOException { // GIVEN // Context with review - DataProcessingContextModel context = mock(DataProcessingContextModel.class); - when(context.isWithReview()).thenReturn(true); - when(dataProcessingContextService.getContextByCollectionInstrumentId(QUESTIONNAIRE_ID)) - .thenReturn(context); + when(surveyUnitQualityToolService.resolveWithReviewValue(QUESTIONNAIRE_ID)) + .thenReturn(true); MetadataModel metadataModel = new MetadataModel(); when(metadataService.loadAndSaveIfNotExists(anyString(), anyString(), any(), any(), any())) @@ -253,8 +286,8 @@ void withReview_sendsProcessedIds() throws GenesisException, IOException { service.processRawData(QUESTIONNAIRE_ID); //THEN - verify(surveyUnitQualityToolPort, times(1)).sendProcessedIds(anyMap()); - } + verify(surveyUnitQualityToolService, times(1)) + .sendProcessedIdsToQualityTool(anyList()); } } @Nested @@ -265,7 +298,7 @@ class ConvertRawDataTests { @DisplayName("Empty raw data list returns empty list") void emptyRawDataList_returnsEmpty() { //WHEN - List result = service.convertRawData(List.of(), new VariablesMap()); + List result = lunaticJsonRawDataConverter.convertRawData(List.of(), new VariablesMap()); //THEN assertThat(result).isEmpty(); @@ -276,13 +309,19 @@ void emptyRawDataList_returnsEmpty() { void noVariables_rawDataIgnored() { //GIVEN LunaticJsonRawDataModel rawData = buildRawDataWithCollected(Map.of()); + List emptySurveyUnitModels = new ArrayList<>(); - //WHEN - List result = service.convertRawData(List.of(rawData), new VariablesMap()); + // WHEN + List result = + lunaticJsonRawDataConverter.convertRawDataAndCollectEmptyModels( + List.of(rawData), + new VariablesMap(), + emptySurveyUnitModels + ); - //THEN + // THEN assertThat(result).isEmpty(); - verify(lunaticJsonRawDataPersistencePort, atLeastOnce()).updateProcessDates(eq(QUESTIONNAIRE_ID), anySet()); + assertThat(emptySurveyUnitModels).isNotEmpty(); } @Test @@ -298,7 +337,7 @@ void withCollectedVariable_producesBothDataStates() { LunaticJsonRawDataModel rawData = buildRawDataWithCollected(collected); //WHEN - List result = service.convertRawData(List.of(rawData), new VariablesMap()); + List result = lunaticJsonRawDataConverter.convertRawData(List.of(rawData), new VariablesMap()); //THEN // Expect 1 COLLECTED (VAR1 has a value) and 0 EDITED (EDITED value is null → empty) @@ -323,7 +362,7 @@ void withEditedVariable_producesEditedModel() { LunaticJsonRawDataModel rawData = buildRawDataWithCollected(collected); //WHEN - List result = service.convertRawData(List.of(rawData), new VariablesMap()); + List result = lunaticJsonRawDataConverter.convertRawData(List.of(rawData), new VariablesMap()); //THEN long editedCount = result.stream() @@ -349,7 +388,7 @@ void filiereModelType_detected() { LunaticJsonRawDataModel rawData = buildRawDataWithCollected(outerData); //WHEN - List result = service.convertRawData(List.of(rawData), new VariablesMap()); + List result = lunaticJsonRawDataConverter.convertRawData(List.of(rawData), new VariablesMap()); // Should not throw — FILIERE path is followed TODO More asserts assertThat(result).isNotNull(); @@ -380,7 +419,7 @@ void optionalFields_mappedCorrectly() { .build(); //WHEN - List result = service.convertRawData(List.of(rawData), new VariablesMap()); + List result = lunaticJsonRawDataConverter.convertRawData(List.of(rawData), new VariablesMap()); //THEN assertThat(result).isNotEmpty(); @@ -415,8 +454,7 @@ void malformedValidationDate_fallsBackToNull() { .build(); //WHEN - List result = service.convertRawData(List.of(rawData), new VariablesMap()); - + List result = lunaticJsonRawDataConverter.convertRawData(List.of(rawData), new VariablesMap()); //THEN assertThat(result).isNotEmpty(); result.stream() @@ -437,7 +475,7 @@ void arrayValues_convertedToMultipleIterations() { LunaticJsonRawDataModel rawData = buildRawDataWithCollected(collected); //WHEN - List result = service.convertRawData(List.of(rawData), new VariablesMap()); + List result = lunaticJsonRawDataConverter.convertRawData(List.of(rawData), new VariablesMap()); //THEN SurveyUnitModel collectedModel = result.stream() diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java index e5565f24..fa1873e1 100644 --- a/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java @@ -4,17 +4,18 @@ import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.TestConstants; import fr.insee.genesis.controller.utils.ControllerUtils; +import fr.insee.genesis.domain.converter.rawdata.RawResponseConverter; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; import fr.insee.genesis.domain.ports.spi.QuestionnaireMetadataPersistencePort; import fr.insee.genesis.domain.ports.spi.RawResponsePersistencePort; -import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; -import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityToolService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import fr.insee.genesis.domain.parser.rawdata.RawResponsePayloadParser; import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.modelefiliere.ModeDto; @@ -55,16 +56,22 @@ @MockitoSettings(strictness = Strictness.LENIENT) class RawResponseServiceUnitTest { - static RawResponseService rawResponseService; + private RawResponseService rawResponseService; @Mock - static RawResponsePersistencePort rawResponsePersistencePort; + private RawResponsePersistencePort rawResponsePersistencePort; @Mock - static ControllerUtils controllerUtils; + private ControllerUtils controllerUtils; @Mock - static QuestionnaireMetadataService metadataService; + private QuestionnaireMetadataService metadataService; @Mock - static SurveyUnitService surveyUnitService; + private SurveyUnitService surveyUnitService; + @Mock + private SurveyUnitQualityService surveyUnitQualityService; + @Mock + private SurveyUnitQualityToolService surveyUnitQualityToolService; + + private RawResponseConverter rawResponseConverter; @Captor private ArgumentCaptor> surveyUnitModelsCaptor; @@ -73,15 +80,17 @@ class RawResponseServiceUnitTest { @BeforeEach void init() { + rawResponseConverter = new RawResponseConverter(new RawResponsePayloadParser()); + rawResponseService = new RawResponseService( controllerUtils, metadataService, surveyUnitService, - mock(SurveyUnitQualityService.class), - mock(SurveyUnitQualityToolPort.class), - mock(DataProcessingContextService.class), + surveyUnitQualityService, + surveyUnitQualityToolService, new FileUtils(TestConstants.getConfigStub()), TestConstants.getConfigStub(), + rawResponseConverter, rawResponsePersistencePort ); } @@ -117,14 +126,14 @@ void getUnprocessedCollectionInstrumentIds_shouldnt_return_if_no_spec() { mock(QuestionnaireMetadataPersistencePort.class) ); rawResponseService = new RawResponseService( - new ControllerUtils(new FileUtils(TestConstants.getConfigStub())), + controllerUtils, metadataService, - mock(SurveyUnitService.class), - mock(SurveyUnitQualityService.class), - mock(SurveyUnitQualityToolPort.class), - mock(DataProcessingContextService.class), + surveyUnitService, + surveyUnitQualityService, + surveyUnitQualityToolService, new FileUtils(TestConstants.getConfigStub()), TestConstants.getConfigStub(), + rawResponseConverter, rawResponsePersistencePort ); @@ -275,6 +284,7 @@ private void givenOkCase(RawResponseDto.QuestionnaireStateEnum questionnaireStat rawResponse.payload().put("questionnaireState", questionnaireState); rawResponse.payload().put("usualSurveyUnitId", TestConstants.DEFAULT_SURVEY_UNIT_ID); rawResponse.payload().put("majorModelVersion", 2); + Map>> dataMap = new HashMap<>(); dataMap.put("COLLECTED", new HashMap<>()); dataMap.get("COLLECTED").put("VAR1", new HashMap<>()); @@ -360,7 +370,7 @@ private void givenInvalidValidationDate(){ //WHENS private List whenProcessByCollectionInstrumentIdAndInterrogationIdList() throws GenesisException { - rawResponseService.processRawResponsesByInterrogationIds(TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID); + rawResponseService.processRawResponsesByCollectionInstrumentId(TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID); verify(surveyUnitService).saveSurveyUnits(surveyUnitModelsCaptor.capture()); return surveyUnitModelsCaptor.getValue(); } @@ -438,7 +448,7 @@ void convertRawResponse_shouldConvertCollectedVariable() { List rawResponses = List.of(rawResponse); // WHEN - List result = rawResponseService.convertRawResponse(rawResponses, variablesMap); + List result = rawResponseConverter.convertRawResponse(rawResponses, variablesMap); // THEN Assertions.assertThat(result).hasSize(1); // only COLLECTED state (EDITED has no data) @@ -455,7 +465,7 @@ void convertRawResponse_shouldConvertEditedVariable() { List rawResponses = List.of(rawResponse); // WHEN - List result = rawResponseService.convertRawResponse(rawResponses, variablesMap); + List result = rawResponseConverter.convertRawResponse(rawResponses, variablesMap); // THEN // On attend 1 modèle EDITED avec des variables @@ -475,7 +485,7 @@ void convertRawResponse_shouldConvertExternalVariables() { List rawResponses = List.of(rawResponse); // WHEN - List result = rawResponseService.convertRawResponse(rawResponses, variablesMap); + List result = rawResponseConverter.convertRawResponse(rawResponses, variablesMap); // THEN List collectedModels = result.stream() @@ -494,7 +504,7 @@ void convertRawResponse_shouldIgnoreEmptyResponse() { List rawResponses = List.of(rawResponse); // WHEN - List result = rawResponseService.convertRawResponse(rawResponses, variablesMap); + List result = rawResponseConverter.convertRawResponse(rawResponses, variablesMap); // THEN Assertions.assertThat(result).isEmpty(); @@ -508,7 +518,7 @@ void convertRawResponse_shouldHandleListValues() { List rawResponses = List.of(rawResponse); // WHEN - List result = rawResponseService.convertRawResponse(rawResponses, variablesMap); + List result = rawResponseConverter.convertRawResponse(rawResponses, variablesMap); // THEN List collectedModels = result.stream() @@ -530,7 +540,7 @@ void convertRawResponse_shouldSkipNullOrEmptyListValues() { List rawResponses = List.of(rawResponse); // WHEN - List result = rawResponseService.convertRawResponse(rawResponses, variablesMap); + List result = rawResponseConverter.convertRawResponse(rawResponses, variablesMap); // THEN List collectedModels = result.stream() diff --git a/src/test/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapterTest.java b/src/test/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapterTest.java index f5d51aae..968f84d8 100644 --- a/src/test/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapterTest.java +++ b/src/test/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapterTest.java @@ -22,7 +22,9 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) class SurveyQualityToolPerretAdapterTest {