SurveyUnitService.java

package fr.insee.genesis.domain.service.surveyunit;

import fr.insee.bpm.metadata.model.VariableType;
import fr.insee.bpm.metadata.model.VariablesMap;
import fr.insee.genesis.controller.dto.CampaignWithQuestionnaire;
import fr.insee.genesis.controller.dto.QuestionnaireWithCampaign;
import fr.insee.genesis.controller.dto.SurveyUnitDto;
import fr.insee.genesis.controller.dto.SurveyUnitInputDto;
import fr.insee.genesis.controller.dto.SurveyUnitSimplifiedDto;
import fr.insee.genesis.controller.dto.VariableDto;
import fr.insee.genesis.controller.dto.VariableInputDto;
import fr.insee.genesis.controller.dto.VariableStateDto;
import fr.insee.genesis.domain.model.surveyunit.DataState;
import fr.insee.genesis.domain.model.surveyunit.InterrogationId;
import fr.insee.genesis.domain.model.surveyunit.Mode;
import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
import fr.insee.genesis.domain.model.surveyunit.VarIdScopeTuple;
import fr.insee.genesis.domain.model.surveyunit.VariableModel;
import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort;
import fr.insee.genesis.domain.ports.spi.SurveyUnitPersistencePort;
import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService;
import fr.insee.genesis.domain.utils.GroupUtils;
import fr.insee.genesis.exceptions.GenesisException;
import fr.insee.genesis.exceptions.QuestionnaireNotFoundException;
import fr.insee.genesis.infrastructure.utils.FileUtils;
import fr.insee.modelefiliere.RawResponseDto;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Service;

import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;

@Service
@Slf4j
public class SurveyUnitService implements SurveyUnitApiPort {

    @Qualifier("surveyUnitMongoAdapter")
    private final SurveyUnitPersistencePort surveyUnitPersistencePort;

    private final QuestionnaireMetadataService metadataService;
    private final FileUtils fileUtils;

    @Autowired
    public SurveyUnitService(SurveyUnitPersistencePort surveyUnitPersistencePort, QuestionnaireMetadataService metadataService,
                             FileUtils fileUtils) {
        this.surveyUnitPersistencePort = surveyUnitPersistencePort;
        this.metadataService = metadataService;
        this.fileUtils = fileUtils;
    }

    @Override
    public void saveSurveyUnits(List<SurveyUnitModel> surveyUnitModels) {
        surveyUnitPersistencePort.saveAll(surveyUnitModels);
    }

    @Override
    public List<SurveyUnitModel> findByIdsInterrogationAndCollectionInstrument(String interrogationId, String collectionInstrumentId) {
        return surveyUnitPersistencePort.findByIds(interrogationId, collectionInstrumentId);
    }

    @Override
    public List<SurveyUnitModel> findByIdsUsualSurveyUnitAndCollectionInstrument(String usualSurveyUnitId, String collectionInstrumentId) {
        return surveyUnitPersistencePort.findByUsualSurveyUnitAndCollectionInstrumentIds(usualSurveyUnitId, collectionInstrumentId);
    }

    @Override
    public List<SurveyUnitModel> findByInterrogationId(String interrogationId) {
        return surveyUnitPersistencePort.findByInterrogationId(interrogationId);
    }

    /**
     * In this method we want to get the latest update for each variable of a survey unit
     * 1 SurveyUnitModel / Mode
     * But we need to separate the updates by mode
     * So we will calculate the latest state for a given collection mode
     * @param interrogationId : Interrogation id
     * @param collectionInstrumentId : Collection instrument id
     * @return the latest update for each variable of a survey unit
     */
    @Override
    public List<SurveyUnitModel> findLatestByIdAndByCollectionInstrumentId(String interrogationId, String collectionInstrumentId) {
        List<SurveyUnitModel> latestUpdatesbyVariables = new ArrayList<>();
        List<SurveyUnitModel> surveyUnitModels = surveyUnitPersistencePort.findByIds(interrogationId, collectionInstrumentId);
        List<Mode> modes = getDistinctsModes(surveyUnitModels);
        modes.forEach(mode ->{
            List<SurveyUnitModel> suByMode = surveyUnitModels.stream()
                    .filter(surveyUnitModel -> surveyUnitModel.getMode().equals(mode))
                    .sorted((o1, o2) -> o2.getRecordDate().compareTo(o1.getRecordDate())) //Sorting update by date (latest updates first by date of upload in database)
                    .toList();

            //We had all the variables of the oldest update
            latestUpdatesbyVariables.add(suByMode.getFirst());
            //We keep the name of already added variables to skip them in older updates
            Set<VarIdScopeTuple> addedVariables = new HashSet<>();
            SurveyUnitModel latestUpdate = suByMode.getFirst();

            if(latestUpdate.getCollectedVariables() == null){
                latestUpdate.setCollectedVariables(new ArrayList<>());
            }
            if(latestUpdate.getExternalVariables() == null){
                latestUpdate.setExternalVariables(new ArrayList<>());
            }

            latestUpdate.getCollectedVariables().forEach(colVar ->
                    addedVariables.add(new VarIdScopeTuple(colVar.varId(), colVar.scope(), colVar.iteration()))
            );
            latestUpdate.getExternalVariables().forEach(extVar ->
                    addedVariables.add(new VarIdScopeTuple(extVar.varId(), extVar.scope(), extVar.iteration())))
            ;

            suByMode.forEach(surveyUnitModel -> {
                //Get non null usualSurveyUnitId
                if (surveyUnitModel.getUsualSurveyUnitId() != null){
                    latestUpdate.setUsualSurveyUnitId(surveyUnitModel.getUsualSurveyUnitId());
                }

                List<VariableModel> collectedVariablesToKeep = new ArrayList<>();
                List<VariableModel> externalVariablesToKeep = new ArrayList<>();
                // We iterate over the variables of the update and add them to the list if they are not already added
                if (surveyUnitModel.getCollectedVariables() != null) {
                    addDataStateIntoCollectedVariables(surveyUnitModel);
                    surveyUnitModel.getCollectedVariables().stream()
                            .filter(colVar -> !addedVariables.contains(new VarIdScopeTuple(colVar.varId(), colVar.scope()
                                    , colVar.iteration())))
                            .forEach(colVar -> {
                                collectedVariablesToKeep.add(colVar);
                                addedVariables.add(new VarIdScopeTuple(colVar.varId(), colVar.scope(), colVar.iteration()));
                            });
                }
                if (surveyUnitModel.getExternalVariables() != null){
                    addDataStateIntoExternalVariables(surveyUnitModel);
                    surveyUnitModel.getExternalVariables().stream()
                         .filter(extVar -> !addedVariables.contains(new VarIdScopeTuple(extVar.varId(), extVar.scope(),
                                 extVar.iteration())))
                         .forEach(extVar -> {
                            externalVariablesToKeep.add(extVar);
                            addedVariables.add(new VarIdScopeTuple(extVar.varId(), extVar.scope(), extVar.iteration()));
                         });
                }

                // If there are new variables, we add the update to the list of latest updates
                if (!collectedVariablesToKeep.isEmpty() || !externalVariablesToKeep.isEmpty()){
                    surveyUnitModel.setCollectedVariables(collectedVariablesToKeep);
                    surveyUnitModel.setExternalVariables(externalVariablesToKeep);
                    latestUpdatesbyVariables.add(surveyUnitModel);
                }
            });
        });
        return latestUpdatesbyVariables;
    }

    private void addDataStateIntoList(List<VariableModel> variableModelList, DataState state){
        variableModelList.replaceAll(
                variableModel -> variableModel.state() == null ?
                        VariableModel.builder()
                                .varId(variableModel.varId())
                                .value(variableModel.value())
                                .state(state)
                                .scope(variableModel.scope())
                                .iteration(variableModel.iteration())
                                .parentId(variableModel.parentId())
                                .build()
                        : variableModel);
    }

    /**
     * This method adds the SurveyUnitModel dataState into its variables, as the variable Document doesn't have it
     */
    private void addDataStateIntoCollectedVariables(SurveyUnitModel surveyUnitModel) {
        addDataStateIntoList(surveyUnitModel.getCollectedVariables(), surveyUnitModel.getState());
    }

    /**
     * This method adds the SurveyUnitModel dataState into its variables, as the variable Document doesn't have it
     */
    private void addDataStateIntoExternalVariables(SurveyUnitModel surveyUnitModel) {
        addDataStateIntoList(surveyUnitModel.getExternalVariables(), surveyUnitModel.getState());
    }

    /**
     * Gets all documents of an interrogation and merges them into a single DTO
     * @return a SurveyUnitSimplifiedDto of the interrogation
     */
    @Override
    public SurveyUnitSimplifiedDto findSimplifiedByCollectionInstrumentIdAndInterrogationId(
            String collectionInstrumentId,
            String interrogationId,
            Mode mode){
        List<SurveyUnitModel> responses = findLatestByIdAndByCollectionInstrumentId(interrogationId, collectionInstrumentId);

        List<VariableModel> outputVariables = new ArrayList<>();
        List<VariableModel> outputExternalVariables = new ArrayList<>();
        RawResponseDto.QuestionnaireStateEnum questionnaireState = null;
        LocalDateTime validationDate = null;

        for (SurveyUnitModel response : responses) {
            if (!mode.equals(response.getMode())) {
                continue;
            }
            if (response.getQuestionnaireState() != null) {
                questionnaireState = response.getQuestionnaireState();
            }
            if (response.getValidationDate() != null) {
                validationDate = response.getValidationDate();
            }
            outputVariables.addAll(response.getCollectedVariables());
            outputExternalVariables.addAll(response.getExternalVariables());
        }

        SurveyUnitModel first = responses.getFirst();
        return SurveyUnitSimplifiedDto.builder()
                .collectionInstrumentId(first.getCollectionInstrumentId())
                .campaignId(first.getCampaignId())
                .interrogationId(first.getInterrogationId())
                .mode(mode)
                .usualSurveyUnitId(first.getUsualSurveyUnitId())
                .isCapturedIndirectly(first.getIsCapturedIndirectly())
                .validationDate(validationDate)
                .questionnaireState(questionnaireState)
                .variablesUpdate(outputVariables)
                .externalVariables(outputExternalVariables)
                .build();
    }

    /**
     * Gets all documents of an interrogation and merges them into DTOs
     * @return a SurveyUnitSimplifiedDto list, 1 DTO / Interrogation
     */
    @Override
    public List<SurveyUnitSimplifiedDto> findSimplifiedByCollectionInstrumentIdAndInterrogationIdList(
            String collectionInstrumentId,
            List<InterrogationId> interrogationIds
    ) {
        List<Mode> modes = findModesByCollectionInstrumentId(collectionInstrumentId);
        return interrogationIds.stream()
                .flatMap(interrogationId -> modes.stream()
                        .map(mode -> findSimplifiedByCollectionInstrumentIdAndInterrogationId(
                                collectionInstrumentId,
                                interrogationId.getInterrogationId(),
                                mode
                        ))
                )
                .filter(Objects::nonNull)
                .toList();
    }


    //========= OPTIMISATIONS PERFS (START) ==========
    /**
     * @author Adrien Marchal
     */
    @Override
    public List<List<SurveyUnitModel>> findLatestByIdAndByQuestionnaireIdAndModeOrdered(String questionnaireId, String mode,
                                                                                        List<InterrogationId> interrogationIds) {
        //return object
        List<List<SurveyUnitModel>> listLatestUpdatesbyVariables = new ArrayList<>();

        //1) QUERY
        // => conversion of "List<InterrogationId>" -> "List<String>" for query using lambda
        List<String> queryInParam = interrogationIds.stream().map(InterrogationId::getInterrogationId).toList();

        //Get !!!all versions!!! of a set of "interrogationIds"
        List<SurveyUnitModel> allResponsesVersionsSet = surveyUnitPersistencePort.findBySetOfIdsAndQuestionnaireIdAndMode(questionnaireId, mode, queryInParam);

        //2) FILTER BY interrogationId AND ORDER BY DATE (MOST RECENT FIRST, oldest last)
        interrogationIds.forEach(interrogationId -> {
            List<SurveyUnitModel> allResponsesVersionsForSingleInterrId = allResponsesVersionsSet.stream()
                            .filter(surveyUnitModel -> surveyUnitModel.getInterrogationId().equals(interrogationId.getInterrogationId()))
                            .sorted((o1, o2) -> o2.getRecordDate().compareTo(o1.getRecordDate())) //Sorting update by date (latest updates first by date of upload in database)
                            .toList();

            List<SurveyUnitModel> latestUpdates = extractLatestUpdates(allResponsesVersionsForSingleInterrId);

            //3) add to result (: keep same existing process)
            listLatestUpdatesbyVariables.add(latestUpdates);
        });

        return listLatestUpdatesbyVariables;
    }


    private List<SurveyUnitModel> extractLatestUpdates(List<SurveyUnitModel> allResponsesVersionsForSingleInterrId) {
        //return
        List<SurveyUnitModel> latestUpdatesbyVariables = new ArrayList<>();

        //We keep the name of already added variables to skip them in older updates
        List<VarIdScopeTuple> addedVariables = new ArrayList<>();

        //No useless process if there is only ONE or NONE element (performances optimisations)
        if(allResponsesVersionsForSingleInterrId.size() <= 1) {
            //We add all the variables of the LATEST update
            latestUpdatesbyVariables.add(allResponsesVersionsForSingleInterrId.getFirst());
            return latestUpdatesbyVariables;
        }

        //ELSE -> CASE WHERE THERE ARE MORE THAN ONE VERSION!
        allResponsesVersionsForSingleInterrId.forEach(surveyUnitModel -> {
            List<VariableModel> collectedVariablesToKeep = new ArrayList<>();
            List<VariableModel> externalVariablesToKeep = new ArrayList<>();
            // We iterate over the variables of the update and add them to the list if they are not already added
            if (surveyUnitModel.getCollectedVariables() != null) {
                surveyUnitModel.getCollectedVariables().stream()
                        .filter(colVar -> !addedVariables.contains(new VarIdScopeTuple(colVar.varId(), colVar.scope(), colVar.iteration())))
                        .forEach(colVar -> {
                            collectedVariablesToKeep.add(colVar);
                            addedVariables.add(new VarIdScopeTuple(colVar.varId(), colVar.scope(), colVar.iteration()));
                        });
            }
            if (surveyUnitModel.getExternalVariables() != null){
                surveyUnitModel.getExternalVariables().stream()
                        .filter(extVar -> !addedVariables.contains(new VarIdScopeTuple(extVar.varId(), extVar.scope(), extVar.iteration())))
                        .forEach(extVar -> {
                            externalVariablesToKeep.add(extVar);
                            addedVariables.add(new VarIdScopeTuple(extVar.varId(), extVar.scope(), extVar.iteration()));
                        });
            }

            // If there are new variables, we add the update to the list of latest updates
            if (!collectedVariablesToKeep.isEmpty() || !externalVariablesToKeep.isEmpty()){
                surveyUnitModel.setCollectedVariables(collectedVariablesToKeep);
                surveyUnitModel.setExternalVariables(externalVariablesToKeep);
                latestUpdatesbyVariables.add(surveyUnitModel);
            }
        });

        return latestUpdatesbyVariables;
    }
    //========= OPTIMISATIONS PERFS (END) ==========


    @Override
    public SurveyUnitDto findLatestValuesByStateByIdAndByCollectionInstrumentId(
            String interrogationId,
            String collectionInstrumentId) throws GenesisException
    {
        SurveyUnitDto surveyUnitDto = SurveyUnitDto.builder()
                .interrogationId(interrogationId)
                .collectedVariables(new ArrayList<>())
                .externalVariables(new ArrayList<>())
                .build();

        //Extract variables
        Map<VarIdScopeTuple, VariableDto> collectedVariableMap = new HashMap<>();
        Map<VarIdScopeTuple, VariableDto> externalVariableMap = new HashMap<>();
        List<SurveyUnitModel> surveyUnitModels = surveyUnitPersistencePort.findByIds(interrogationId, collectionInstrumentId);
        List<Mode> modes = getDistinctsModes(surveyUnitModels);
        for (Mode mode : modes) {
            List<SurveyUnitModel> suByMode = surveyUnitModels.stream()
                    .filter(surveyUnitModel -> surveyUnitModel.getMode().equals(mode))
                    .sorted((o1, o2) -> o2.getRecordDate().compareTo(o1.getRecordDate())) //Sorting update by date (latest updates first by date of upload in database)
                    .toList();

            VariablesMap variablesMap = null;
            for (SurveyUnitModel surveyUnitModel : suByMode) {
                if (variablesMap == null) {
                    variablesMap = metadataService.loadAndSaveIfNotExists(
                            surveyUnitModel.getCollectionInstrumentId(),
                            surveyUnitModel.getCollectionInstrumentId(),
                            surveyUnitModel.getMode(),
                            fileUtils,
                            new ArrayList<>()).getVariables();
                }
                extractVariables(surveyUnitModel, collectedVariableMap,
                        externalVariableMap, variablesMap);
            }
        }
        collectedVariableMap.keySet().forEach(variableTuple -> surveyUnitDto.getCollectedVariables().add(collectedVariableMap.get(variableTuple)));
        externalVariableMap.keySet().forEach(variableName -> surveyUnitDto.getExternalVariables().add(externalVariableMap.get(variableName)));
        return surveyUnitDto;
    }

    @Override
    public List<InterrogationId> findDistinctInterrogationIdsByQuestionnaireId(String questionnaireId) {
        List<SurveyUnitModel> surveyUnitModels = surveyUnitPersistencePort.findInterrogationIdsByCollectionInstrumentId(questionnaireId);
        List<InterrogationId> suIds = new ArrayList<>();
        surveyUnitModels.forEach(surveyUnitModel -> suIds.add(new InterrogationId(surveyUnitModel.getInterrogationId())));
        return suIds.stream().distinct().toList();
    }

    @Override
    public List<InterrogationId> findDistinctInterrogationIdsByQuestionnaireIdAndDateAfter(String questionnaireId, LocalDateTime since) {
        return  surveyUnitPersistencePort
                .findInterrogationIdsByQuestionnaireIdAndDateAfter(questionnaireId, since)
                .stream()
                .map(su -> new InterrogationId(su.getInterrogationId()))
                .distinct()
                .toList();
    }

    @Override
    public List<InterrogationId> findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(String collectionInstrumentId, LocalDateTime start, LocalDateTime end) {
        return surveyUnitPersistencePort
                .findInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(collectionInstrumentId,start,end)
                .stream()
                .map(su -> new InterrogationId(su.getInterrogationId()))
                .distinct()
                .toList();
    }

    //============ OPTIMISATIONS PERFS (START) ============

    /**
     * @author Adrien Marchal
     * Calculations made to establish the data a worker will be responsible for, among the whole data to be processed.
     * (needed for distributed process / horizontal scaling)
     */
    @Override
    public List<InterrogationId> findDistinctPageableInterrogationIdsByQuestionnaireId(String questionnaireId, long totalSize,
                                                                                       long blockSize, long page) {
        long calculatedTotalSize;
        if(totalSize == 0) {
            calculatedTotalSize = this.countResponsesByCollectionInstrumentId(questionnaireId);
        } else {
            calculatedTotalSize = totalSize;
        }

        //Check arguments
        long skip = page * blockSize;
        if(page < 0 || skip > calculatedTotalSize) {
            //return empty list
            return List.of();
        }
        long calculatedBlockSize = (skip + blockSize) > calculatedTotalSize ? calculatedTotalSize - skip : blockSize;

        List<SurveyUnitModel> surveyUnitModels = surveyUnitPersistencePort.findPageableInterrogationIdsByQuestionnaireId(questionnaireId, skip, calculatedBlockSize);
        List<InterrogationId> suIds = new ArrayList<>();
        surveyUnitModels.forEach(surveyUnitModel -> suIds.add(new InterrogationId(surveyUnitModel.getInterrogationId())));
        return suIds.stream().distinct().toList();
    }


    /**
     * @author Adrien Marchal
     */
    @Override
    public long countResponsesByCollectionInstrumentId(String collectionInstrumentId) {
        return surveyUnitPersistencePort.countByCollectionInstrumentId(collectionInstrumentId);
    }
    //=========== OPTIMISATIONS PERFS (END) =============


    @Override
    public List<SurveyUnitModel> findInterrogationIdsAndModesByQuestionnaireId(String questionnaireId) {
        List<SurveyUnitModel> surveyUnitModels = surveyUnitPersistencePort.findInterrogationIdsByCollectionInstrumentId(questionnaireId);
        return surveyUnitModels.stream().distinct().toList();
    }

    @Override
    public List<Mode> findModesByCollectionInstrumentId(String collectionInstrumentId) {
        List<SurveyUnitModel> surveyUnitModels = surveyUnitPersistencePort.findInterrogationIdsByCollectionInstrumentId(collectionInstrumentId);
        if (surveyUnitModels == null || surveyUnitModels.isEmpty()) {
            log.warn("No collectionInstrument found with id: {}", collectionInstrumentId);
            throw new QuestionnaireNotFoundException(collectionInstrumentId);
        }
        List<Mode> sources =  surveyUnitModels.stream().map(SurveyUnitModel::getMode).distinct().toList();
        return sources;
    }

    @Override
    public List<Mode> findModesByCampaignId(String campaignId) {
        List<SurveyUnitModel> surveyUnitModels = surveyUnitPersistencePort.findInterrogationIdsByCampaignId(campaignId);
        List<Mode> sources = new ArrayList<>();
        surveyUnitModels.forEach(surveyUnitModel -> sources.add(surveyUnitModel.getMode()));
        return sources.stream().distinct().toList();
    }

    //========= OPTIMISATIONS PERFS (START) ==========
    @Override
    public List<Mode> findModesByQuestionnaireIdV2(String questionnaireId) {
        List<SurveyUnitModel> surveyUnitModels = surveyUnitPersistencePort.findModesByQuestionnaireIdV2(questionnaireId);
        List<Mode> sources = new ArrayList<>();
        surveyUnitModels.forEach(surveyUnitModel -> sources.add(surveyUnitModel.getMode()));
        return sources.stream().distinct().toList();
    }

    @Override
    public List<Mode> findModesByCampaignIdV2(String campaignId) {
        List<SurveyUnitModel> surveyUnitModels = surveyUnitPersistencePort.findModesByCampaignIdV2(campaignId);
        List<Mode> sources = new ArrayList<>();
        surveyUnitModels.forEach(surveyUnitModel -> sources.add(surveyUnitModel.getMode()));
        return sources.stream().distinct().toList();
    }
    //========= OPTIMISATIONS PERFS (END) ==========

    @Override
    public Long deleteByCollectionInstrumentId(String collectionInstrumentId) {
        return surveyUnitPersistencePort.deleteByCollectionInstrumentId(collectionInstrumentId);
    }

    @Override
    public long countResponses() {
        return surveyUnitPersistencePort.count();
    }

    @Override
    public Set<String> findQuestionnaireIdsByCampaignId(String campaignId) {
            return surveyUnitPersistencePort.findQuestionnaireIdsByCampaignId(campaignId);
    }

    //========= OPTIMISATIONS PERFS (START) ==========
    /**
     * @author Adrien Marchal
     */
    @Override
    public Set<String> findQuestionnaireIdsByCampaignIdV2(String campaignId) {
        return surveyUnitPersistencePort.findQuestionnaireIdsByCampaignIdV2(campaignId);
    }
    //========= OPTIMISATIONS PERFS (END) ==========

    @Override
    public Set<String> findDistinctCampaignIds() {
        return surveyUnitPersistencePort.findDistinctCampaignIds();
    }

    @Override
    public List<CampaignWithQuestionnaire> findCampaignsWithQuestionnaires() {
        List<CampaignWithQuestionnaire> campaignsWithQuestionnaireList = new ArrayList<>();
        for(String campaignId : findDistinctCampaignIds()){
            Set<String> questionnaires = findQuestionnaireIdsByCampaignId(campaignId);
            campaignsWithQuestionnaireList.add(new CampaignWithQuestionnaire(campaignId,questionnaires));
        }
        return campaignsWithQuestionnaireList;
    }

    @Override
    public long countResponsesByCampaignId(String campaignId){
        return surveyUnitPersistencePort.countByCampaignId(campaignId);
    }

    @Override
    public Set<String> findDistinctQuestionnairesAndCollectionInstrumentIds() {
        return surveyUnitPersistencePort.findDistinctQuestionnairesAndCollectionInstrumentIds();
    }

    @Override
    public List<QuestionnaireWithCampaign> findQuestionnairesWithCampaigns() {
        List<QuestionnaireWithCampaign> questionnaireWithCampaignList = new ArrayList<>();
        for(String questionnaireId : findDistinctQuestionnairesAndCollectionInstrumentIds()){
            Set<String> campaigns = surveyUnitPersistencePort.findCampaignIdsByQuestionnaireId(questionnaireId);
            questionnaireWithCampaignList.add(new QuestionnaireWithCampaign(
                    questionnaireId,
                    campaigns)
            );

        }
        return questionnaireWithCampaignList;
    }

    @Override
    public List<SurveyUnitModel> parseEditedVariables(
            SurveyUnitInputDto surveyUnitInputDto,
            String userIdentifier,
            VariablesMap variablesMap
    ) throws GenesisException {

        List<DataState> statesReceived = surveyUnitInputDto.getCollectedVariables().stream()
                .map(colVar -> colVar.getVariableStateInputDto().getState())
                .distinct()
                .toList();

        if (statesReceived.contains(DataState.COLLECTED)){
            throw new GenesisException(400,"You can not persist in database a new value with the state COLLECTED");
        }

        List<SurveyUnitModel> surveyUnitModels = new ArrayList<>();

        for (DataState state : statesReceived){
            SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder()
                    .campaignId(surveyUnitInputDto.getCampaignId())
                    .mode(surveyUnitInputDto.getMode())
                    .collectionInstrumentId(surveyUnitInputDto.getQuestionnaireId().toUpperCase())
                    .interrogationId(surveyUnitInputDto.getInterrogationId())
                    .state(state)
                    .recordDate(LocalDateTime.now())
                    .collectedVariables(new ArrayList<>())
                    .externalVariables(new ArrayList<>())
                    .modifiedBy(userIdentifier)
                    .build();

            //Keep only variable dtos who has the corresponding state
            List<VariableInputDto> editedCollectedVariables = surveyUnitInputDto.getCollectedVariables().stream()
                    .filter(colVar -> colVar.getVariableStateInputDto().getState() == state).toList();

            //Collected variables management
            for(VariableInputDto editedVariableDto : editedCollectedVariables){
                VariableModel collectedVariable = VariableModel.builder()
                        .varId(editedVariableDto.getVariableName())
                        .value(editedVariableDto.getVariableStateInputDto().getValue() == null ?
                                null
                                : editedVariableDto.getVariableStateInputDto().getValue().toString()
                            )
                        .parentId(GroupUtils.getParentGroupName(editedVariableDto.getVariableName(), variablesMap))
                        .scope(variablesMap.getVariable(editedVariableDto.getVariableName()).getGroupName())
                        .iteration(editedVariableDto.getIteration())
                        .build();

                surveyUnitModel.getCollectedVariables().add(collectedVariable);

            }
            surveyUnitModels.add(surveyUnitModel);
        }

        return surveyUnitModels;
    }

    @Override
    public String findQuestionnaireIdByInterrogationId(String interrogationId) throws GenesisException {
        List<SurveyUnitModel> surveyUnitModels = surveyUnitPersistencePort.findByInterrogationId(interrogationId);
        if (surveyUnitModels.isEmpty()){
            throw new GenesisException(404,String.format("The interrogationId %s is not in database",interrogationId));
        }
        Set<String> questionnaireIds = new HashSet<>();
        for(SurveyUnitModel surveyUnitModel : surveyUnitModels){
            questionnaireIds.add(surveyUnitModel.getCollectionInstrumentId());
        }
        if(questionnaireIds.size() > 1){
            throw new GenesisException(207,String.format("Multiple questionnaires for %s :%n%s",
                    interrogationId,
                    String.join("\n", questionnaireIds)
            ));
        }

        return questionnaireIds.iterator().next(); //Return first (and supposed only) element of set
    }

    @Override
    public Set<String> findCampaignIdsFrom(SurveyUnitInputDto dto) {
        List<SurveyUnitModel> responses = findByIdsInterrogationAndCollectionInstrument(
                dto.getInterrogationId(),
                dto.getQuestionnaireId()
        );
        return responses.stream()
                .map(SurveyUnitModel::getCampaignId)
                .collect(Collectors.toSet());
    }

    @Override
    public long countResponsesByQuestionnaireId(String questionnaireId) {
        return surveyUnitPersistencePort.countByQuestionnaireId(questionnaireId);
    }

    //Utils
    private static List<Mode> getDistinctsModes(List<SurveyUnitModel> surveyUnitModels) {
        List<Mode> sources = new ArrayList<>();
        surveyUnitModels.forEach(surveyUnitModel -> sources.add(surveyUnitModel.getMode()));
        return sources.stream().distinct().toList();
    }

    /**
     * Extract collected variables from a model class to a variable map
     * @param surveyUnitModel survey unit model
     * @param collectedVariableMap Collected variable DTO map to populate
     * @param externalVariableMap External variable DTO map to populate
     */
    private void extractVariables(SurveyUnitModel surveyUnitModel,
                                  Map<VarIdScopeTuple, VariableDto> collectedVariableMap,
                                  Map<VarIdScopeTuple, VariableDto> externalVariableMap,
                                  VariablesMap variablesMap
                                  ) {

        if(surveyUnitModel.getCollectedVariables() == null){
            surveyUnitModel.setCollectedVariables(new ArrayList<>());
        }
        for (VariableModel collectedVariable : surveyUnitModel.getCollectedVariables()) {
            VarIdScopeTuple loopIdTuple = new VarIdScopeTuple(collectedVariable.varId(), collectedVariable.scope(),
                    collectedVariable.iteration());

            VariableDto variableDto = collectedVariableMap.computeIfAbsent(loopIdTuple, k ->
                    VariableDto.builder()
                            .variableName(collectedVariable.varId())
                            .scope(collectedVariable.scope())
                            .iteration(collectedVariable.iteration())
                            .variableStateDtoList(new ArrayList<>())
                            .build()
            );

            //Extract variable state
            if (isMostRecentForSameState(surveyUnitModel, variableDto)) {
                variableDto.getVariableStateDtoList().add(
                        VariableStateDto.builder()
                                .state(surveyUnitModel.getState())
                                .active(isLastVariableState(surveyUnitModel, variableDto))
                                .value(getValueWithType(
                                        collectedVariable.varId(),
                                        collectedVariable.value(),
                                        variablesMap))
                                .date(surveyUnitModel.getRecordDate())
                                .build()
                );
            }
        }

        if(surveyUnitModel.getExternalVariables() == null){
            surveyUnitModel.setExternalVariables(new ArrayList<>());
        }
        for(VariableModel externalVariable : surveyUnitModel.getExternalVariables()){
            VarIdScopeTuple loopIdTuple = new VarIdScopeTuple(externalVariable.varId(), externalVariable.scope(), externalVariable.iteration());

            VariableDto variableDto = externalVariableMap.computeIfAbsent(loopIdTuple, k ->
                    VariableDto.builder()
                            .variableName(externalVariable.varId())
                            .scope(externalVariable.scope())
                            .iteration(externalVariable.iteration())
                            .variableStateDtoList(new ArrayList<>())
                            .build()
            );


            //Extract variable state
            if(isMostRecentForSameState(surveyUnitModel, variableDto)){
                variableDto.getVariableStateDtoList().add(
                        VariableStateDto.builder()
                                .state(surveyUnitModel.getState())
                                .active(isLastVariableState(surveyUnitModel, variableDto))
                                .value(getValueWithType(
                                        externalVariable.varId(),
                                        externalVariable.value(),
                                        variablesMap))
                                .date(surveyUnitModel.getRecordDate())
                                .build()
                );
            }
        }
    }

    private Object getValueWithType(String variableName, String value, VariablesMap variablesMap) {
        if(!variablesMap.hasVariable(variableName)){
            log.warn("Variable {} not found in variableMap", variableName);
            return value;
        }
        if(value == null) return null;
        if(value.isEmpty()) return value;
        VariableType variableType = variablesMap.getVariable(variableName).getType();
        try {
            switch (variableType) {
                case INTEGER -> {
                    return Long.parseLong(value);
                }
                case BOOLEAN -> {
                    return Boolean.parseBoolean(value);
                }
                case NUMBER -> {
                    return Double.parseDouble(value);
                }
                default -> {
                    return value;
                }
            }
        }catch (NumberFormatException e){
            log.warn("Invalid value {} for variable {}, expected type {}", value, variableName, variableType);
            return value;
        }
    }


    /**
     * Check if there is any other more recent variable value for a same state in Variable DTO
     * @param surveyUnitModel model containing variable
     * @param variableDto DTO to check in
     * @return true if there's no more recent variable for the same state
     */
    private boolean isMostRecentForSameState(SurveyUnitModel surveyUnitModel, VariableDto variableDto) {
        List<VariableStateDto> variableStatesSameState = variableDto.getVariableStateDtoList().stream().filter(
                variableStateDto -> variableStateDto.getState().equals(surveyUnitModel.getState())
        )
                .sorted((o1, o2) -> o2.getDate().compareTo(o1.getDate()))
                .toList();
        if(variableStatesSameState.isEmpty()){
            //Variable doesn't contain state
            return true;
        }
        LocalDateTime mostRecentStateDateTime = variableStatesSameState.getFirst().getDate();
        return mostRecentStateDateTime.isBefore(surveyUnitModel.getRecordDate());
    }

    /**
     * Check if model is more recent that any variable state in variable DTO regardless of state
     * Used for active variable
     * @param surveyUnitModel model used to compare
     * @param variableDto variable to check
     * @return false if there is any variable state that comes from a more recent model already
     */
    private boolean isLastVariableState(SurveyUnitModel surveyUnitModel, VariableDto variableDto) {
        for(VariableStateDto variableStateDTO : variableDto.getVariableStateDtoList()){
            if(variableStateDTO.getDate().isAfter(surveyUnitModel.getRecordDate())){
                return false;
            }
            variableStateDTO.setActive(false);
        }
        return true;
    }

    @Override
    public long countDistinctInterrogationIdsByQuestionnaireAndCollectionInstrumentId(String id) {
        return surveyUnitPersistencePort.countDistinctInterrogationIdsByQuestionnaireAndCollectionInstrumentId(id);
    }
}