diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml
index c9fe9de1c..d510a114c 100644
--- a/.github/workflows/docker.yaml
+++ b/.github/workflows/docker.yaml
@@ -3,7 +3,8 @@ name: Build snapshot docker image
on:
push:
branches-ignore:
- - main
+ - * # main
+ # turned off for now (needs secrets renewal)
jobs:
build-snapshot:
diff --git a/src/main/java/fr/insee/genesis/controller/exception/ExceptionController.java b/src/main/java/fr/insee/genesis/controller/exception/ExceptionController.java
new file mode 100644
index 000000000..0be8fdb55
--- /dev/null
+++ b/src/main/java/fr/insee/genesis/controller/exception/ExceptionController.java
@@ -0,0 +1,75 @@
+package fr.insee.genesis.controller.exception;
+
+import fr.insee.genesis.exceptions.*;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ProblemDetail;
+import org.springframework.web.bind.annotation.ControllerAdvice;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+
+/**
+ * This controller uses Spring's ControllerAdvice annotation to intercept exceptions.
+ * It implements the RFC 9457 by returning
+ * Spring's ProblemDetail object.
+ */
+@ControllerAdvice
+@Slf4j
+public class ExceptionController {
+
+ // Note: No handler for uncaught Exception.class for now since it breaks soms tests.
+
+ @ExceptionHandler
+ public ProblemDetail handleGenericGenesisException(GenesisException genesisException) {
+ log.error("GenesisException: {}", genesisException.getMessage(), genesisException);
+ return ProblemDetail.forStatusAndDetail(
+ resolveHttpCode(genesisException.getStatus()),
+ genesisException.getMessage());
+ }
+
+ /** Returns the corresponding http status, or 500 if the given code does not match a http status. */
+ private static HttpStatus resolveHttpCode(int statusCode) {
+ HttpStatus httpStatus = HttpStatus.resolve(statusCode);
+ return httpStatus != null ? httpStatus : HttpStatus.INTERNAL_SERVER_ERROR;
+ }
+
+ @ExceptionHandler(InvalidDateIntervalException.class)
+ public ProblemDetail handleInvalidDateIntervalException(InvalidDateIntervalException e) {
+ log.error("InvalidDateIntervalException: {}", e.getMessage());
+ return ProblemDetail.forStatusAndDetail(
+ HttpStatus.BAD_REQUEST,
+ e.getMessage());
+ }
+
+ @ExceptionHandler(ModesConflictException.class)
+ public ProblemDetail handleModesConflictException(ModesConflictException e) {
+ log.error("ModesConflictException: {}", e.getMessage());
+ return ProblemDetail.forStatusAndDetail(
+ HttpStatus.CONFLICT,
+ e.getMessage());
+ }
+
+ @ExceptionHandler(UndefinedModesException.class)
+ public ProblemDetail handleUndefinedModesException(UndefinedModesException e) {
+ log.error("UndefinedModesException: {}", e.getMessage());
+ return ProblemDetail.forStatusAndDetail(
+ HttpStatus.NOT_FOUND,
+ e.getMessage());
+ }
+
+ @ExceptionHandler(UndefinedMetadataException.class)
+ public ProblemDetail handleUndefinedMetadataException(UndefinedMetadataException e) {
+ log.error("UndefinedMetadataException: {}", e.getMessage());
+ return ProblemDetail.forStatusAndDetail(
+ HttpStatus.NOT_FOUND,
+ e.getMessage());
+ }
+
+ @ExceptionHandler(InvalidMetadataException.class)
+ public ProblemDetail handleInvalidMetadataException(InvalidMetadataException e) {
+ log.error("InvalidMetadataException: {}", e.getMessage());
+ return ProblemDetail.forStatusAndDetail(
+ HttpStatus.BAD_REQUEST,
+ e.getMessage());
+ }
+
+}
diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java
index 157d38e16..697a377fb 100644
--- a/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java
+++ b/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java
@@ -15,6 +15,7 @@
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
+import java.time.Instant;
import java.time.LocalDateTime;
import java.util.List;
@@ -55,17 +56,17 @@ public ResponseEntity> getAllInterrogationIdsByCollectionI
@DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
@Parameter(
description = "sinceDate",
- schema = @Schema(type = "string", format = "date-time", example = "2026-01-01T00:00:00")
+ schema = @Schema(type = "string", format = "date-time", example = "2026-01-01T00:00:00Z")
)
- LocalDateTime start,
+ Instant start,
@RequestParam("end")
@DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
@Parameter(
description = "untilDate",
- schema = @Schema(type = "string", format = "date-time", example = "2026-01-31T23:59:59")
+ schema = @Schema(type = "string", format = "date-time", example = "2026-01-31T23:59:59Z")
)
- LocalDateTime end) {
+ Instant end) {
List responses = surveyUnitService.findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(collectionInstrumentId, start,end);
return ResponseEntity.ok(responses);
}
diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java
index e692852e3..f79efe4cb 100644
--- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java
+++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java
@@ -15,6 +15,7 @@
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import jakarta.validation.Valid;
+import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
@@ -43,23 +44,16 @@
@Slf4j
@Controller
+@RequiredArgsConstructor
public class RawResponseController {
private static final String SUCCESS_MESSAGE = "Interrogation %s saved";
private static final String INTERROGATION_ID = "interrogationId";
- public static final String NB_DOCS_WITH_FORMATTED = "%d document(s) processed, including %d FORMATTED after data verification for collectionInstrumentId %s";
- public static final String NB_DOCS = "%d document(s) processed for collectionInstrumentId %s";
+
private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort;
private final RawResponseApiPort rawResponseApiPort;
private final RawResponseInputRepository rawRepository;
-
- public RawResponseController(LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort, RawResponseApiPort rawResponseApiPort, RawResponseInputRepository rawRepository) {
- this.lunaticJsonRawDataApiPort = lunaticJsonRawDataApiPort;
- this.rawResponseApiPort = rawResponseApiPort;
- this.rawRepository = rawRepository;
- }
-
@Operation(summary = "Save lunatic json data from one interrogation in Genesis Database")
@PutMapping(path = "/responses/raw/lunatic-json/save")
@PreAuthorize("hasRole('COLLECT_PLATFORM')")
@@ -118,11 +112,8 @@ public ResponseEntity processRawResponses(
log.info("Try to process raw responses for collectionInstrumentId {} and {} interrogationIds", collectionInstrumentId, interrogationIdList.size());
List errors = new ArrayList<>();
try {
- DataProcessResult result = rawResponseApiPort.processRawResponses(collectionInstrumentId, interrogationIdList, errors);
- return result.formattedDataCount() == 0 ?
- ResponseEntity.ok(NB_DOCS.formatted(result.dataCount(), collectionInstrumentId))
- : ResponseEntity.ok(NB_DOCS_WITH_FORMATTED
- .formatted(result.dataCount(), result.formattedDataCount(), collectionInstrumentId));
+ DataProcessResult result = rawResponseApiPort.processRawResponsesByInterrogationIds(collectionInstrumentId, interrogationIdList, errors);
+ return ResponseEntity.ok(result.message(collectionInstrumentId));
} catch (GenesisException e) {
return ResponseEntity.status(e.getStatus()).body(e.getMessage());
}
@@ -140,11 +131,8 @@ public ResponseEntity processRawResponsesByCollectionInstrumentId(
) {
log.info("Try to process raw responses for collectionInstrumentId {}", collectionInstrumentId);
try {
- DataProcessResult result = rawResponseApiPort.processRawResponses(collectionInstrumentId);
- return result.formattedDataCount() == 0 ?
- ResponseEntity.ok(NB_DOCS.formatted(result.dataCount(), collectionInstrumentId))
- : ResponseEntity.ok(NB_DOCS_WITH_FORMATTED
- .formatted(result.dataCount(), result.formattedDataCount(), collectionInstrumentId));
+ DataProcessResult result = rawResponseApiPort.processRawResponsesByInterrogationIds(collectionInstrumentId);
+ return ResponseEntity.ok(result.message(collectionInstrumentId));
} catch (GenesisException e) {
return ResponseEntity.status(e.getStatus()).body(e.getMessage());
}
@@ -183,7 +171,7 @@ public ResponseEntity getJsonRawData(
@RequestParam("campaignName") String campaignName,
@RequestParam(value = "mode") Mode modeSpecified
) {
- List data = lunaticJsonRawDataApiPort.getRawData(campaignName, modeSpecified, List.of(interrogationId));
+ List data = lunaticJsonRawDataApiPort.getRawDataByQuestionnaireId(campaignName, modeSpecified, List.of(interrogationId));
return ResponseEntity.ok(data.getFirst());
}
@@ -201,7 +189,7 @@ public ResponseEntity processJsonRawData(
List errors = new ArrayList<>();
try {
- DataProcessResult result = lunaticJsonRawDataApiPort.processRawData(campaignName, interrogationIdList, errors);
+ DataProcessResult result = lunaticJsonRawDataApiPort.processRawDataByInterrogationIds(campaignName, interrogationIdList, errors);
return result.formattedDataCount() == 0 ?
ResponseEntity.ok("%d document(s) processed".formatted(result.dataCount()))
: ResponseEntity.ok("%d document(s) processed, including %d FORMATTED after data verification"
diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseReprocessController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseReprocessController.java
new file mode 100644
index 000000000..2f9d87250
--- /dev/null
+++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseReprocessController.java
@@ -0,0 +1,102 @@
+package fr.insee.genesis.controller.rest.responses;
+
+import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
+import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType;
+import fr.insee.genesis.domain.ports.api.ReprocessRawResponseApiPort;
+import fr.insee.genesis.exceptions.GenesisException;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.format.annotation.DateTimeFormat;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+
+import java.time.Instant;
+
+@Controller
+@RequiredArgsConstructor
+@Slf4j
+public class RawResponseReprocessController {
+
+ private final ReprocessRawResponseApiPort reprocessRawResponseApiPort;
+
+ @Operation(summary = "Reprocess raw response of a collection instrument.")
+ @PostMapping(path = "/raw-responses/{collectionInstrumentId}/reprocess")
+ @PreAuthorize("hasRole('ADMIN')")
+ public ResponseEntity reProcessRawResponsesByCollectionInstrumentId(
+ @Parameter(
+ description = "Id of the collection instrument (old questionnaireId)",
+ example = "ENQTEST2025X00")
+ @PathVariable("collectionInstrumentId")
+ String collectionInstrumentId,
+
+ @Parameter(
+ description = "Extract since",
+ schema = @Schema(type = "string", format = "date-time", example = "2026-01-01T00:00:00Z")
+ )
+ @RequestParam(value = "sinceDate", required = false)
+ @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
+ Instant sinceDate,
+
+ @Parameter(
+ description = "Extract until",
+ schema = @Schema(type = "string", format = "date-time", example = "2026-02-02T00:00:00Z")
+ )
+ @RequestParam(value = "endDate", required = false)
+ @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
+ Instant endDate
+ ) throws GenesisException {
+
+ DataProcessResult result = reprocessRawResponseApiPort.reprocessRawResponses(
+ RawDataModelType.FILIERE,
+ collectionInstrumentId,
+ sinceDate,
+ endDate);
+
+ return ResponseEntity.ok(result.message(collectionInstrumentId));
+ }
+
+ @Operation(summary = "Reprocess Lunatic raw data for a questionnaire model. " +
+ "**Note**: Lunatic raw data is the legacy format of raw responses.")
+ @PostMapping(path = "/responses/raw/lunatic-json/{questionnaireId}/reprocess")
+ @PreAuthorize("hasRole('ADMIN')")
+ public ResponseEntity reProcessJsonRawDataByQuestionnaireId(
+ @Parameter(
+ description = "Questionnaire model id (old name for collection instrument id).",
+ example = "ENQTEST2025X00")
+ @PathVariable("questionnaireId")
+ String collectionInstrumentId, // 'questionnaireId' is the legacy name for 'collectionInstrumentId'
+
+ @Parameter(
+ description = "Extract since",
+ schema = @Schema(type = "string", format = "date-time", example = "2026-01-01T00:00:00Z")
+ )
+ @RequestParam(value = "sinceDate", required = false)
+ @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
+ Instant sinceDate,
+
+ @Parameter(
+ description = "Extract until",
+ schema = @Schema(type = "string", format = "date-time", example = "2026-02-02T00:00:00Z")
+ )
+ @RequestParam(value = "endDate", required = false)
+ @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
+ Instant endDate
+ ) throws GenesisException {
+
+ DataProcessResult result = reprocessRawResponseApiPort.reprocessRawResponses(
+ RawDataModelType.LEGACY,
+ collectionInstrumentId,
+ sinceDate,
+ endDate);
+
+ return ResponseEntity.ok(result.message(collectionInstrumentId));
+ }
+
+}
diff --git a/src/main/java/fr/insee/genesis/controller/utils/ControllerUtils.java b/src/main/java/fr/insee/genesis/controller/utils/ControllerUtils.java
index 83051a4e1..00398e9f5 100644
--- a/src/main/java/fr/insee/genesis/controller/utils/ControllerUtils.java
+++ b/src/main/java/fr/insee/genesis/controller/utils/ControllerUtils.java
@@ -1,16 +1,18 @@
package fr.insee.genesis.controller.utils;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+import fr.insee.genesis.exceptions.ModesConflictException;
+import fr.insee.genesis.exceptions.UndefinedModesException;
+import fr.insee.genesis.infrastructure.utils.FileUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
-import fr.insee.genesis.domain.model.surveyunit.Mode;
-import fr.insee.genesis.exceptions.GenesisException;
-import fr.insee.genesis.infrastructure.utils.FileUtils;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+// Note: this class should be moved in the domain service layer.
@Component
@Slf4j
@@ -23,25 +25,23 @@ public ControllerUtils(FileUtils fileUtils) {
this.fileUtils = fileUtils;
}
-
/**
* If a mode is specified, we treat only this mode.
- * If no mode is specified, we treat all modes in the campaign.
+ * If no mode is specified, we treat all modes in the questionnaireId.
* If no mode is specified and no specs are found, we return an error
- * @param campaign campaign id to get modes
+ * @param questionnaireId questionnaireId id to get modes
* @param modeSpecified a Mode to use, null if we want all modes available
* @return a list with the mode in modeSpecified or all modes if null
- * @throws GenesisException if error in specs structure
*/
- public List getModesList(String campaign, Mode modeSpecified) throws GenesisException {
+ public List getModesList(String questionnaireId, Mode modeSpecified) {
if (modeSpecified != null){
return Collections.singletonList(modeSpecified);
}
List modes = new ArrayList<>();
- String specFolder = fileUtils.getSpecFolder(campaign);
+ String specFolder = fileUtils.getSpecFolder(questionnaireId);
List modeSpecFolders = fileUtils.listFolders(specFolder);
if (modeSpecFolders.isEmpty()) {
- throw new GenesisException(404, "No specification folder found " + specFolder);
+ throw new UndefinedModesException("No specification folder found " + specFolder);
}
for(String modeSpecFolder : modeSpecFolders){
if(Mode.getEnumFromModeName(modeSpecFolder) == null) {
@@ -51,9 +51,18 @@ public List getModesList(String campaign, Mode modeSpecified) throws Genes
modes.add(Mode.getEnumFromModeName(modeSpecFolder));
}
if (modes.contains(Mode.F2F) && modes.contains(Mode.TEL)) {
- throw new GenesisException(409, "Cannot treat simultaneously TEL and FAF modes");
+ throw new ModesConflictException("Cannot treat simultaneously TEL and FAF modes");
}
return modes;
}
+ /**
+ * Returns the applicable modes for the collection instrument with the given identifier.
+ * @param collectionInstrumentId Collection instrument identifier.
+ * @return A list of modes.
+ */
+ public List getModesList(String collectionInstrumentId) {
+ return getModesList(collectionInstrumentId, null);
+ }
+
}
diff --git a/src/main/java/fr/insee/genesis/domain/model/context/DataProcessingContextModel.java b/src/main/java/fr/insee/genesis/domain/model/context/DataProcessingContextModel.java
index d0a147469..20775dca2 100644
--- a/src/main/java/fr/insee/genesis/domain/model/context/DataProcessingContextModel.java
+++ b/src/main/java/fr/insee/genesis/domain/model/context/DataProcessingContextModel.java
@@ -17,18 +17,22 @@
@NoArgsConstructor
@AllArgsConstructor
public class DataProcessingContextModel {
+
+ /** (Added to the class only to remove a warning) */
@Id
- private ObjectId id; //Used to remove warning
+ private ObjectId id;
@Deprecated(forRemoval = true)
private String partitionId;
- private String collectionInstrumentId; //QuestionnaireId
+ /** New name of legacy 'questionnaireId' property. */
+ private String collectionInstrumentId;
private LocalDateTime lastExecution;
List kraftwerkExecutionScheduleList;
+ /** Determines if some review service must be called during the process. */
boolean withReview;
public ScheduleDto toScheduleDto(){
@@ -39,4 +43,5 @@ public ScheduleDto toScheduleDto(){
.kraftwerkExecutionScheduleList(kraftwerkExecutionScheduleList)
.build();
}
+
}
diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/DataProcessResult.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/DataProcessResult.java
index b8191dd03..e7eccec1c 100644
--- a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/DataProcessResult.java
+++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/DataProcessResult.java
@@ -4,5 +4,31 @@
import java.util.List;
-public record DataProcessResult(int dataCount, int formattedDataCount, List errors) {
+public record DataProcessResult(
+ int dataCount,
+ int formattedDataCount,
+ List errors) {
+
+ public String message(String collectionInstrumentId) {
+ return String.format("%s%s%s.",
+ interrogationCountMessage(dataCount),
+ formattedCountMessage(formattedDataCount),
+ collectionIdMessage(collectionInstrumentId));
+ }
+
+ private static String interrogationCountMessage(int processedInterrogationsCount) {
+ boolean plural = processedInterrogationsCount > 1;
+ return "%d interrogation%s processed".formatted(processedInterrogationsCount, plural ? "s" : "");
+ }
+
+ private static String collectionIdMessage(String collectionInstrumentId) {
+ return " for collectionInstrumentId '%s'".formatted(collectionInstrumentId);
+ }
+
+ private static String formattedCountMessage(int formattedCount) {
+ if (formattedCount == 0)
+ return "";
+ return " (including %d FORMATTED after data verification)".formatted(formattedCount);
+ }
+
}
diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawDataModelType.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawDataModelType.java
index adb8a88cd..9ecc12db9 100644
--- a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawDataModelType.java
+++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawDataModelType.java
@@ -1,5 +1,20 @@
package fr.insee.genesis.domain.model.surveyunit.rawdata;
+/** Format of raw data to be imported into the data storage. */
public enum RawDataModelType {
- DEFAULT, FILIERE
+
+ /** Legacy format of raw data ('Lunatic'). */
+ LEGACY,
+
+ /** 'Filière' raw response model. */
+ FILIERE;
+
+ @Override
+ public String toString() {
+ return switch (this) {
+ case LEGACY -> "LEGACY (Lunatic)";
+ case FILIERE -> "FILIERE raw responses";
+ };
+ }
+
}
diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java
index 2031d80d7..1e3aa96c4 100644
--- a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java
+++ b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java
@@ -11,8 +11,8 @@
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
-import java.time.LocalDateTime;
import java.time.Instant;
+import java.time.LocalDateTime;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -20,9 +20,9 @@
public interface LunaticJsonRawDataApiPort {
void save(LunaticJsonRawDataModel rawData);
- List getRawData(String campaignName, Mode mode, List interrogationIdList);
List getRawDataByQuestionnaireId(String questionnaireId, Mode mode, List interrogationIdList);
List convertRawData(List rawData, VariablesMap variablesMap);
+
List getUnprocessedDataIds();
Set getUnprocessedDataQuestionnaireIds();
void updateProcessDates(List surveyUnitModels);
@@ -33,8 +33,10 @@ public interface LunaticJsonRawDataApiPort {
List getRawDataByInterrogationId(String interrogationId);
- @Deprecated(since = "1.13.0")
- DataProcessResult processRawData(String campaignName, List interrogationIdList, List errors) throws GenesisException;
+ /**
+ * @deprecated Use the method with 'collectionInstrumentId' instead.
+ */
+ DataProcessResult processRawDataByInterrogationIds(String campaignName, List interrogationIdList, List errors) throws GenesisException;
DataProcessResult processRawData(String collectionInstrumentId) throws GenesisException;
diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java
index 464295a8f..d7e65ee7a 100644
--- a/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java
+++ b/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java
@@ -1,7 +1,6 @@
package fr.insee.genesis.domain.ports.api;
import fr.insee.bpm.metadata.model.VariablesMap;
-import fr.insee.genesis.domain.model.surveyunit.Mode;
import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel;
@@ -16,10 +15,9 @@
public interface RawResponseApiPort {
- List getRawResponses(String collectionInstrumentId, Mode mode, List interrogationIdList);
- List getRawResponsesByInterrogationID(String interrogationId);
- DataProcessResult processRawResponses(String collectionInstrumentId, List interrogationIdList, List errors) throws GenesisException;
- DataProcessResult processRawResponses(String collectionInstrumentId) throws GenesisException;
+ DataProcessResult processRawResponsesByInterrogationIds(String collectionInstrumentId, List interrogationIdList, List errors) throws GenesisException;
+ DataProcessResult processRawResponsesByInterrogationIds(String collectionInstrumentId) throws GenesisException;
+
List convertRawResponse(List rawResponses, VariablesMap variablesMap);
List getUnprocessedCollectionInstrumentIds();
void updateProcessDates(List surveyUnitModels);
diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/ReprocessRawResponseApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/ReprocessRawResponseApiPort.java
new file mode 100644
index 000000000..c379ed637
--- /dev/null
+++ b/src/main/java/fr/insee/genesis/domain/ports/api/ReprocessRawResponseApiPort.java
@@ -0,0 +1,26 @@
+package fr.insee.genesis.domain.ports.api;
+
+import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
+import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType;
+import fr.insee.genesis.exceptions.GenesisException;
+
+import java.time.Instant;
+
+public interface ReprocessRawResponseApiPort {
+
+ /**
+ * Reprocesses raw data of the collection that correspond to the given identifier.
+ * An optional date interval can be given to reprocess a subset of the collection.
+ * @param rawDataModelType {@link RawDataModelType}
+ * @param collectionInstrumentId Collection instrument identifier.
+ * @param sinceDate Start of the date interval.
+ * @param endDate End of the date interval.
+ * @return Data processing result record.
+ * @see DataProcessResult
+ */
+ DataProcessResult reprocessRawResponses(
+ RawDataModelType rawDataModelType,
+ String collectionInstrumentId, Instant sinceDate, Instant endDate)
+ throws GenesisException;
+
+}
diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java
index 7362cc512..77769696d 100644
--- a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java
+++ b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java
@@ -1,16 +1,13 @@
package fr.insee.genesis.domain.ports.api;
import fr.insee.bpm.metadata.model.VariablesMap;
-import fr.insee.genesis.controller.dto.CampaignWithQuestionnaire;
-import fr.insee.genesis.controller.dto.QuestionnaireWithCampaign;
-import fr.insee.genesis.controller.dto.SurveyUnitDto;
-import fr.insee.genesis.controller.dto.SurveyUnitInputDto;
-import fr.insee.genesis.controller.dto.SurveyUnitSimplifiedDto;
+import fr.insee.genesis.controller.dto.*;
import fr.insee.genesis.domain.model.surveyunit.InterrogationId;
import fr.insee.genesis.domain.model.surveyunit.Mode;
import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
import fr.insee.genesis.exceptions.GenesisException;
+import java.time.Instant;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Set;
@@ -52,7 +49,7 @@ List findSimplifiedByCollectionInstrumentIdAndInterroga
List findDistinctInterrogationIdsByQuestionnaireIdAndDateAfter(String questionnaireId, LocalDateTime since);
- List findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(String collectionInstrumentId, LocalDateTime start, LocalDateTime end);
+ List findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(String collectionInstrumentId, Instant start, Instant end);
//========= OPTIMISATIONS PERFS (START) ==========
long countResponsesByCollectionInstrumentId(String questionnaireId);
@@ -73,6 +70,11 @@ List findDistinctPageableInterrogationIdsByQuestionnaireId(Stri
Long deleteByCollectionInstrumentId(String collectionInstrumentId);
+ Long deleteByQuestionnaireIdAndInterrogationIds(
+ String questionnaireId,
+ Set interrogationIds
+ );
+
long countResponses();
Set findQuestionnaireIdsByCampaignId(String campaignId);
diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java
index e8bbadb3a..cc2f355e9 100644
--- a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java
+++ b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java
@@ -14,12 +14,12 @@
public interface LunaticJsonRawDataPersistencePort {
void save(LunaticJsonRawDataModel rawData);
- List findRawData(String campaignName, Mode mode, List interrogationIdList);
List findRawDataByQuestionnaireId(String questionnaireId, Mode mode, List interrogationIdList);
Page findRawDataByQuestionnaireId(String questionnaireId, Pageable pageable);
List findRawDataByInterrogationID(String interrogationId);
List getAllUnprocessedData();
void updateProcessDates(String campaignId, Set interrogationIds);
+
Set findDistinctQuestionnaireIds();
Set findDistinctQuestionnaireIdsByNullProcessDate();
Set findModesByQuestionnaire(String questionnaireId);
@@ -31,4 +31,5 @@ public interface LunaticJsonRawDataPersistencePort {
boolean existsByInterrogationId(String interrogationId);
long countDistinctInterrogationIdsByQuestionnaireId(String questionnaireId);
+
}
diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java
index 649d942ae..c8db17a73 100644
--- a/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java
+++ b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java
@@ -21,8 +21,7 @@ public interface RawResponsePersistencePort {
Page findByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable);
long countByCollectionInstrumentId(String collectionInstrumentId);
Set findDistinctCollectionInstrumentIds();
- long countDistinctInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId);
Page findByCollectionInstrumentId(String collectionInstrumentId, Pageable pageable);
-
boolean existsByInterrogationId(String interrogationId);
+
}
diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponseReprocessPersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponseReprocessPersistencePort.java
new file mode 100644
index 000000000..8560faa10
--- /dev/null
+++ b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponseReprocessPersistencePort.java
@@ -0,0 +1,16 @@
+package fr.insee.genesis.domain.ports.spi;
+
+import java.time.Instant;
+import java.util.Set;
+
+public interface RawResponseReprocessPersistencePort {
+
+ Set findProcessedInterrogationIdsByCollectionInstrumentId(
+ String collectionInstrumentId);
+
+ Set findProcessedInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(
+ String collectionInstrumentId, Instant sinceDate, Instant endDate);
+
+ void resetProcessDates(String collectionInstrumentId, Set interrogationIds);
+
+}
diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponseReprocessPersistenceRouter.java b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponseReprocessPersistenceRouter.java
new file mode 100644
index 000000000..ed77a8981
--- /dev/null
+++ b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponseReprocessPersistenceRouter.java
@@ -0,0 +1,9 @@
+package fr.insee.genesis.domain.ports.spi;
+
+import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType;
+
+public interface RawResponseReprocessPersistenceRouter {
+
+ RawResponseReprocessPersistencePort resolve(RawDataModelType rawDataModelType);
+
+}
diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java
index 0629b3029..9e3b952ca 100644
--- a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java
+++ b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java
@@ -2,6 +2,7 @@
import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
+import java.time.Instant;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Set;
@@ -34,8 +35,8 @@ public interface SurveyUnitPersistencePort {
List findInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(
String collectionInstrumentId,
- LocalDateTime start,
- LocalDateTime end
+ Instant start,
+ Instant end
);
//======== OPTIMISATIONS PERFS (START) ========
@@ -52,6 +53,16 @@ List findInterrogationIdsByCollectionInstrumentIdAndRecordDateB
Long deleteByCollectionInstrumentId(String collectionInstrumentId);
+ Long deleteByCollectionInstrumentIdAndInterrogationIds(
+ String collectionInstrumentId,
+ Set interrogationIds
+ );
+
+ Long deleteByQuestionnaireIdAndInterrogationIds(
+ String questionnaireId,
+ Set interrogationIds
+ );
+
long count();
Set findQuestionnaireIdsByCampaignId(String campaignId);
diff --git a/src/main/java/fr/insee/genesis/domain/service/metadata/QuestionnaireMetadataService.java b/src/main/java/fr/insee/genesis/domain/service/metadata/QuestionnaireMetadataService.java
index 2e254a87c..898466d49 100644
--- a/src/main/java/fr/insee/genesis/domain/service/metadata/QuestionnaireMetadataService.java
+++ b/src/main/java/fr/insee/genesis/domain/service/metadata/QuestionnaireMetadataService.java
@@ -45,7 +45,7 @@ public MetadataModel find(String collectionInstrumentId, Mode mode) throws Genes
}
@Override
- public MetadataModel loadAndSaveIfNotExists(String campaignName, String collectionInstrumentId, Mode mode, FileUtils fileUtils,
+ public MetadataModel loadAndSaveIfNotExists(String questionnaireId, String collectionInstrumentId, Mode mode, FileUtils fileUtils,
List errors) throws GenesisException {
List questionnaireMetadataModels =
questionnaireMetadataPersistencePort.find(collectionInstrumentId.toUpperCase(), mode);
diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java
index 15abc3e2d..32d9ddff6 100644
--- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java
+++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java
@@ -40,6 +40,7 @@
import java.math.BigDecimal;
import java.time.Instant;
import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -47,6 +48,7 @@
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
+
import static fr.insee.genesis.domain.service.rawdata.RawResponseService.processCollectedVariable;
@Service
@@ -96,11 +98,6 @@ public void save(LunaticJsonRawDataModel rawData) {
lunaticJsonRawDataPersistencePort.save(rawData);
}
- @Override
- public List getRawData(String campaignName, Mode mode, List interrogationIdList) {
- return lunaticJsonRawDataPersistencePort.findRawData(campaignName, mode, interrogationIdList);
- }
-
@Override
public List getRawDataByQuestionnaireId(String questionnaireId, Mode mode, List interrogationIdList) {
return lunaticJsonRawDataPersistencePort.findRawDataByQuestionnaireId(questionnaireId, mode, interrogationIdList);
@@ -112,16 +109,15 @@ public List getRawDataByInterrogationId(String interrog
}
@Override
- @Deprecated(since = "1.13.0")
- public DataProcessResult processRawData(String campaignName, List interrogationIdList, List errors) throws GenesisException {
+ public DataProcessResult processRawDataByInterrogationIds(String questionnaireId, List interrogationIdList, List errors) throws GenesisException {
int dataCount=0;
int formattedDataCount=0;
DataProcessingContextModel dataProcessingContext =
- dataProcessingContextService.getContextByCollectionInstrumentId(campaignName);
- List modesList = controllerUtils.getModesList(campaignName, null);
+ dataProcessingContextService.getContextByCollectionInstrumentId(questionnaireId);
+ List modesList = controllerUtils.getModesList(questionnaireId, null);
for (Mode mode : modesList) {
//Load and save metadata into database, throw exception if none
- VariablesMap variablesMap = getVariablesMap(campaignName, mode, errors);
+ VariablesMap variablesMap = getVariablesMap(questionnaireId, mode, errors);
int totalBatchs = Math.ceilDiv(interrogationIdList.size() , config.getRawDataProcessingBatchSize());
int batchNumber = 1;
List interrogationIdListForMode = new ArrayList<>(interrogationIdList);
@@ -130,7 +126,7 @@ public DataProcessResult processRawData(String campaignName, List interr
int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize());
List interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex);
- List rawData = getRawData(campaignName, mode, interrogationIdToProcess);
+ List rawData = getRawDataByQuestionnaireId(questionnaireId, mode, interrogationIdToProcess);
List surveyUnitModels = convertRawData(
rawData,
@@ -222,6 +218,7 @@ public DataProcessResult processRawData(String questionnaireId) throws GenesisEx
return new DataProcessResult(dataCount, formattedDataCount, errors);
}
+
private VariablesMap getVariablesMap(String questionnaireId, Mode mode, List errors) throws GenesisException {
VariablesMap variablesMap = metadataService.loadAndSaveIfNotExists(questionnaireId, questionnaireId, mode, fileUtils,
errors).getVariables();
@@ -269,6 +266,7 @@ private Map> getProcessedIdsMap(List survey
return processedInterrogationIdsPerQuestionnaire;
}
+
@Override
public List convertRawData(List rawDataList, VariablesMap variablesMap) {
//Convert to genesis model
@@ -328,13 +326,13 @@ public List convertRawData(List rawDat
private static RawDataModelType getRawDataModelType(LunaticJsonRawDataModel rawData) {
return rawData.data().containsKey("data") ?
RawDataModelType.FILIERE :
- RawDataModelType.DEFAULT;
+ RawDataModelType.LEGACY;
}
private static LocalDateTime getValidationDate(LunaticJsonRawDataModel rawData) {
try{
return rawData.data().get("validationDate") == null ? null :
- LocalDateTime.parse(rawData.data().get("validationDate").toString());
+ LocalDateTime.parse(rawData.data().get("validationDate").toString(), DateTimeFormatter.ISO_OFFSET_DATE_TIME);
}catch(Exception e){
log.warn("Exception when parsing validation date : {}}",e.toString());
return null;
diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java
index 9635a4f16..66bea080f 100644
--- a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java
+++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java
@@ -23,6 +23,8 @@
import fr.insee.genesis.domain.utils.JsonUtils;
import fr.insee.genesis.exceptions.GenesisError;
import fr.insee.genesis.exceptions.GenesisException;
+import fr.insee.genesis.exceptions.InvalidMetadataException;
+import fr.insee.genesis.exceptions.UndefinedMetadataException;
import fr.insee.genesis.infrastructure.utils.FileUtils;
import fr.insee.modelefiliere.ModeDto;
import fr.insee.modelefiliere.RawResponseDto;
@@ -37,12 +39,7 @@
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import java.util.stream.Collectors;
import static fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService.getValueString;
@@ -75,120 +72,61 @@ public RawResponseService(ControllerUtils controllerUtils, QuestionnaireMetadata
this.rawResponsePersistencePort = rawResponsePersistencePort;
}
- @Override
- public List getRawResponses(String collectionInstrumentId, Mode mode, List interrogationIdList) {
+ private List getRawResponses(String collectionInstrumentId, Mode mode, List interrogationIdList) {
return rawResponsePersistencePort.findRawResponses(collectionInstrumentId,mode,interrogationIdList);
}
@Override
- public List getRawResponsesByInterrogationID(String interrogationId) {
- return rawResponsePersistencePort.findRawResponsesByInterrogationID(interrogationId);
+ public DataProcessResult processRawResponsesByInterrogationIds(String collectionInstrumentId) {
+ List interrogationIds = rawResponsePersistencePort
+ .findUnprocessedInterrogationIdsByCollectionInstrumentId(collectionInstrumentId).stream().toList();
+ return processRawResponsesByInterrogationIds(collectionInstrumentId, interrogationIds, new ArrayList<>());
}
@Override
- public DataProcessResult processRawResponses(String collectionInstrumentId, List interrogationIdList, List errors) throws GenesisException {
- int dataCount=0;
- int formattedDataCount=0;
+ public DataProcessResult processRawResponsesByInterrogationIds(
+ String collectionInstrumentId, List interrogationIdList, List errors) {
+
DataProcessingContextModel dataProcessingContext =
dataProcessingContextService.getContextByCollectionInstrumentId(collectionInstrumentId);
- List modesList = controllerUtils.getModesList(collectionInstrumentId, null);
- for (Mode mode : modesList) {
- //Load and save metadata into database, throw exception if none
- VariablesMap variablesMap = getVariablesMap(collectionInstrumentId,mode,errors);
- int totalBatchs = Math.ceilDiv(interrogationIdList.size() , config.getRawDataProcessingBatchSize());
- int batchNumber = 1;
- List interrogationIdListForMode = new ArrayList<>(interrogationIdList);
- while(!interrogationIdListForMode.isEmpty()){
- log.info("Processing raw data batch {}/{}", batchNumber, totalBatchs);
- int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize());
- List interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex);
+ List modesList = controllerUtils.getModesList(collectionInstrumentId);
- List rawResponseModels = getRawResponses(collectionInstrumentId, mode, interrogationIdToProcess);
-
- List surveyUnitModels = convertRawResponse(
- rawResponseModels,
- variablesMap
- );
-
- //Save converted data
- surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap);
- surveyUnitService.saveSurveyUnits(surveyUnitModels);
+ int dataCount = 0;
+ int formattedDataCount = 0;
+ int batchSize = config.getRawDataProcessingBatchSize();
+ int totalBatches = Math.ceilDiv(interrogationIdList.size(), batchSize);
+ boolean shouldUseQualityTool = resolveWithReviewValue(dataProcessingContext, collectionInstrumentId);
- //Update process dates
- updateProcessDates(surveyUnitModels);
-
- //Increment data count
- dataCount += surveyUnitModels.size();
- formattedDataCount += surveyUnitModels.stream()
- .filter(surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED))
- .toList()
- .size();
+ for (Mode mode : modesList) {
+ VariablesMap variablesMap = loadAndSaveMetadata(collectionInstrumentId, mode, errors);
- //Send processed ids grouped by questionnaire (if review activated)
- if(dataProcessingContext != null && dataProcessingContext.isWithReview()) {
- sendProcessedIdsToQualityTool(surveyUnitModels);
- } else {
- log.warn("Data processing context not found for collection instrument {}. Ids processed not send to quality tool.",collectionInstrumentId);
- }
+ List interrogationIdListForMode = new ArrayList<>(interrogationIdList);
+ int batchNumber = 1;
- //Remove processed ids from list
- interrogationIdListForMode = interrogationIdListForMode.subList(maxIndex, interrogationIdListForMode.size());
+ while(! interrogationIdListForMode.isEmpty()) {
+ log.info("Processing raw data batch {}/{}", batchNumber, totalBatches);
- batchNumber++;
- }
- }
- return new DataProcessResult(dataCount, formattedDataCount, errors);
- }
+ int maxIndex = Math.min(interrogationIdListForMode.size(), batchSize);
+ List interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex);
- @Override
- public DataProcessResult processRawResponses(String collectionInstrumentId) throws GenesisException {
- int dataCount=0;
- int formattedDataCount=0;
- DataProcessingContextModel dataProcessingContext =
- dataProcessingContextService.getContextByCollectionInstrumentId(collectionInstrumentId);
- List errors = new ArrayList<>();
+ List rawResponseModels = getRawResponses(collectionInstrumentId, mode, interrogationIdToProcess);
+ rawResponseModels.removeIf(rawResponseModel -> rawResponseModel.processDate() != null);
+ // (Don't process raw responses that have already been processed.)
- List modesList = controllerUtils.getModesList(collectionInstrumentId, null);
- for (Mode mode : modesList) {
- //Load and save metadata into database, throw exception if none
- VariablesMap variablesMap = getVariablesMap(collectionInstrumentId,mode,errors);
- Set interrogationIds =
- rawResponsePersistencePort.findUnprocessedInterrogationIdsByCollectionInstrumentId(collectionInstrumentId);
+ List surveyUnitModels = convertRawResponse(rawResponseModels, variablesMap);
- int totalBatchs = Math.ceilDiv(interrogationIds.size() , config.getRawDataProcessingBatchSize());
- int batchNumber = 1;
- List interrogationIdListForMode = new ArrayList<>(interrogationIds);
- while(!interrogationIdListForMode.isEmpty()){
- log.info("Processing raw data batch {}/{}", batchNumber, totalBatchs);
- int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize());
-
- List surveyUnitModels = getConvertedSurveyUnits(
- collectionInstrumentId,
- mode,
- interrogationIdListForMode,
- maxIndex,
- variablesMap);
-
- //Save converted data
surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap);
surveyUnitService.saveSurveyUnits(surveyUnitModels);
-
- //Update process dates
updateProcessDates(surveyUnitModels);
- //Increment data count
dataCount += surveyUnitModels.size();
- formattedDataCount += surveyUnitModels.stream()
+ formattedDataCount += (int) surveyUnitModels.stream()
.filter(surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED))
- .toList()
- .size();
+ .count();
- //Send processed ids grouped by questionnaire (if review activated)
- if(dataProcessingContext != null && dataProcessingContext.isWithReview()) {
+ if (shouldUseQualityTool)
sendProcessedIdsToQualityTool(surveyUnitModels);
- }
- //Remove processed ids from list
interrogationIdListForMode = interrogationIdListForMode.subList(maxIndex, interrogationIdListForMode.size());
batchNumber++;
}
@@ -196,23 +134,35 @@ public DataProcessResult processRawResponses(String collectionInstrumentId) thro
return new DataProcessResult(dataCount, formattedDataCount, errors);
}
- private List getConvertedSurveyUnits(String collectionInstrumentId, Mode mode, List interrogationIdListForMode, int maxIndex, VariablesMap variablesMap) {
- List interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex);
- List rawResponseModels = getRawResponses(collectionInstrumentId, mode, interrogationIdToProcess);
- return convertRawResponse(
- rawResponseModels,
- variablesMap
- );
+ /**
+ * Returns the value of the 'withReview' property in the context object.
+ * @param dataProcessingContext {@link DataProcessingContextModel}
+ * @param collectionInstrumentId Passed for logging purposes.
+ * @return The 'withReview' value, false if context is null.
+ */
+ private static boolean resolveWithReviewValue(DataProcessingContextModel dataProcessingContext, String collectionInstrumentId) {
+ if (dataProcessingContext == null) {
+ log.warn("Data processing context not found for collection instrument {}. " +
+ "Ids processed not send to quality tool.", collectionInstrumentId);
+ return false;
+ }
+ return dataProcessingContext.isWithReview();
}
- private VariablesMap getVariablesMap(String collectionInstrumentId, Mode mode, List errors) throws GenesisException {
- VariablesMap variablesMap = metadataService.loadAndSaveIfNotExists(collectionInstrumentId, collectionInstrumentId, mode, fileUtils,
- errors).getVariables();
+ /** Load and save metadata into database, throw exception if none. */
+ private VariablesMap loadAndSaveMetadata(String collectionInstrumentId, Mode mode, List errors) {
+ VariablesMap variablesMap;
+ try {
+ variablesMap = metadataService.loadAndSaveIfNotExists(
+ collectionInstrumentId, collectionInstrumentId, mode, fileUtils, errors).getVariables();
+ } catch (GenesisException genesisException) {
+ throw new UndefinedMetadataException(
+ "Cannot load metadata for collection instrument %s and mode %s.".formatted(collectionInstrumentId, mode),
+ genesisException);
+ }
if (variablesMap == null) {
- throw new GenesisException(400,
- "Error during metadata parsing for mode %s :%n%s"
- .formatted(mode, errors.getLast().getMessage())
- );
+ throw new InvalidMetadataException(
+ "Error during metadata parsing for mode %s :%n%s".formatted(mode, errors.getLast().getMessage()));
}
return variablesMap;
}
@@ -333,18 +283,13 @@ private boolean isSpecsPresentForCollectionInstrumentAndMode(String unprocessedC
@Override
public void updateProcessDates(List surveyUnitModels) {
- Set collectionInstrumentIds = new HashSet<>();
- for (SurveyUnitModel surveyUnitModel : surveyUnitModels) {
- collectionInstrumentIds.add(surveyUnitModel.getCollectionInstrumentId());
- }
-
- for (String collectionInstrumentId : collectionInstrumentIds) {
+ surveyUnitModels.stream().map(SurveyUnitModel::getCollectionInstrumentId).distinct().forEach(collectionInstrumentId -> {
Set interrogationIds = surveyUnitModels.stream()
.filter(su -> su.getCollectionInstrumentId().equals(collectionInstrumentId))
.map(SurveyUnitModel::getInterrogationId)
.collect(Collectors.toSet());
rawResponsePersistencePort.updateProcessDates(collectionInstrumentId, interrogationIds);
- }
+ });
}
@Override
diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/ReprocessRawResponseService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/ReprocessRawResponseService.java
new file mode 100644
index 000000000..f0e2c10eb
--- /dev/null
+++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/ReprocessRawResponseService.java
@@ -0,0 +1,118 @@
+package fr.insee.genesis.domain.service.rawdata;
+
+import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
+import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType;
+import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort;
+import fr.insee.genesis.domain.ports.api.RawResponseApiPort;
+import fr.insee.genesis.domain.ports.api.ReprocessRawResponseApiPort;
+import fr.insee.genesis.domain.ports.spi.RawResponseReprocessPersistencePort;
+import fr.insee.genesis.domain.ports.spi.RawResponseReprocessPersistenceRouter;
+import fr.insee.genesis.domain.ports.spi.SurveyUnitPersistencePort;
+import fr.insee.genesis.exceptions.GenesisException;
+import fr.insee.genesis.exceptions.InvalidDateIntervalException;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.lang.NonNull;
+import org.springframework.stereotype.Service;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Set;
+
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class ReprocessRawResponseService implements ReprocessRawResponseApiPort {
+
+ private final SurveyUnitPersistencePort surveyUnitPersistence;
+
+ private final RawResponseApiPort rawResponseService;
+ private final LunaticJsonRawDataApiPort lunaticJsonRawDataService;
+ // Polymorphism for the raw data services would cause too much refactor.
+
+ private final RawResponseReprocessPersistenceRouter rawResponseReprocessPersistenceRouter;
+ private RawResponseReprocessPersistencePort rawResponseReprocessPersistencePort;
+
+ private enum InputFilterType {
+ COLLECTION_ID,
+ COLLECTION_ID_AND_DATE
+ }
+
+ @Override
+ public DataProcessResult reprocessRawResponses(
+ @NonNull RawDataModelType rawDataModelType,
+ @NonNull String collectionInstrumentId,
+ Instant sinceDate,
+ Instant endDate) throws GenesisException {
+
+ log.info("Start reprocess {} data for collectionInstrumentId={}, sinceDate={}, endDate={}",
+ rawDataModelType, collectionInstrumentId, sinceDate, endDate);
+
+ InputFilterType inputFilterType = validateInputs(sinceDate, endDate);
+
+ rawResponseReprocessPersistencePort = rawResponseReprocessPersistenceRouter.resolve(rawDataModelType);
+
+ Set interrogationIds = switch (inputFilterType) {
+ case COLLECTION_ID ->
+ rawResponseReprocessPersistencePort
+ .findProcessedInterrogationIdsByCollectionInstrumentId(
+ collectionInstrumentId);
+ case COLLECTION_ID_AND_DATE ->
+ rawResponseReprocessPersistencePort
+ .findProcessedInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(
+ collectionInstrumentId,
+ sinceDate,
+ effectiveEndDate(endDate));
+ };
+
+ return reprocessInterrogations(rawDataModelType, collectionInstrumentId, interrogationIds);
+ }
+
+ private static InputFilterType validateInputs(Instant sinceDate, Instant endDate) {
+ if (bothDatesAreNull(sinceDate, endDate)) {
+ return InputFilterType.COLLECTION_ID;
+ }
+ if (sinceDate == null) {
+ throw new InvalidDateIntervalException("'endDate' cannot be provided without 'sinceDate'.");
+ }
+ if (endIsBeforeSince(sinceDate, endDate)) {
+ throw new InvalidDateIntervalException("'endDate' value cannot be before 'sinceDate'.");
+ }
+ return InputFilterType.COLLECTION_ID_AND_DATE;
+ }
+
+ private static boolean endIsBeforeSince(Instant sinceDate, Instant endDate) {
+ return endDate != null && endDate.isBefore(sinceDate);
+ }
+
+ private static boolean bothDatesAreNull(Instant sinceDate, Instant endDate) {
+ return sinceDate == null && endDate == null;
+ }
+
+ private static Instant effectiveEndDate(Instant endDate) {
+ if (endDate != null)
+ return endDate;
+ var now = Instant.now();
+ log.info("Effective end date: {}", now);
+ return now;
+ }
+
+ private DataProcessResult reprocessInterrogations(
+ RawDataModelType rawDataModelType, String collectionInstrumentId, Set interrogationIds)
+ throws GenesisException {
+ if (interrogationIds.isEmpty()) {
+ return new DataProcessResult(0, 0, new ArrayList<>());
+ }
+
+ surveyUnitPersistence.deleteByCollectionInstrumentIdAndInterrogationIds(collectionInstrumentId, interrogationIds);
+ rawResponseReprocessPersistencePort.resetProcessDates(collectionInstrumentId, interrogationIds);
+
+ return switch (rawDataModelType) {
+ case FILIERE -> rawResponseService.processRawResponsesByInterrogationIds(
+ collectionInstrumentId, new ArrayList<>(interrogationIds), new ArrayList<>());
+ case LEGACY -> lunaticJsonRawDataService.processRawDataByInterrogationIds(
+ collectionInstrumentId, new ArrayList<>(interrogationIds), new ArrayList<>());
+ };
+ }
+
+}
diff --git a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java
index 5be97c319..00f56fd66 100644
--- a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java
+++ b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java
@@ -2,20 +2,8 @@
import fr.insee.bpm.metadata.model.VariableType;
import fr.insee.bpm.metadata.model.VariablesMap;
-import fr.insee.genesis.controller.dto.CampaignWithQuestionnaire;
-import fr.insee.genesis.controller.dto.QuestionnaireWithCampaign;
-import fr.insee.genesis.controller.dto.SurveyUnitDto;
-import fr.insee.genesis.controller.dto.SurveyUnitInputDto;
-import fr.insee.genesis.controller.dto.SurveyUnitSimplifiedDto;
-import fr.insee.genesis.controller.dto.VariableDto;
-import fr.insee.genesis.controller.dto.VariableInputDto;
-import fr.insee.genesis.controller.dto.VariableStateDto;
-import fr.insee.genesis.domain.model.surveyunit.DataState;
-import fr.insee.genesis.domain.model.surveyunit.InterrogationId;
-import fr.insee.genesis.domain.model.surveyunit.Mode;
-import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
-import fr.insee.genesis.domain.model.surveyunit.VarIdScopeTuple;
-import fr.insee.genesis.domain.model.surveyunit.VariableModel;
+import fr.insee.genesis.controller.dto.*;
+import fr.insee.genesis.domain.model.surveyunit.*;
import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort;
import fr.insee.genesis.domain.ports.spi.SurveyUnitPersistencePort;
import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService;
@@ -29,14 +17,9 @@
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Service;
+import java.time.Instant;
import java.time.LocalDateTime;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
+import java.util.*;
import java.util.stream.Collectors;
@Service
@@ -397,9 +380,18 @@ public List findDistinctInterrogationIdsByQuestionnaireIdAndDat
}
@Override
- public List findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(String collectionInstrumentId, LocalDateTime start, LocalDateTime end) {
+ public List findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(
+ String collectionInstrumentId,
+ Instant start,
+ Instant end
+ ) {
+
return surveyUnitPersistencePort
- .findInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(collectionInstrumentId,start,end)
+ .findInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(
+ collectionInstrumentId,
+ start,
+ end
+ )
.stream()
.map(su -> new InterrogationId(su.getInterrogationId()))
.distinct()
@@ -496,6 +488,14 @@ public Long deleteByCollectionInstrumentId(String collectionInstrumentId) {
return surveyUnitPersistencePort.deleteByCollectionInstrumentId(collectionInstrumentId);
}
+ @Override
+ public Long deleteByQuestionnaireIdAndInterrogationIds(
+ String questionnaireId,
+ Set interrogationIds
+ ) {
+ return surveyUnitPersistencePort.deleteByQuestionnaireIdAndInterrogationIds(questionnaireId, interrogationIds);
+ }
+
@Override
public long countResponses() {
return surveyUnitPersistencePort.count();
diff --git a/src/main/java/fr/insee/genesis/exceptions/InvalidDateIntervalException.java b/src/main/java/fr/insee/genesis/exceptions/InvalidDateIntervalException.java
new file mode 100644
index 000000000..c4b7d1018
--- /dev/null
+++ b/src/main/java/fr/insee/genesis/exceptions/InvalidDateIntervalException.java
@@ -0,0 +1,9 @@
+package fr.insee.genesis.exceptions;
+
+public class InvalidDateIntervalException extends RuntimeException {
+
+ public InvalidDateIntervalException(String message) {
+ super(message);
+ }
+
+}
diff --git a/src/main/java/fr/insee/genesis/exceptions/InvalidMetadataException.java b/src/main/java/fr/insee/genesis/exceptions/InvalidMetadataException.java
new file mode 100644
index 000000000..85477ddde
--- /dev/null
+++ b/src/main/java/fr/insee/genesis/exceptions/InvalidMetadataException.java
@@ -0,0 +1,7 @@
+package fr.insee.genesis.exceptions;
+
+public class InvalidMetadataException extends RuntimeException {
+ public InvalidMetadataException(String message) {
+ super(message);
+ }
+}
diff --git a/src/main/java/fr/insee/genesis/exceptions/ModesConflictException.java b/src/main/java/fr/insee/genesis/exceptions/ModesConflictException.java
new file mode 100644
index 000000000..000466326
--- /dev/null
+++ b/src/main/java/fr/insee/genesis/exceptions/ModesConflictException.java
@@ -0,0 +1,7 @@
+package fr.insee.genesis.exceptions;
+
+public class ModesConflictException extends RuntimeException {
+ public ModesConflictException(String message) {
+ super(message);
+ }
+}
diff --git a/src/main/java/fr/insee/genesis/exceptions/UndefinedMetadataException.java b/src/main/java/fr/insee/genesis/exceptions/UndefinedMetadataException.java
new file mode 100644
index 000000000..c031c356c
--- /dev/null
+++ b/src/main/java/fr/insee/genesis/exceptions/UndefinedMetadataException.java
@@ -0,0 +1,12 @@
+package fr.insee.genesis.exceptions;
+
+public class UndefinedMetadataException extends RuntimeException {
+
+ public UndefinedMetadataException(String message) {
+ super(message);
+ }
+
+ public UndefinedMetadataException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/src/main/java/fr/insee/genesis/exceptions/UndefinedModesException.java b/src/main/java/fr/insee/genesis/exceptions/UndefinedModesException.java
new file mode 100644
index 000000000..245c4e6cc
--- /dev/null
+++ b/src/main/java/fr/insee/genesis/exceptions/UndefinedModesException.java
@@ -0,0 +1,7 @@
+package fr.insee.genesis.exceptions;
+
+public class UndefinedModesException extends RuntimeException {
+ public UndefinedModesException(String message) {
+ super(message);
+ }
+}
diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java
index cd12f1094..9a5eb9ddb 100644
--- a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java
+++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java
@@ -30,6 +30,7 @@
@Service
@Qualifier("lunaticJsonMongoAdapter")
public class LunaticJsonRawDataMongoAdapter implements LunaticJsonRawDataPersistencePort {
+
private final LunaticJsonMongoDBRepository repository;
private final MongoTemplate mongoTemplate;
@@ -59,12 +60,6 @@ public Set findModesByQuestionnaire(String questionnaireId) {
return new HashSet<>(repository.findModesByQuestionnaireId(questionnaireId));
}
- @Override
- public List findRawData(String campaignName, Mode mode, List interrogationIdList) {
- List rawDataDocs = repository.findByCampaignModeAndInterrogations(campaignName, mode, interrogationIdList);
- return LunaticJsonRawDataDocumentMapper.INSTANCE.listDocumentToListModel(rawDataDocs);
- }
-
@Override
public List findRawDataByQuestionnaireId(String questionnaireId, Mode mode, List interrogationIdList) {
List rawDataDocs = repository.findByQuestionnaireModeAndInterrogations(questionnaireId, mode, interrogationIdList);
@@ -147,4 +142,5 @@ public long countDistinctInterrogationIdsByQuestionnaireId(String questionnaireI
Long count = repository.countDistinctInterrogationIdsByQuestionnaireId(questionnaireId);
return count != null ? count : 0;
}
+
}
diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonReprocessMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonReprocessMongoAdapter.java
new file mode 100644
index 000000000..028b3fada
--- /dev/null
+++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonReprocessMongoAdapter.java
@@ -0,0 +1,60 @@
+package fr.insee.genesis.infrastructure.adapter;
+
+import fr.insee.genesis.Constants;
+import fr.insee.genesis.domain.ports.spi.RawResponseReprocessPersistencePort;
+import fr.insee.genesis.infrastructure.repository.LunaticJsonMongoDBRepository;
+import lombok.AllArgsConstructor;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.data.mongodb.core.query.Update;
+import org.springframework.stereotype.Service;
+
+import java.time.Instant;
+import java.util.HashSet;
+import java.util.Set;
+
+@Service
+@AllArgsConstructor
+@Qualifier("lunaticJsonReprocessMongoAdapter")
+public class LunaticJsonReprocessMongoAdapter implements RawResponseReprocessPersistencePort {
+
+ private final LunaticJsonMongoDBRepository repository;
+ private final MongoTemplate mongoTemplate;
+
+ /**
+ * @param questionnaireId Legacy name for 'collectionInstrumentId'.
+ */
+ @Override
+ public Set findProcessedInterrogationIdsByCollectionInstrumentId(String questionnaireId) {
+ return new HashSet<>(repository.findProcessedInterrogationIdsByQuestionnaireId(questionnaireId));
+ }
+
+ /**
+ * @param questionnaireId Legacy name for 'collectionInstrumentId'.
+ */
+ @Override
+ public Set findProcessedInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(
+ String questionnaireId, Instant sinceDate, Instant endDate) {
+ return new HashSet<>(
+ repository.findProcessedInterrogationIdsByQuestionnaireIdAndRecordDateBetween(
+ questionnaireId, sinceDate, endDate));
+ }
+
+ /**
+ * @param questionnaireId Legacy name for 'collectionInstrumentId'.
+ */
+ @Override
+ public void resetProcessDates(String questionnaireId, Set interrogationIds) {
+ mongoTemplate.updateMulti(
+ Query.query(
+ Criteria.where("questionnaireId").is(questionnaireId)
+ .and("interrogationId").in(interrogationIds)
+ ),
+ new Update().unset("processDate"),
+ Constants.MONGODB_LUNATIC_RAWDATA_COLLECTION_NAME
+ );
+ }
+
+}
diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java
index bdb4c2fe7..b8aea6374 100644
--- a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java
+++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java
@@ -53,10 +53,11 @@ public List findRawResponsesByInterrogationID(String interroga
@Override
public void updateProcessDates(String collectionInstrumentId, Set interrogationIds) {
mongoTemplate.updateMulti(
- Query.query(Criteria.where("collectionInstrumentId").is(collectionInstrumentId).and("interrogationId").in(interrogationIds))
- , new Update().set("processDate", LocalDateTime.now())
- , Constants.MONGODB_RAW_RESPONSES_COLLECTION_NAME
- );
+ Query.query(Criteria.where("collectionInstrumentId")
+ .is(collectionInstrumentId)
+ .and("interrogationId").in(interrogationIds)),
+ new Update().set("processDate", LocalDateTime.now()),
+ Constants.MONGODB_RAW_RESPONSES_COLLECTION_NAME);
}
@Override
@@ -92,12 +93,6 @@ public Set findDistinctCollectionInstrumentIds() {
return new HashSet<>(repository.findDistinctCollectionInstrumentId());
}
- @Override
- public long countDistinctInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) {
- Long count = repository.countDistinctInterrogationIdsByCollectionInstrumentId(collectionInstrumentId);
- return count != null ? count : 0;
- }
-
@Override
public Page findByCollectionInstrumentId(String collectionInstrumentId, Pageable pageable) {
Page rawDataDocs = repository.findByCollectionInstrumentId(collectionInstrumentId, pageable);
diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseReprocessMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseReprocessMongoAdapter.java
new file mode 100644
index 000000000..731ade641
--- /dev/null
+++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseReprocessMongoAdapter.java
@@ -0,0 +1,51 @@
+package fr.insee.genesis.infrastructure.adapter;
+
+import fr.insee.genesis.Constants;
+import fr.insee.genesis.domain.ports.spi.RawResponseReprocessPersistencePort;
+import fr.insee.genesis.infrastructure.repository.RawResponseRepository;
+import lombok.RequiredArgsConstructor;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.data.mongodb.core.query.Update;
+import org.springframework.stereotype.Service;
+
+import java.time.Instant;
+import java.util.HashSet;
+import java.util.Set;
+
+@Service
+@RequiredArgsConstructor
+@Qualifier("rawResponseMongoAdapter")
+public class RawResponseReprocessMongoAdapter implements RawResponseReprocessPersistencePort {
+
+ private final RawResponseRepository repository;
+ private final MongoTemplate mongoTemplate;
+
+ @Override
+ public Set findProcessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) {
+ return new HashSet<>(repository.findProcessedInterrogationIdsByCollectionInstrumentId(collectionInstrumentId));
+ }
+
+ @Override
+ public Set findProcessedInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(
+ String collectionInstrumentId, Instant sinceDate, Instant endDate) {
+ return new HashSet<>(
+ repository.findProcessedInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(
+ collectionInstrumentId, sinceDate, endDate));
+ }
+
+ @Override
+ public void resetProcessDates(String collectionInstrumentId, Set interrogationIds) {
+ mongoTemplate.updateMulti(
+ Query.query(
+ Criteria.where("collectionInstrumentId").is(collectionInstrumentId)
+ .and("interrogationId").in(interrogationIds)
+ ),
+ new Update().unset("processDate"),
+ Constants.MONGODB_RAW_RESPONSES_COLLECTION_NAME
+ );
+ }
+
+}
diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseReprocessMongoRouter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseReprocessMongoRouter.java
new file mode 100644
index 000000000..b059bd356
--- /dev/null
+++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseReprocessMongoRouter.java
@@ -0,0 +1,24 @@
+package fr.insee.genesis.infrastructure.adapter;
+
+import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType;
+import fr.insee.genesis.domain.ports.spi.RawResponseReprocessPersistencePort;
+import fr.insee.genesis.domain.ports.spi.RawResponseReprocessPersistenceRouter;
+import lombok.AllArgsConstructor;
+import org.springframework.stereotype.Component;
+
+@Component
+@AllArgsConstructor
+public class RawResponseReprocessMongoRouter implements RawResponseReprocessPersistenceRouter {
+
+ private final RawResponseReprocessMongoAdapter rawResponseReprocessMongoAdapter;
+ private final LunaticJsonReprocessMongoAdapter lunaticJsonReprocessMongoAdapter;
+
+ @Override
+ public RawResponseReprocessPersistencePort resolve(RawDataModelType rawDataModelType) {
+ return switch (rawDataModelType) {
+ case FILIERE -> rawResponseReprocessMongoAdapter;
+ case LEGACY -> lunaticJsonReprocessMongoAdapter;
+ };
+ }
+
+}
diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java
index bfdd3e840..cf661c1c9 100644
--- a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java
+++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java
@@ -18,6 +18,7 @@
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.stereotype.Service;
+import java.time.Instant;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Collections;
@@ -107,6 +108,28 @@ public Long deleteByCollectionInstrumentId(String collectionInstrumentId) {
return countDeleted;
}
+ @Override
+ public Long deleteByCollectionInstrumentIdAndInterrogationIds(
+ String collectionInstrumentId,
+ Set interrogationIds
+ ) {
+ return mongoRepository.deleteByCollectionInstrumentIdAndInterrogationIdIn(
+ collectionInstrumentId,
+ interrogationIds
+ );
+ }
+
+ @Override
+ public Long deleteByQuestionnaireIdAndInterrogationIds(
+ String questionnaireId,
+ Set interrogationIds
+ ) {
+ return mongoRepository.deleteByQuestionnaireIdAndInterrogationIdIn(
+ questionnaireId,
+ interrogationIds
+ );
+ }
+
@Override
public long count() {
return mongoRepository.count();
@@ -176,7 +199,7 @@ public List findInterrogationIdsByQuestionnaireIdAndDateAfter(S
}
@Override
- public List findInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(String collectionInstrumentId, LocalDateTime start, LocalDateTime end) {
+ public List findInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(String collectionInstrumentId, Instant start, Instant end) {
List results = new ArrayList<>();
results.addAll(mongoRepository.findInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(collectionInstrumentId,start,end));
results.addAll(mongoRepository.findInterrogationIdsQuestionnaireIdAndRecordDateBetween(collectionInstrumentId,start,end));
diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java
index 5d06a0b5b..a0e1121dd 100644
--- a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java
+++ b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java
@@ -111,4 +111,23 @@ public interface LunaticJsonMongoDBRepository extends MongoRepository findByQuestionnaireId(String questionnaireId, Pageable pageable);
boolean existsByInterrogationId(String interrogationId);
+
+
+ @Aggregation(pipeline = {
+ "{ $match: { questionnaireId: ?0, processDate: { $ne: null }, recordDate: { $gte: ?1, $lte: ?2 } } }",
+ "{ $group: { _id: '$interrogationId' } }",
+ "{ $project: { _id: 0, interrogationId: '$_id' } }"
+ })
+ List findProcessedInterrogationIdsByQuestionnaireIdAndRecordDateBetween(
+ String questionnaireId,
+ Instant sinceDate,
+ Instant endDate
+ );
+
+ @Aggregation(pipeline = {
+ "{ $match: { questionnaireId: ?0, processDate: { $ne: null } } }",
+ "{ $group: { _id: '$interrogationId' } }",
+ "{ $project: { _id: 0, interrogationId: '$_id' } }"
+ })
+ List findProcessedInterrogationIdsByQuestionnaireId(String questionnaireId);
}
diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java
index de0583aa7..d37b039b3 100644
--- a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java
+++ b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java
@@ -24,6 +24,24 @@ public interface RawResponseRepository extends MongoRepository findDistinctCollectionInstrumentIdByProcessDateIsNull();
+ @Aggregation(pipeline = {
+ "{ $match: { collectionInstrumentId: ?0, processDate: { $ne: null }, recordDate: { $gte: ?1, $lte: ?2 } } }",
+ "{ $group: { _id: '$interrogationId' } }",
+ "{ $project: { _id: 0, interrogationId: '$_id' } }"
+ })
+ List findProcessedInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(
+ String collectionInstrumentId,
+ Instant sinceDate,
+ Instant endDate
+ );
+
+ @Aggregation(pipeline = {
+ "{ $match: { collectionInstrumentId: ?0, processDate: { $ne: null } } }",
+ "{ $group: { _id: '$interrogationId' } }",
+ "{ $project: { _id: 0, interrogationId: '$_id' } }"
+ })
+ List findProcessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId);
+
@Aggregation(pipeline = {
"{ $match: { collectionInstrumentId: ?0,processDate: null } }",
"{ $project: { _id: 0, interrogationId: '$interrogationId' } }"
@@ -54,11 +72,4 @@ public interface RawResponseRepository extends MongoRepository findInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(
String collectionInstrumentId,
- LocalDateTime start,
- LocalDateTime end
+ Instant start,
+ Instant end
);
@Query(
@@ -59,8 +60,8 @@ List findInterrogationIdsByCollectionInstrumentIdAndRecordDa
)
List findInterrogationIdsQuestionnaireIdAndRecordDateBetween(
String questionnaireId,
- LocalDateTime start,
- LocalDateTime end
+ Instant start,
+ Instant end
);
/**
@@ -103,6 +104,16 @@ List findInterrogationIdsQuestionnaireIdAndRecordDateBetween
Long deleteByQuestionnaireId(String questionnaireId);
Long deleteByCollectionInstrumentId(String collectionInstrumentId);
+ Long deleteByCollectionInstrumentIdAndInterrogationIdIn(
+ String collectionInstrumentId,
+ Set interrogationIds
+ );
+
+ Long deleteByQuestionnaireIdAndInterrogationIdIn(
+ String questionnaireId,
+ Set interrogationIds
+ );
+
@Meta(cursorBatchSize = 20)
Stream findByQuestionnaireId(String questionnaireId);
diff --git a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java
index 8c1790ea5..194506f64 100644
--- a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java
+++ b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java
@@ -23,12 +23,7 @@
import fr.insee.genesis.exceptions.GenesisException;
import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository;
import fr.insee.genesis.infrastructure.utils.FileUtils;
-import fr.insee.genesis.stubs.ConfigStub;
-import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub;
-import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub;
-import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub;
-import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub;
-import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub;
+import fr.insee.genesis.stubs.*;
import fr.insee.modelefiliere.RawResponseDto;
import io.cucumber.java.Before;
import io.cucumber.java.en.Given;
@@ -41,12 +36,7 @@
import org.springframework.boot.test.web.server.LocalServerPort;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
-import org.springframework.http.HttpEntity;
-import org.springframework.http.HttpHeaders;
-import org.springframework.http.HttpMethod;
-import org.springframework.http.HttpStatus;
-import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
+import org.springframework.http.*;
import org.springframework.test.context.ContextConfiguration;
import java.io.IOException;
@@ -109,23 +99,14 @@ public void saveAsRawJson(RawResponseDto dto) {
};
RawResponseApiPort rawResponseApiPortStub = new RawResponseApiPort() {
- @Override
- public List getRawResponses(String questionnaireModelId, Mode mode, List interrogationIdList) {
- return List.of();
- }
-
- @Override
- public List getRawResponsesByInterrogationID(String interrogationId) {
- return List.of();
- }
@Override
- public DataProcessResult processRawResponses(String questionnaireId, List interrogationIdList, List errors) throws GenesisException {
+ public DataProcessResult processRawResponsesByInterrogationIds(String questionnaireId, List interrogationIdList, List errors) throws GenesisException {
return null;
}
@Override
- public DataProcessResult processRawResponses(String collectionInstrumentId) throws GenesisException {
+ public DataProcessResult processRawResponsesByInterrogationIds(String collectionInstrumentId) throws GenesisException {
return null;
}
diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java
index 533740dc5..971919a7f 100644
--- a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java
+++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java
@@ -26,13 +26,7 @@
import fr.insee.genesis.infrastructure.mappers.DataProcessingContextMapper;
import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository;
import fr.insee.genesis.infrastructure.utils.FileUtils;
-import fr.insee.genesis.stubs.ConfigStub;
-import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub;
-import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub;
-import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub;
-import fr.insee.genesis.stubs.RawResponseDataPersistanceStub;
-import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub;
-import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub;
+import fr.insee.genesis.stubs.*;
import fr.insee.modelefiliere.RawResponseDto;
import lombok.extern.slf4j.Slf4j;
import org.assertj.core.api.Assertions;
@@ -86,23 +80,14 @@ public void saveAsRawJson(RawResponseDto dto) {
};
RawResponseApiPort rawResponseApiPortStub = new RawResponseApiPort() {
- @Override
- public List getRawResponses(String questionnaireModelId, Mode mode, List interrogationIdList) {
- return List.of();
- }
-
- @Override
- public List getRawResponsesByInterrogationID(String interrogationId) {
- return List.of();
- }
@Override
- public DataProcessResult processRawResponses(String questionnaireId, List interrogationIdList, List errors) throws GenesisException {
+ public DataProcessResult processRawResponsesByInterrogationIds(String questionnaireId, List interrogationIdList, List errors) throws GenesisException {
return null;
}
@Override
- public DataProcessResult processRawResponses(String collectionInstrumentId) throws GenesisException {
+ public DataProcessResult processRawResponsesByInterrogationIds(String collectionInstrumentId) throws GenesisException {
return null;
}
diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java
index 09c6049ed..8c31bdf7f 100644
--- a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java
+++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java
@@ -18,12 +18,7 @@
import fr.insee.genesis.infrastructure.mappers.DataProcessingContextMapper;
import fr.insee.genesis.infrastructure.mappers.LunaticJsonRawDataDocumentMapper;
import fr.insee.genesis.infrastructure.utils.FileUtils;
-import fr.insee.genesis.stubs.ConfigStub;
-import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub;
-import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub;
-import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub;
-import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub;
-import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub;
+import fr.insee.genesis.stubs.*;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
@@ -692,4 +687,5 @@ void getValueString_double_test(){
Assertions.assertThat(getValueString(doubleObject)).isEqualTo("101010101010.111");
}
-}
\ No newline at end of file
+
+}
diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticRawDataReprocessTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticRawDataReprocessTest.java
new file mode 100644
index 000000000..3c9849a2c
--- /dev/null
+++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticRawDataReprocessTest.java
@@ -0,0 +1,71 @@
+package fr.insee.genesis.domain.service.rawdata;
+
+import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
+import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType;
+import fr.insee.genesis.exceptions.InvalidDateIntervalException;
+import fr.insee.genesis.stubs.LunaticJsonRawDataServiceStub;
+import fr.insee.genesis.stubs.RawResponseReprocessPersistenceRouterStub;
+import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+
+class LunaticRawDataReprocessTest {
+
+ private ReprocessRawResponseService reprocessRawResponseService;
+
+ @BeforeEach
+ void freshStart() {
+ reprocessRawResponseService = new ReprocessRawResponseService(
+ new SurveyUnitPersistencePortStub(),
+ null,
+ new LunaticJsonRawDataServiceStub(),
+ new RawResponseReprocessPersistenceRouterStub());
+ }
+
+ @Test
+ void reprocessRawData_should_return_empty_result_when_no_processed_interrogation_ids_found() throws Exception {
+ // GIVEN
+ String questionnaireId = "TESTIDQUEST";
+
+ // WHEN
+ DataProcessResult result = reprocessRawResponseService.reprocessRawResponses(
+ RawDataModelType.LEGACY, questionnaireId, null, null);
+
+ // THEN
+ Assertions.assertThat(result).isNotNull();
+ Assertions.assertThat(result.dataCount()).isZero();
+ Assertions.assertThat(result.formattedDataCount()).isZero();
+ Assertions.assertThat(result.errors()).isEmpty();
+ }
+
+ @Test
+ void reprocessRawData_should_throw_when_endDate_is_provided_without_sinceDate() {
+ String questionnaireId = "TESTIDQUEST";
+ Instant endDate = Instant.now();
+
+ Assertions.assertThatThrownBy(() ->
+ reprocessRawResponseService.reprocessRawResponses(
+ RawDataModelType.LEGACY, questionnaireId, null, endDate))
+ .isInstanceOf(InvalidDateIntervalException.class)
+ .hasMessage("'endDate' cannot be provided without 'sinceDate'.");
+ }
+
+ @Test
+ void reprocessRawData_should_throw_when_endDate_is_before_sinceDate() {
+ String questionnaireId = "TESTIDQUEST";
+ Instant sinceDate = LocalDateTime.of(2024, 1, 10, 10, 0).toInstant(ZoneOffset.UTC);
+ Instant endDate = LocalDateTime.of(2024, 1, 9, 10, 0).toInstant(ZoneOffset.UTC);
+
+ Assertions.assertThatThrownBy(() ->
+ reprocessRawResponseService.reprocessRawResponses(
+ RawDataModelType.LEGACY, questionnaireId, sinceDate, endDate))
+ .isInstanceOf(InvalidDateIntervalException.class)
+ .hasMessage("'endDate' value cannot be before 'sinceDate'.");
+ }
+
+}
diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java
index 9349c33df..624c1cfe8 100644
--- a/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java
+++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java
@@ -4,8 +4,10 @@
import fr.insee.bpm.metadata.model.VariablesMap;
import fr.insee.genesis.TestConstants;
import fr.insee.genesis.controller.utils.ControllerUtils;
+import fr.insee.genesis.domain.model.context.DataProcessingContextModel;
import fr.insee.genesis.domain.model.surveyunit.Mode;
import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
+import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel;
import fr.insee.genesis.domain.ports.spi.QuestionnaireMetadataPersistencePort;
import fr.insee.genesis.domain.ports.spi.RawResponsePersistencePort;
@@ -21,10 +23,8 @@
import fr.insee.modelefiliere.RawResponseDto;
import lombok.SneakyThrows;
import org.assertj.core.api.Assertions;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.DisplayName;
-import org.junit.jupiter.api.Nested;
-import org.junit.jupiter.api.Test;
+import org.bson.types.ObjectId;
+import org.junit.jupiter.api.*;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
import org.mockito.ArgumentCaptor;
@@ -32,18 +32,12 @@
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import static fr.insee.genesis.TestConstants.DEFAULT_INTERROGATION_ID;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.*;
class RawResponseServiceUnitTest {
@@ -52,6 +46,7 @@ class RawResponseServiceUnitTest {
static ControllerUtils controllerUtils;
static QuestionnaireMetadataService metadataService;
static SurveyUnitService surveyUnitService;
+ static DataProcessingContextService dataProcessingContextService;
private ArgumentCaptor> surveyUnitModelsCaptor;
@@ -64,6 +59,7 @@ void init() {
controllerUtils = mock(ControllerUtils.class);
metadataService = mock(QuestionnaireMetadataService.class);
surveyUnitService = mock(SurveyUnitService.class);
+ dataProcessingContextService = mock(DataProcessingContextService.class);
rawResponseService = new RawResponseService(
controllerUtils,
@@ -71,7 +67,7 @@ void init() {
surveyUnitService,
mock(SurveyUnitQualityService.class),
mock(SurveyUnitQualityToolPort.class),
- mock(DataProcessingContextService.class),
+ dataProcessingContextService,
new FileUtils(new ConfigStub()),
new ConfigStub(),
rawResponsePersistencePort
@@ -152,7 +148,14 @@ void existsByInterrogationId_shouldReturnFalse_whenNotExists() {
}
@Nested
- @DisplayName("Non regression tests of #22875 : validation date and questionnaire state in processed responses")
+ @DisplayName("Non regression tests of InseeFr/Genesis-API#365: validation date and questionnaire state in processed responses")
+ @Disabled("to be fixed") /* hint:
+ since processRawResponsesByInterrogationIds is mocked, after refactor where
+ processRawResponsesByInterrogationIds(String collectionInstrumentId) doesn't directly call
+ surveyUnitService.saveSurveyUnits(...), but calls
+ processRawResponsesByInterrogationIds(String collectionInstrumentId, List interrogationIds, List errors),
+ the assertion "surveyUnitService.saveSurveyUnits(...) should be called" no longer passes.
+ */
class ValidationDateAndQuestionnaireStateTests{
//OK cases
@ParameterizedTest
@@ -354,12 +357,12 @@ private void givenInvalidValidationDate(){
//WHENS
private List whenProcessByCollectionInstrumentIdAndInterrogationIdList() throws GenesisException {
- rawResponseService.processRawResponses(TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID);
+ rawResponseService.processRawResponsesByInterrogationIds(TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID);
verify(surveyUnitService).saveSurveyUnits(surveyUnitModelsCaptor.capture());
return surveyUnitModelsCaptor.getValue();
}
private List whenProcessRawResponsesCollectionInstrumentId() throws GenesisException {
- rawResponseService.processRawResponses(
+ rawResponseService.processRawResponsesByInterrogationIds(
TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID,
Collections.singletonList(TestConstants.DEFAULT_INTERROGATION_ID),
new ArrayList<>()
@@ -412,4 +415,47 @@ void getDistinctCollectionInstrumentIds_test(){
//WHEN + THEN
Assertions.assertThat(rawResponseService.getDistinctCollectionInstrumentIds()).containsExactly(collectionInstrumentId);
}
-}
\ No newline at end of file
+
+ @Test
+ void processWithDuplicateInterrogationId() throws GenesisException {
+ // Given
+ String fooCollectionInstrumentId = "FOOX00";
+ Mode fooMode = Mode.WEB;
+
+ DataProcessingContextModel fooProcessingContext = DataProcessingContextModel.builder()
+ .collectionInstrumentId(fooCollectionInstrumentId).withReview(false)
+ .build();
+
+ Set interrogationIds = Set.of("interrogation-id-1", "interrogation-id-2");
+ List interrogationIdList = interrogationIds.stream().toList();
+ Map fooVariable = Map.of("COLLECTED", "some value");
+ Map fooCollectedContent = Map.of("SOME_VARIABLE", fooVariable);
+ Map fooData = Map.of("COLLECTED", fooCollectedContent);
+ Map fooPayload = Map.of(
+ "questionnaireState", "FOO_QUESTIONNAIRE_STATE",
+ "data", fooData);
+
+ LocalDateTime recordDate1 = LocalDateTime.of(2026, 1, 1, 8, 0);
+ LocalDateTime processDate = LocalDateTime.of(2026, 1, 1, 9, 0);
+ LocalDateTime recordDate2 = LocalDateTime.of(2026, 1, 1, 10, 0);
+
+ List mockedRawResponses = new ArrayList<>(List.of(
+ new RawResponseModel(new ObjectId(), "interrogation-id-1", fooCollectionInstrumentId, fooMode, fooPayload, recordDate1, processDate),
+ new RawResponseModel(new ObjectId(), "interrogation-id-1", fooCollectionInstrumentId, fooMode, fooPayload, recordDate2, null),
+ new RawResponseModel(new ObjectId(), "interrogation-id-2", fooCollectionInstrumentId, fooMode, fooPayload, recordDate2, null)
+ ));
+
+ Mockito.when(rawResponsePersistencePort.findUnprocessedInterrogationIdsByCollectionInstrumentId(fooCollectionInstrumentId)).thenReturn(interrogationIds);
+ Mockito.when(rawResponsePersistencePort.findRawResponses(fooCollectionInstrumentId, fooMode, interrogationIdList)).thenReturn(mockedRawResponses);
+ Mockito.when(controllerUtils.getModesList(fooCollectionInstrumentId)).thenReturn(List.of(fooMode));
+ Mockito.when(dataProcessingContextService.getContextByCollectionInstrumentId(fooCollectionInstrumentId)).thenReturn(fooProcessingContext);
+ Mockito.when(metadataService.loadAndSaveIfNotExists(eq(fooCollectionInstrumentId), eq(fooCollectionInstrumentId), eq(fooMode), any(), any())).thenReturn(new MetadataModel());
+
+ // When
+ DataProcessResult dataProcessResult = rawResponseService.processRawResponsesByInterrogationIds(fooCollectionInstrumentId);
+
+ // Then
+ assertEquals(2, dataProcessResult.dataCount());
+ }
+
+}
diff --git a/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java
index 8cb4453bc..e68e4c151 100644
--- a/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java
+++ b/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java
@@ -18,6 +18,7 @@
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
+import java.time.Instant;
import java.time.LocalDateTime;
import java.time.Month;
import java.util.ArrayList;
@@ -270,8 +271,8 @@ void findDistinctInterrogationIdsByQuestionnaireIdAndDateAfterTest_doc_in_period
void findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetweenTest_no_doc_in_period() {
addAdditionnalSurveyUnitModelToMongoStub();
- LocalDateTime start = LocalDateTime.of(2025, 9, 1, 0, 0, 0);
- LocalDateTime end = LocalDateTime.of(2025, 9, 2, 0, 0, 0);
+ Instant start = Instant.parse("2025-09-01T00:00:00Z");
+ Instant end = Instant.parse("2025-09-02T00:00:00Z");
Assertions.assertThat(
surveyUnitServiceStatic.findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(
@@ -285,8 +286,8 @@ void findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetweenTes
void findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetweenTest_doc_in_period() {
addAdditionnalSurveyUnitModelToMongoStub();
- LocalDateTime start = LocalDateTime.of(2022, 1, 1, 0, 0, 0);
- LocalDateTime end = LocalDateTime.of(2026, 1, 1, 0, 0, 0);
+ Instant start = Instant.parse("2022-01-01T00:00:00Z");
+ Instant end = Instant.parse("2026-01-01T00:00:00Z");
Assertions.assertThat(
surveyUnitServiceStatic.findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(
diff --git a/src/test/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapterTest.java b/src/test/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapterTest.java
index 594b15557..79cc23e44 100644
--- a/src/test/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapterTest.java
+++ b/src/test/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapterTest.java
@@ -73,7 +73,7 @@ void findRawDataTest(){
//WHEN
repository.getDocuments().add(doc);
//THEN
- List rawdatas = adapter.findRawData("campaign01",Mode.WEB,List.of("interrogation01"));
+ List rawdatas = adapter.findRawDataByQuestionnaireId("questionnaire01",Mode.WEB,List.of("interrogation01"));
Assertions.assertThat(rawdatas).hasSize(1);
}
diff --git a/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java b/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java
index 2c0e4dc77..547e8918d 100644
--- a/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java
+++ b/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java
@@ -13,6 +13,7 @@
import java.time.LocalDateTime;
import java.time.Instant;
+import java.time.chrono.ChronoLocalDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -196,6 +197,33 @@ public boolean existsByInterrogationId(String interrogationId) {
return DEFAULT_INTERROGATION_ID.equals(interrogationId);
}
+ @Override
+ public List findProcessedInterrogationIdsByQuestionnaireId(String questionnaireId) {
+ return documents.stream()
+ .filter(doc -> Objects.equals(doc.questionnaireId(), questionnaireId))
+ .filter(doc -> doc.processDate() != null)
+ .map(LunaticJsonRawDataDocument::interrogationId)
+ .distinct()
+ .toList();
+ }
+
+ @Override
+ public List findProcessedInterrogationIdsByQuestionnaireIdAndRecordDateBetween(
+ String questionnaireId,
+ Instant sinceDate,
+ Instant endDate
+ ) {
+ return documents.stream()
+ .filter(doc -> Objects.equals(doc.questionnaireId(), questionnaireId))
+ .filter(doc -> doc.processDate() != null)
+ .filter(doc -> doc.recordDate() != null)
+ .filter(doc -> !doc.recordDate().isBefore(ChronoLocalDateTime.from(sinceDate)))
+ .filter(doc -> !doc.recordDate().isAfter(ChronoLocalDateTime.from(endDate)))
+ .map(LunaticJsonRawDataDocument::interrogationId)
+ .distinct()
+ .toList();
+ }
+
// Implémentations vides requises par MongoRepository
@Override public S save(S entity) { return null; }
@Override public Optional findById(String s) { return Optional.empty(); }
diff --git a/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java b/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java
index 9f32a324b..9f2aba791 100644
--- a/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java
+++ b/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java
@@ -69,23 +69,19 @@ public Set findModesByQuestionnaire(String questionnaireId) {
}
@Override
- public List findRawData(String campaignName, Mode mode, List interrogationIdList) {
+ public List findRawDataByQuestionnaireId(String questionnaireId, Mode mode, List interrogationIdList) {
List docs = mongoStub.stream().filter(lunaticJsonRawDataDocument ->
- lunaticJsonRawDataDocument.campaignId().equals(campaignName)
- && lunaticJsonRawDataDocument.mode().equals(mode)
- && interrogationIdList.contains(lunaticJsonRawDataDocument.interrogationId())
+ getQuestionnaireId(lunaticJsonRawDataDocument).equals(questionnaireId)
+ && lunaticJsonRawDataDocument.mode().equals(mode)
+ && interrogationIdList.contains(lunaticJsonRawDataDocument.interrogationId())
).toList();
return LunaticJsonRawDataDocumentMapper.INSTANCE.listDocumentToListModel(docs);
}
-
- @Override
- public List findRawDataByQuestionnaireId(String questionnaireId, Mode mode, List interrogationIdList) {
- List docs = mongoStub.stream().filter(lunaticJsonRawDataDocument ->
- lunaticJsonRawDataDocument.questionnaireId().equals(questionnaireId)
- && lunaticJsonRawDataDocument.mode().equals(mode)
- && interrogationIdList.contains(lunaticJsonRawDataDocument.interrogationId())
- ).toList();
- return LunaticJsonRawDataDocumentMapper.INSTANCE.listDocumentToListModel(docs); }
+ /** 'questionnaireId' is the new name of the 'campaignId' property. */
+ private static String getQuestionnaireId(LunaticJsonRawDataDocument lunaticJsonRawDataDocument) {
+ return lunaticJsonRawDataDocument.questionnaireId() != null ?
+ lunaticJsonRawDataDocument.questionnaireId() : lunaticJsonRawDataDocument.campaignId();
+ }
@Override
public Page findRawDataByQuestionnaireId(String questionnaireId, Pageable pageable) {
@@ -246,4 +242,5 @@ public boolean existsByInterrogationId(String interrogationId) {
public long countDistinctInterrogationIdsByQuestionnaireId(String questionnaireId) {
return 0;
}
+
}
diff --git a/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataServiceStub.java b/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataServiceStub.java
new file mode 100644
index 000000000..9c3439ebf
--- /dev/null
+++ b/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataServiceStub.java
@@ -0,0 +1,101 @@
+package fr.insee.genesis.stubs;
+
+import fr.insee.bpm.metadata.model.VariablesMap;
+import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto;
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
+import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
+import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel;
+import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort;
+import fr.insee.genesis.exceptions.GenesisError;
+import fr.insee.genesis.exceptions.GenesisException;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
+
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class LunaticJsonRawDataServiceStub implements LunaticJsonRawDataApiPort {
+ @Override
+ public void save(LunaticJsonRawDataModel rawData) {
+ // stub, this method unused in tests yet.
+ }
+
+ @Override
+ public List getRawDataByQuestionnaireId(String questionnaireId, Mode mode, List interrogationIdList) {
+ return List.of();
+ }
+
+ @Override
+ public List convertRawData(List rawData, VariablesMap variablesMap) {
+ return List.of();
+ }
+
+ @Override
+ public List getUnprocessedDataIds() {
+ return List.of();
+ }
+
+ @Override
+ public Set getUnprocessedDataQuestionnaireIds() {
+ return Set.of();
+ }
+
+ @Override
+ public void updateProcessDates(List surveyUnitModels) {
+ // stub, this method unused in tests yet.
+ }
+
+ @Override
+ public Set findDistinctQuestionnaireIds() {
+ return Set.of();
+ }
+
+ @Override
+ public long countRawResponsesByQuestionnaireId(String campaignId) {
+ return 0;
+ }
+
+ @Override
+ public long countDistinctInterrogationIdsByQuestionnaireId(String questionnaireId) {
+ return 0;
+ }
+
+ @Override
+ public Page findRawDataByCampaignIdAndDate(String campaignId, Instant startDt, Instant endDt, Pageable pageable) {
+ return null;
+ }
+
+ @Override
+ public List getRawDataByInterrogationId(String interrogationId) {
+ return List.of();
+ }
+
+ @Override
+ public DataProcessResult processRawDataByInterrogationIds(String campaignName, List interrogationIdList, List errors) throws GenesisException {
+ return null;
+ }
+
+ @Override
+ public DataProcessResult processRawData(String collectionInstrumentId) throws GenesisException {
+ return null;
+ }
+
+ @Override
+ public Map> findProcessedIdsgroupedByQuestionnaireSince(LocalDateTime since) {
+ return Map.of();
+ }
+
+ @Override
+ public Page findRawDataByQuestionnaireId(String questionnaireId, Pageable pageable) {
+ return null;
+ }
+
+ @Override
+ public boolean existsByInterrogationId(String interrogationId) {
+ return false;
+ }
+}
diff --git a/src/test/java/fr/insee/genesis/stubs/RawResponseDataPersistanceStub.java b/src/test/java/fr/insee/genesis/stubs/RawResponseDataPersistanceStub.java
index 697a57d08..ac922a5e0 100644
--- a/src/test/java/fr/insee/genesis/stubs/RawResponseDataPersistanceStub.java
+++ b/src/test/java/fr/insee/genesis/stubs/RawResponseDataPersistanceStub.java
@@ -44,6 +44,7 @@ public void updateProcessDates(String collectionInstrumentId, Set interr
return;
}
+
@Override
public List getUnprocessedCollectionIds() {
return List.of();
@@ -104,9 +105,4 @@ public Set findDistinctCollectionInstrumentIds() {
public boolean existsByInterrogationId(String interrogationId) {
return DEFAULT_INTERROGATION_ID.equals(interrogationId);
}
-
- @Override
- public long countDistinctInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) {
- return 0;
- }
}
diff --git a/src/test/java/fr/insee/genesis/stubs/RawResponseReprocessPersistenceRouterStub.java b/src/test/java/fr/insee/genesis/stubs/RawResponseReprocessPersistenceRouterStub.java
new file mode 100644
index 000000000..e50aba991
--- /dev/null
+++ b/src/test/java/fr/insee/genesis/stubs/RawResponseReprocessPersistenceRouterStub.java
@@ -0,0 +1,14 @@
+package fr.insee.genesis.stubs;
+
+import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType;
+import fr.insee.genesis.domain.ports.spi.RawResponseReprocessPersistencePort;
+import fr.insee.genesis.domain.ports.spi.RawResponseReprocessPersistenceRouter;
+
+public class RawResponseReprocessPersistenceRouterStub implements RawResponseReprocessPersistenceRouter {
+
+ @Override
+ public RawResponseReprocessPersistencePort resolve(RawDataModelType rawDataModelType) {
+ return new RawResponseReprocessPersistenceStub();
+ }
+
+}
diff --git a/src/test/java/fr/insee/genesis/stubs/RawResponseReprocessPersistenceStub.java b/src/test/java/fr/insee/genesis/stubs/RawResponseReprocessPersistenceStub.java
new file mode 100644
index 000000000..4732c2451
--- /dev/null
+++ b/src/test/java/fr/insee/genesis/stubs/RawResponseReprocessPersistenceStub.java
@@ -0,0 +1,50 @@
+package fr.insee.genesis.stubs;
+
+import fr.insee.genesis.domain.ports.spi.RawResponseReprocessPersistencePort;
+import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonRawDataDocument;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+public class RawResponseReprocessPersistenceStub implements RawResponseReprocessPersistencePort {
+
+ List mongoStub = new ArrayList<>();
+
+ @Override
+ public Set findProcessedInterrogationIdsByCollectionInstrumentId(String questionnaireId) {
+ return Set.of();
+ }
+
+ @Override
+ public Set findProcessedInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(String questionnaireId, Instant sinceDate, Instant endDate) {
+ return Set.of();
+ }
+
+ @Override
+ public void resetProcessDates(String questionnaireId, Set interrogationIds) {
+ for (int i = 0; i < mongoStub.size(); i++) {
+ LunaticJsonRawDataDocument document = mongoStub.get(i);
+
+ if (document.questionnaireId().equals(questionnaireId)
+ && interrogationIds.contains(document.interrogationId())) {
+
+ LunaticJsonRawDataDocument newDocument = LunaticJsonRawDataDocument.builder()
+ .id(document.id())
+ .campaignId(document.campaignId())
+ .questionnaireId(document.questionnaireId())
+ .interrogationId(document.interrogationId())
+ .idUE(document.idUE())
+ .mode(document.mode())
+ .data(document.data())
+ .recordDate(document.recordDate())
+ .processDate(null)
+ .build();
+
+ mongoStub.set(i, newDocument);
+ }
+ }
+ }
+
+}
diff --git a/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java b/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java
index 65f5ddd6b..c6da9de62 100644
--- a/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java
+++ b/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java
@@ -4,7 +4,9 @@
import fr.insee.genesis.domain.ports.spi.SurveyUnitPersistencePort;
import lombok.Getter;
+import java.time.Instant;
import java.time.LocalDateTime;
+import java.time.ZoneId;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -135,18 +137,30 @@ public List findInterrogationIdsByQuestionnaireIdAndDateAfter(S
}
@Override
- public List findInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(String collectionInstrumentId, LocalDateTime start, LocalDateTime end) {
+ public List findInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(
+ String collectionInstrumentId,
+ Instant start,
+ Instant end
+ ) {
List surveyUnitModelList = new ArrayList<>();
- for(SurveyUnitModel surveyUnitModel : mongoStub){
- if(surveyUnitModel.getCollectionInstrumentId().equals(collectionInstrumentId)
- && !surveyUnitModel.getRecordDate().isBefore(start)
- && surveyUnitModel.getRecordDate().isBefore(end))
+ ZoneId zone = ZoneId.of("Europe/Paris");
+
+ for (SurveyUnitModel surveyUnitModel : mongoStub) {
+ Instant recordDateInstant = surveyUnitModel.getRecordDate()
+ .atZone(zone)
+ .toInstant();
+
+ if (surveyUnitModel.getCollectionInstrumentId().equals(collectionInstrumentId)
+ && !recordDateInstant.isBefore(start)
+ && recordDateInstant.isBefore(end)) {
surveyUnitModelList.add(
new SurveyUnitModel(surveyUnitModel.getInterrogationId(), surveyUnitModel.getMode())
);
+ }
}
- return surveyUnitModelList; }
+ return surveyUnitModelList;
+ }
//======== OPTIMISATIONS PERFS (START) ========
@@ -175,6 +189,16 @@ public Long deleteByCollectionInstrumentId(String collectionInstrumentId) {
return (long) mongoStub.stream().filter(su -> !su.getCollectionInstrumentId().equals(collectionInstrumentId)).toList().size();
}
+ @Override
+ public Long deleteByCollectionInstrumentIdAndInterrogationIds(String collectionInstrumentId, Set interrogationIds) {
+ return 0L;
+ }
+
+ @Override
+ public Long deleteByQuestionnaireIdAndInterrogationIds(String questionnaireId, Set interrogationIds) {
+ return 0L;
+ }
+
@Override
public long count() {
return mongoStub.size();