Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions cds_migrator_kit/rdm/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,27 @@ Run the below command to migrate records in the created community from before:
invenio migration run
```

#### EP approval records (`--ep-approval`)

EP approval records must be migrated in a **separate stream**. Do not mix them with regular records.

1. **Dump** them filtered by `9031_:EPPHAPP`, for example (FASER):

```bash
inveniomigrator dump records -q '980__a:ARTICLE or 980__a:PREPRINT and 693:"FASER" not 980:CONFERENCEPAPER not 591__b:"Draft" 9031_:EPPHAPP -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY' --file-prefix faser-papers-cern-ep --chunk-size=1000
```

2. Configure a dedicated collection entry in `streams.yaml` pointing at that dump.

3. **Migrate** with the `--ep-approval` flag (dry run first):

```shell
invenio migration run --collection faser-ep --ep-approval --dry-run
invenio migration run --collection faser-ep --ep-approval
```

Without `--ep-approval`, the loader will reject EP approval records.

### Migrate the statistics for the successfully migrated records

When the `invenio migration run` command ends it will produce a `rdm_records_state.json` file which has linked information about the migrated records and the old system. The format will be similar to below:
Expand Down
13 changes: 11 additions & 2 deletions cds_migrator_kit/rdm/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
CommentsStreamDefinition,
)
from cds_migrator_kit.rdm.records.streams import ( # UserStreamDefinition,
RecordEPApprovalStreamDefinition,
RecordStreamDefinition,
)
from cds_migrator_kit.rdm.stats.runner import RecordStatsRunner
Expand Down Expand Up @@ -69,12 +70,20 @@ def migration():
"Can also be set per-collection in streams.yaml under transform.workers."
),
)
@click.option(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how will we use it?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I explained in the PR description
#541 (comment)

invenio migration run --collection <collection-name> --ep-approval --dry-run

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please remember to create a recipe following the other migrations we have done or doing https://gitlab.cern.ch/cds-team/cds-rdm-openshift/-/work_items?show=eyJpaWQiOiIxNSIsImZ1bGxfcGF0aCI6ImNkcy10ZWFtL2Nkcy1yZG0tb3BlbnNoaWZ0IiwiaWQiOjM3NTE2MX0%3D and take it into account there

"--ep-approval",
is_flag=True,
help="Use the EP approval load stream (pre-EP draft snapshots without legacy minting).",
)
@with_appcontext
def run(collection, dry_run=False, keep_logs=False, workers=None):
def run(collection, dry_run=False, keep_logs=False, workers=None, ep_approval=False):
"""Run."""
stream_config = current_app.config["CDS_MIGRATOR_KIT_STREAM_CONFIG"]
stream_definition = (
RecordEPApprovalStreamDefinition if ep_approval else RecordStreamDefinition
)
runner = Runner(
stream_definitions=[RecordStreamDefinition],
stream_definitions=[stream_definition],
# stream_definitions=[UserStreamDefinition],
config_filepath=Path(stream_config).absolute(),
dry_run=dry_run,
Expand Down
25 changes: 25 additions & 0 deletions cds_migrator_kit/rdm/migration_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -359,6 +359,11 @@ def _(x): # needed to avoid start time failure with lazy strings
"validator": always_valid,
"datacite": "CDS",
},
"apprn": {
"label": _("Approval Report Number"),
"validator": schemes.is_approval_report_number,
"datacite": "CDS",
},
"aleph": {
"label": _("Aleph number"),
"validator": schemes.is_aleph,
Expand Down Expand Up @@ -483,6 +488,9 @@ def resolve_record_pid(pid):
"EligibilityHRCirc": ["eligibility-retr-actual", "hr-web-gacepa"],
"CERNPeopleEligibility": ["cern-personnel", "eligibility-retr-actual"],
"FAPDepRestrFile": ["fap-dep"],
# Access group already added to the record as 506__m when draft created, no need to add:
# https://gitlab.cern.ch/cds-team/cds-legacy/-/blob/master/src/wn-cdsweb/lib/python/invenio/websubmit_functions/EPPHAPP_Test_values.py#L249-266
"EP Restricted Draft": [],
# CERN E-guide restricted docs: https://cds.cern.ch/admin/webaccess/webaccessadmin.py/showroledetails?id_role=69 CERN personnel has view rights
}

Expand Down Expand Up @@ -523,3 +531,20 @@ def resolve_record_pid(pid):

# don't generate logs for migration
AUDIT_LOGS_ENABLED = False

### EP Approval configuration only needed for local, it should use cds-rdm config for de/sandbox/prod
# ===========================
CDS_CERN_SCIENTIFIC_COMMUNITY_ID = "78b3c4aa-c4e6-4502-8226-67ba2d347afe"
"""The id of the CERN Scientific community."""

CDS_COMMITTEE_APPROVAL_COMMUNITIES = {
"dd13404c-bcd6-4b15-aeef-38d678c61ff1": {
"label": "EP approval", # shown in UI buttons/headings
"referee_group": "cds-ph-ep-publication", # CERN e-group slug
"report_number": {
"prefix": "CERN-EP", # literal prefix, e.g. "CERN-EP"
"include_year": True, # append the current year after prefix
"counter_digits": 3, # zero-padding width, e.g. 3 → "001"
},
},
}
3 changes: 2 additions & 1 deletion cds_migrator_kit/rdm/records/load/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

"""CDS-RDM Migration load package."""

from .ep_approval_load import CDSEPApprovalRecordServiceLoad
from .load import CDSRecordServiceLoad

__all__ = ("CDSRecordServiceLoad",)
__all__ = ("CDSEPApprovalRecordServiceLoad", "CDSRecordServiceLoad")
Loading
Loading