From 36056a8d75058cfc1ed54aa6b2bac4b1922173a8 Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Thu, 30 Jul 2026 13:00:57 +0200 Subject: [PATCH] ZCU-PUB/Add a configurable noindex meta tag for item pages Adds item.noIndex - a list of item uuids or handles whose item page gets and has its citation_pdf_url dropped. Non-discoverable items get the tag too. Server-side rendered, so crawlers see the tag in the first HTML response. Emitted via addMetaTag() so clearMetaTags() removes it on the next route change instead of leaking it onto every page visited afterwards. No nofollow: crawlers should keep following bitstream links to pick up the X-Robots-Tag served for the files. Ships with an empty list on purpose - this config is transferred to the browser unsanitized, so entries would be publicly readable in every page's HTML. Guards on Array.isArray() because a misconfigured scalar would otherwise throw from the first statement of setDSOMetaTags() and strip the meta tags off every page. Tests: metadata.service.spec.ts 31 SUCCESS (19 existing + 12 new); reverting the production change turns 6 of them red. Co-Authored-By: Claude Opus 5 (1M context) --- config/config.example.yml | 2 + config/config.yml | 3 + .../core/metadata/metadata.service.spec.ts | 89 ++++++++++++++++++- src/app/core/metadata/metadata.service.ts | 49 ++++++++++ src/app/shared/mocks/item.mock.ts | 9 ++ src/config/default-app-config.ts | 2 + src/config/item-config.interface.ts | 3 + src/environments/environment.test.ts | 2 + 8 files changed, 158 insertions(+), 1 deletion(-) diff --git a/config/config.example.yml b/config/config.example.yml index 9aae1fc79b4..b43d15196db 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -286,6 +286,8 @@ item: undoTimeout: 10000 # 10 seconds # Show the item access status label in items lists showAccessStatuses: false + # Item uuids or handles to exclude from search engine indexes. Empty = off. + noIndex: [] bitstream: # Number of entries in the bitstream list in the item view page. # Rounded to the nearest size in the list of selectable sizes on the diff --git a/config/config.yml b/config/config.yml index cdf957944f8..f1c986ed661 100644 --- a/config/config.yml +++ b/config/config.yml @@ -192,3 +192,6 @@ item: bitstream: # Per-bitstream embargo-date badge in the Item View showAccessStatuses: true + # Item uuids or handles to exclude from search engine indexes. Empty = off. + # Keep empty: this config is served to the browser, so entries here are public. + noIndex: [] diff --git a/src/app/core/metadata/metadata.service.spec.ts b/src/app/core/metadata/metadata.service.spec.ts index fac3937643e..bc07445554a 100644 --- a/src/app/core/metadata/metadata.service.spec.ts +++ b/src/app/core/metadata/metadata.service.spec.ts @@ -12,8 +12,10 @@ import { ItemMock, MockBitstream1, MockBitstream3, - MockBitstream2 + MockBitstream2, + NonDiscoverableItemMock } from '../../shared/mocks/item.mock'; +import { DSpaceObject } from '../shared/dspace-object.model'; import { createSuccessfulRemoteDataObject, createSuccessfulRemoteDataObject$ } from '../../shared/remote-data.utils'; import { PaginatedList } from '../data/paginated-list.model'; import { Bitstream } from '../shared/bitstream.model'; @@ -96,6 +98,7 @@ describe('MetadataService', () => { appConfig = { item: { + noIndex: [], bitstream: { pageSize: 5 } @@ -406,6 +409,90 @@ describe('MetadataService', () => { }); }); + describe('robots meta tag', () => { + const noIndexTag = { name: 'robots', content: 'noindex, noarchive' }; + + const routeTo = (dso: any) => { + (metadataService as any).processRouteChange({ + data: { value: { dso: createSuccessfulRemoteDataObject(dso) } } + }); + tick(); + }; + + it('should not add a robots tag for a normal discoverable item', fakeAsync(() => { + routeTo(ItemMock); + expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'robots' })); + })); + + it('should add a robots noindex tag for a non-discoverable item', fakeAsync(() => { + routeTo(NonDiscoverableItemMock); + expect(meta.addTag).toHaveBeenCalledWith(noIndexTag); + })); + + it('should add a robots noindex tag when the item uuid is configured', fakeAsync(() => { + appConfig.item.noIndex = ['0ec7ff22-f211-40ab-a69e-c819b0b1f357']; + routeTo(ItemMock); + expect(meta.addTag).toHaveBeenCalledWith(noIndexTag); + })); + + it('should add a robots noindex tag when the item handle is configured', fakeAsync(() => { + appConfig.item.noIndex = ['10673/6']; + routeTo(ItemMock); + expect(meta.addTag).toHaveBeenCalledWith(noIndexTag); + })); + + it('should normalize handle URLs, casing and whitespace in item.noIndex', fakeAsync(() => { + appConfig.item.noIndex = [' HTTP://hdl.handle.net/10673/6 ']; + routeTo(ItemMock); + expect(meta.addTag).toHaveBeenCalledWith(noIndexTag); + })); + + it('should not add a robots tag for an item that is not configured', fakeAsync(() => { + appConfig.item.noIndex = ['11025/9501', 'f4c45569-cdfc-4b3d-98df-46bfeba016b9']; + routeTo(ItemMock); + expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'robots' })); + })); + + it('should not add a robots tag for a non-Item DSpaceObject', fakeAsync(() => { + appConfig.item.noIndex = ['10673/6']; + routeTo(Object.assign(new DSpaceObject(), { uuid: '10673/6', handle: '10673/6', metadata: {} })); + expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'robots' })); + })); + + it('should register the robots tag in the meta tag store so it is cleared on the next route change', fakeAsync(() => { + appConfig.item.noIndex = ['10673/6']; + routeTo(ItemMock); + expect(store.dispatch).toHaveBeenCalledWith(new AddMetaTagAction('robots')); + })); + + it('should suppress citation_pdf_url for a noindex item', fakeAsync(() => { + appConfig.item.noIndex = ['10673/6']; + routeTo(ItemMock); + expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_pdf_url' })); + expect(meta.addTag).toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_title' })); + })); + + it('should keep citation_pdf_url for a normal item', fakeAsync(() => { + routeTo(ItemMock); + expect(meta.addTag).toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_pdf_url' })); + })); + + it('should not break the other meta tags when item.noIndex is a scalar instead of a list', fakeAsync(() => { + // Must degrade to "off", never throw - that would strip the meta tags off every page. + appConfig.item.noIndex = '10673/6' as any; + expect(() => routeTo(ItemMock)).not.toThrow(); + expect(meta.addTag).toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_title' })); + expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'robots' })); + })); + + it('should ignore non-string entries in item.noIndex without throwing', fakeAsync(() => { + appConfig.item.noIndex = [9501 as any, null, '10673/6']; + expect(() => routeTo(ItemMock)).not.toThrow(); + expect(meta.addTag).toHaveBeenCalledWith(noIndexTag); + expect(meta.addTag).toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_title' })); + })); + }); + describe(`when there's no bitstream with an allowed format on the first page`, () => { let bitstreams; diff --git a/src/app/core/metadata/metadata.service.ts b/src/app/core/metadata/metadata.service.ts index e774dd342c7..8da1f501e6e 100644 --- a/src/app/core/metadata/metadata.service.ts +++ b/src/app/core/metadata/metadata.service.ts @@ -63,6 +63,9 @@ const tagsInUseSelector = (state: MetaTagState) => state.tagsInUse, ); +// No `nofollow`: crawlers should still follow the bitstream links to pick up their own noindex. +export const NO_INDEX_META_CONTENT = 'noindex, noarchive'; + @Injectable() export class MetadataService { @@ -147,6 +150,8 @@ export class MetadataService { private setDSOMetaTags(): void { + this.setNoIndexTag(); + this.setTitleTag(); this.setDescriptionTag(); @@ -194,6 +199,46 @@ export class MetadataService { } + /** + * Add for Items that must not be indexed by search engines. + * + * Uses addMetaTag() so the tag is registered in the meta tag store and cleared on the next route + * change; this.meta.addTag() would leak it onto every page visited afterwards. + */ + protected setNoIndexTag(): void { + if (this.isNoIndex()) { + this.addMetaTag('robots', NO_INDEX_META_CONTENT); + } + } + + private isNoIndex(): boolean { + if (!(this.currentObject.value instanceof Item)) { + return false; + } + const item = this.currentObject.value as Item; + if (item.isDiscoverable === false) { + return true; + } + // Array.isArray, not hasNoValue: a misconfigured scalar also has a length, and throwing here + // would strip the meta tags off every page. + const configured = this.appConfig?.item?.noIndex; + if (!Array.isArray(configured) || configured.length === 0) { + return false; + } + const itemIds = [item.uuid, item.handle] + .filter((id) => isNotEmpty(id)) + .map((id) => this.normalizeNoIndexId(id)); + return configured.some((id: any) => typeof id === 'string' && isNotEmpty(id) + && itemIds.includes(this.normalizeNoIndexId(id))); + } + + // Accepts a bare handle, a hdl.handle.net URL or a uuid, in any casing. + private normalizeNoIndexId(id: string): string { + return id.trim().toLowerCase() + .replace(/^https?:\/\/hdl\.handle\.net\//, '') + .replace(/^\/+/, ''); + } + /** * Add to the */ @@ -349,6 +394,10 @@ export class MetadataService { * Add to the */ private setCitationPdfUrlTag(): void { + // Google Scholar keys off this tag and does not reliably honour the landing page robots tag. + if (this.isNoIndex()) { + return; + } if (this.currentObject.value instanceof Item) { const item = this.currentObject.value as Item; diff --git a/src/app/shared/mocks/item.mock.ts b/src/app/shared/mocks/item.mock.ts index 77685cca9ac..1e78ebc98f6 100644 --- a/src/app/shared/mocks/item.mock.ts +++ b/src/app/shared/mocks/item.mock.ts @@ -294,3 +294,12 @@ export const ItemMock: Item = Object.assign(new Item(), { ) }); /* eslint-enable @typescript-eslint/no-shadow */ + +// `metadata` is copied because spec helpers mutate it in place and would hit ItemMock too. +export const NonDiscoverableItemMock: Item = Object.assign(new Item(), ItemMock, { + handle: '10673/7', + id: '0ec7ff22-f211-40ab-a69e-c819b0b1f358', + uuid: '0ec7ff22-f211-40ab-a69e-c819b0b1f358', + isDiscoverable: false, + metadata: Object.assign({}, ItemMock.metadata), +}); diff --git a/src/config/default-app-config.ts b/src/config/default-app-config.ts index fad2b56ab33..892a8f1089f 100644 --- a/src/config/default-app-config.ts +++ b/src/config/default-app-config.ts @@ -281,6 +281,8 @@ export class DefaultAppConfig implements AppConfig { }, // Show the item access status label in items lists showAccessStatuses: false, + // Item uuids or handles to exclude from search engine indexes + noIndex: [], bitstream: { // Number of entries in the bitstream list in the item view page. // Rounded to the nearest size in the list of selectable sizes on the diff --git a/src/config/item-config.interface.ts b/src/config/item-config.interface.ts index f3141f77238..3b392795fa8 100644 --- a/src/config/item-config.interface.ts +++ b/src/config/item-config.interface.ts @@ -7,6 +7,9 @@ export interface ItemConfig extends Config { // This is used to show the access status label of items in results lists showAccessStatuses: boolean; + // Item uuids or handles to exclude from search engine indexes. Empty = off. + noIndex: string[]; + bitstream: { // Number of entries in the bitstream list in the item view page. // Rounded to the nearest size in the list of selectable sizes on the diff --git a/src/environments/environment.test.ts b/src/environments/environment.test.ts index 46164ae5da4..2be8007f1ff 100644 --- a/src/environments/environment.test.ts +++ b/src/environments/environment.test.ts @@ -250,6 +250,8 @@ export const environment: BuildConfig = { }, // Show the item access status label in items lists showAccessStatuses: false, + // Items excluded from search engine indexes (uuids or handles) + noIndex: [], bitstream: { // Number of entries in the bitstream list in the item view page. // Rounded to the nearest size in the list of selectable sizes on the