Skip to content

Commit ddf246a

Browse files
lmarinilmarini
authored
Merge pull request #302 from clowder-framework/release/1.19.3
Release v1.19.3
2 parents 42a3d9c + 6bdfa75 commit ddf246a

19 files changed

Lines changed: 108 additions & 41 deletions

‎CHANGELOG.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file.
44
The format is based on [Keep a Changelog](http://keepachangelog.com/)
55
and this project adheres to [Semantic Versioning](http://semver.org/).
66

7+
## 1.19.3 - 2021-11-11
8+
9+
See [fix-counts.js](scripts/updates/fix-count.js) for a script that can be run before this update (as well as the
10+
update for 1.19.0) to pre populate the migration. This will speed up the update and will not impact the running
11+
instance.
12+
13+
### Fixed
14+
- If a space has a lot of datasets, rendering the space page is very slow. Files in a space is now cached.
15+
- Set permissions for folders to be 777 this fixes [clowder-helm#5](https://github.com/clowder-framework/clowder-helm/issues/5)
16+
717
## 1.19.2 - 2021-10-20
818

919
### Fixed

‎Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ COPY docker/custom.conf docker/play.plugins /home/clowder/custom/
6767
# Containers should NOT run as root as a good practice
6868
# numeric id to be compatible with openshift, will run as random userid:0
6969
RUN mkdir -p /home/clowder/data && \
70-
chmod g+w /home/clowder/logs /home/clowder/data /home/clowder/custom
70+
chmod 777 /home/clowder/logs /home/clowder/data /home/clowder/custom
7171
USER 10001
7272

7373
# command to run when starting docker

‎app/api/Files.scala‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1678,6 +1678,7 @@ class Files @Inject()(
16781678
val ds_spaces = ds.spaces
16791679
for (ds_s <- ds_spaces) {
16801680
spaces.decrementSpaceBytes(ds_s, file.length)
1681+
spaces.decrementFileCounter(ds_s, 1)
16811682
}
16821683
}
16831684

‎app/api/Spaces.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ class Spaces @Inject()(spaces: SpaceService,
4141
val userId = request.user.get.id
4242
val c = ProjectSpace(name = name, description = description, created = new Date(), creator = userId,
4343
homePage = List.empty, logoURL = None, bannerURL = None, collectionCount = 0,
44-
datasetCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty)
44+
datasetCount = 0, fileCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty)
4545
spaces.insert(c) match {
4646
case Some(id) => {
4747
appConfig.incrementCount('spaces, 1)

‎app/controllers/SecuredController.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ trait SecuredController extends Controller {
153153
val spaces: SpaceService = DI.injector.getInstance(classOf[SpaceService])
154154
spaces.get(id) match {
155155
case None => Future.successful(BadRequest(views.html.notFound(spaceTitle + " does not exist.")(user)))
156-
case Some(space) => Future.successful(Forbidden(views.html.spaces.space(space,List(),List(),List(),List(),"", Map(),List(),0,0)(user)))
156+
case Some(space) => Future.successful(Forbidden(views.html.spaces.space(space,List(),List(),List(),List(),"", Map(),List())(user)))
157157
}
158158
}
159159

‎app/controllers/Spaces.scala‎

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -167,8 +167,6 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
167167
var creatorActual: User = null
168168
val collectionsInSpace = spaces.getCollectionsInSpace(Some(id.stringify), Some(size))
169169
val datasetsInSpace = datasets.listSpace(size, id.toString(), user)
170-
val spaceBytes : Long = s.spaceBytes
171-
val spaceFiles : Integer = getFilesPerSpace(id, user)
172170
val publicDatasetsInSpace = datasets.listSpaceStatus(size, id.toString(), "publicAll", user)
173171
val usersInSpace = spaces.getUsersInSpace(id, None)
174172
var curationObjectsInSpace: List[CurationObject] = List()
@@ -217,7 +215,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
217215
case None => List.empty
218216
}
219217
sinkService.logSpaceViewEvent(s, user)
220-
Ok(views.html.spaces.space(Utils.decodeSpaceElements(s), collectionsInSpace, publicDatasetsInSpace, datasetsInSpace, rs, play.Play.application().configuration().getString("SEADservices.uri"), userRoleMap, userSelections, spaceBytes, spaceFiles))
218+
Ok(views.html.spaces.space(Utils.decodeSpaceElements(s), collectionsInSpace, publicDatasetsInSpace, datasetsInSpace, rs, play.Play.application().configuration().getString("SEADservices.uri"), userRoleMap, userSelections))
221219
}
222220
case None => BadRequest(views.html.notFound(spaceTitle + " does not exist."))
223221
}
@@ -414,7 +412,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
414412
val newSpace = ProjectSpace(name = formData.name, description = formData.description,
415413
created = new Date, creator = userId, homePage = formData.homePage,
416414
logoURL = formData.logoURL, bannerURL = formData.bannerURL,
417-
collectionCount = 0, datasetCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty,
415+
collectionCount = 0, datasetCount = 0, fileCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty,
418416
resourceTimeToLive = formData.resourceTimeToLive * 60 * 60 * 1000L, isTimeToLiveEnabled = formData.isTimeToLiveEnabled,
419417
status = formData.access,
420418
affiliatedSpaces = formData.affSpace)
@@ -640,15 +638,4 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
640638
case None => BadRequest(views.html.notFound(spaceTitle + " does not exist."))
641639
}
642640
}
643-
644-
private def getFilesPerSpace(spaceId: UUID, user: Option[User]) : Integer = {
645-
var spaceFiles: Integer = 0
646-
val allDatasetsInSpace = datasets.listSpace(0, spaceId.toString(), user)
647-
for (ds <- allDatasetsInSpace) {
648-
val files_in_ds = ds.files.length
649-
spaceFiles += files_in_ds
650-
}
651-
spaceFiles
652-
}
653-
654641
}

‎app/models/Space.scala‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ case class ProjectSpace (
2222
bannerURL: Option[URL],
2323
collectionCount: Integer,
2424
datasetCount: Integer,
25+
fileCount: Integer,
2526
userCount: Integer,
2627
spaceBytes: Long,
2728
metadata: List[Metadata],

‎app/services/SpaceService.scala‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,10 @@ trait SpaceService {
9494

9595
def incrementCollectionCounter(collection: UUID, space: UUID, increment: Int)
9696

97+
def incrementFileCounter(space: UUID, increment: Long)
98+
99+
def decrementFileCounter(space: UUID, decrement: Long)
100+
97101
def incrementSpaceBytes(space: UUID, increment: Long)
98102

99103
def decrementSpaceBytes(space: UUID, decrement: Long)

‎app/services/mongodb/MongoDBDatasetService.scala‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1399,6 +1399,15 @@ class MongoDBDatasetService @Inject() (
13991399
}
14001400
for (f <- dataset.files) {
14011401
val notTheDataset = for (currDataset <- findByFileIdDirectlyContain(f) if !dataset.id.toString.equals(currDataset.id.toString)) yield currDataset
1402+
files.get(f) match {
1403+
case Some(file) => {
1404+
for(space <- dataset.spaces) {
1405+
spaces.decrementFileCounter(space, 1)
1406+
spaces.decrementSpaceBytes(space, file.length)
1407+
}
1408+
}
1409+
case None => Logger.error(s"Error file with with id ${f} no longer exists")
1410+
}
14021411
if (notTheDataset.size == 0)
14031412
files.removeFile(f, host, apiKey, user)
14041413
}

‎app/services/mongodb/MongoDBSpaceService.scala‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -375,15 +375,23 @@ class MongoDBSpaceService @Inject() (
375375
}
376376

377377
def decrementCollectionCounter(collection: UUID, space: UUID, decrement: Int): Unit = {
378-
ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("collectionCount" -> -1), upsert=false, multi=false, WriteConcern.Safe)
378+
ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("collectionCount" -> -1 * decrement), upsert=false, multi=false, WriteConcern.Safe)
379+
}
380+
381+
def incrementFileCounter(space: UUID, increment: Long ): Unit = {
382+
ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("fileCount" -> increment), upsert=false, multi=false, WriteConcern.Safe)
383+
}
384+
385+
def decrementFileCounter(space: UUID, decrement: Long): Unit = {
386+
ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("fileCount" -> -1 * decrement), upsert=false, multi=false, WriteConcern.Safe)
379387
}
380388

381389
def incrementSpaceBytes(space: UUID, increment: Long ): Unit = {
382390
ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> increment), upsert=false, multi=false, WriteConcern.Safe)
383391
}
384392

385393
def decrementSpaceBytes(space: UUID, decrement: Long): Unit = {
386-
ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> decrement), upsert=false, multi=false, WriteConcern.Safe)
394+
ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> -1 * decrement), upsert=false, multi=false, WriteConcern.Safe)
387395
}
388396

389397
def removeCollection(collection:UUID, space:UUID): Unit = {
@@ -398,11 +406,15 @@ class MongoDBSpaceService @Inject() (
398406
*/
399407
def addDataset(dataset: UUID, space: UUID): Unit = {
400408
log.debug(s"Space Service - Adding $dataset to $space")
401-
val datasetBytes = datasets.getBytesForDataset(dataset)
402-
datasets.addToSpace(dataset, space)
403-
ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> datasetBytes), upsert=false, multi=false, WriteConcern.Safe)
404-
ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> 1), upsert=false, multi=false, WriteConcern.Safe)
405-
409+
datasets.get(dataset) match {
410+
case Some(x) => {
411+
val datasetBytes = datasets.getBytesForDataset(dataset)
412+
datasets.addToSpace(dataset, space)
413+
ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> datasetBytes), upsert=false, multi=false, WriteConcern.Safe)
414+
ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("fileCount" -> x.files.length), upsert=false, multi=false, WriteConcern.Safe)
415+
ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> 1), upsert=false, multi=false, WriteConcern.Safe)
416+
}
417+
}
406418
}
407419

408420
/**
@@ -414,9 +426,15 @@ class MongoDBSpaceService @Inject() (
414426
def removeDataset(dataset:UUID, space:UUID): Unit = {
415427
log.debug(s"Space Service - removing $dataset from $space")
416428
datasets.removeFromSpace(dataset, space)
417-
val datasetBytes = datasets.getBytesForDataset(dataset)
418-
ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> -datasetBytes), upsert=false, multi=false, WriteConcern.Safe)
419-
ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> -1), upsert=false, multi=false, WriteConcern.Safe)
429+
datasets.get(dataset) match {
430+
case Some(x) => {
431+
val datasetBytes = datasets.getBytesForDataset(dataset)
432+
datasets.addToSpace(dataset, space)
433+
ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> -1 * datasetBytes), upsert=false, multi=false, WriteConcern.Safe)
434+
ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("fileCount" -> -1 * x.files.length), upsert=false, multi=false, WriteConcern.Safe)
435+
ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> -1), upsert=false, multi=false, WriteConcern.Safe)
436+
}
437+
}
420438
}
421439

422440
/**

0 commit comments

Comments
 (0)