Skip to content

Commit 7468a15

Browse files
pull[bot]pull[bot]
authored
Merge pull request #46 from KelvinTegelaar/master
[pull] master from KelvinTegelaar:master
2 parents a4c083b + a41c494 commit 7468a15

37 files changed

Lines changed: 274 additions & 108 deletions

‎.github/workflows/publish_release.yml‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,36 @@ jobs:
4848
echo "tag_exists=false" >> $GITHUB_ENV
4949
fi
5050
51+
# Get Previous Tag
52+
- name: Get Previous Tag
53+
id: previous_tag
54+
if: env.tag_exists == 'false'
55+
run: |
56+
PREV_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "")
57+
echo "previous_tag=$PREV_TAG" >> $GITHUB_OUTPUT
58+
echo "Previous tag: $PREV_TAG"
59+
5160
# Generate Release Notes
5261
- name: Generate Release Notes
5362
id: changelog
5463
if: env.tag_exists == 'false'
55-
uses: mikepenz/release-changelog-builder-action@v5.0.0
64+
uses: actions/github-script@v7
65+
with:
66+
script: |
67+
const params = {
68+
owner: context.repo.owner,
69+
repo: context.repo.repo,
70+
tag_name: '${{ steps.get_version.outputs.version }}',
71+
target_commitish: context.sha
72+
};
73+
74+
const previousTag = '${{ steps.previous_tag.outputs.previous_tag }}';
75+
if (previousTag) {
76+
params.previous_tag_name = previousTag;
77+
}
78+
79+
const { data } = await github.rest.repos.generateReleaseNotes(params);
80+
core.setOutput('changelog', data.body);
5681
env:
5782
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
5883

@@ -75,10 +100,6 @@ jobs:
75100
if: env.tag_exists == 'false'
76101
run: |
77102
mkdir -p src/releases
78-
zip -r src/releases/release_${{ steps.get_version.outputs.version }}.zip . \
79-
--exclude "./src/releases/*" \
80-
--exclude ".*" \
81-
--exclude ".*/**"
82103
zip -r src/releases/latest.zip . \
83104
--exclude "./src/releases/*" \
84105
--exclude ".*" \

‎Modules/CIPPCore/Public/Add-CIPPDbItem.ps1‎

Lines changed: 42 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -69,23 +69,55 @@ function Add-CIPPDbItem {
6969
if ($ExistingEntities) {
7070
Remove-AzDataTableEntity @Table -Entity $ExistingEntities -Force | Out-Null
7171
}
72-
$Entities = foreach ($Item in $Data) {
73-
$ItemId = $Item.id ?? $Item.ExternalDirectoryObjectId ?? $Item.Identity ?? $Item.skuId
74-
@{
75-
PartitionKey = $TenantFilter
76-
RowKey = Format-RowKey "$Type-$ItemId"
77-
Data = [string]($Item | ConvertTo-Json -Depth 10 -Compress)
78-
Type = $Type
72+
73+
# Calculate batch size based on available memory
74+
$AvailableMemory = [System.GC]::GetTotalMemory($false)
75+
$AvailableMemoryMB = [math]::Round($AvailableMemory / 1MB, 2)
76+
77+
# Estimate item size from first item (with fallback)
78+
$EstimatedItemSizeBytes = 1KB # Default assumption
79+
if ($Data.Count -gt 0) {
80+
$SampleJson = $Data[0] | ConvertTo-Json -Depth 10 -Compress
81+
$EstimatedItemSizeBytes = [System.Text.Encoding]::UTF8.GetByteCount($SampleJson)
82+
}
83+
84+
# Use 25% of available memory for batch processing, with min/max bounds
85+
$TargetBatchMemoryMB = [Math]::Max(50, $AvailableMemoryMB * 0.25)
86+
$CalculatedBatchSize = [Math]::Floor(($TargetBatchMemoryMB * 1MB) / $EstimatedItemSizeBytes)
87+
# Reduce max to 500 to prevent OOM with large datasets
88+
$BatchSize = [Math]::Max(100, [Math]::Min(500, $CalculatedBatchSize))
89+
90+
$TotalCount = $Data.Count
91+
$ProcessedCount = 0
92+
Write-Information "Adding $TotalCount items of type $Type to CIPP Reporting DB for tenant $TenantFilter | Available Memory: ${AvailableMemoryMB}MB | Target Memory: ${TargetBatchMemoryMB}MB | Calculated: $CalculatedBatchSize | Batch Size: $BatchSize (est. item size: $([math]::Round($EstimatedItemSizeBytes/1KB, 2))KB)"
93+
for ($i = 0; $i -lt $TotalCount; $i += $BatchSize) {
94+
$BatchEnd = [Math]::Min($i + $BatchSize, $TotalCount)
95+
$Batch = $Data[$i..($BatchEnd - 1)]
96+
97+
$Entities = foreach ($Item in $Batch) {
98+
$ItemId = $Item.id ?? $Item.ExternalDirectoryObjectId ?? $Item.Identity ?? $Item.skuId
99+
@{
100+
PartitionKey = $TenantFilter
101+
RowKey = Format-RowKey "$Type-$ItemId"
102+
Data = [string]($Item | ConvertTo-Json -Depth 10 -Compress)
103+
Type = $Type
104+
}
79105
}
106+
107+
Add-CIPPAzDataTableEntity @Table -Entity $Entities -Force | Out-Null
108+
$ProcessedCount += $Batch.Count
109+
110+
# Clear batch variables to free memory
111+
$Entities = $null
112+
$Batch = $null
113+
[System.GC]::Collect()
80114
}
81-
Add-CIPPAzDataTableEntity @Table -Entity $Entities -Force | Out-Null
82115

83116
}
84-
85117
Write-LogMessage -API 'CIPPDbItem' -tenant $TenantFilter -message "Added $($Data.Count) items of type $Type$(if ($Count) { ' (count mode)' })" -sev Debug
86118

87119
} catch {
88-
Write-LogMessage -API 'CIPPDbItem' -tenant $TenantFilter -message "Failed to add items of type $Type : $($_.Exception.Message)" -sev Error
120+
Write-LogMessage -API 'CIPPDbItem' -tenant $TenantFilter -message "Failed to add items of type $Type : $($_.Exception.Message)" -sev Error -LogData (Get-CippException -Exception $_)
89121
throw
90122
}
91123
}

‎Modules/CIPPCore/Public/Add-CippTestResult.ps1‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ function Add-CippTestResult {
4848
[string]$TestId,
4949

5050
[Parameter(Mandatory = $false)]
51-
[string]$testType = 'Identity',
51+
[string]$TestType = 'Identity',
5252

5353
[Parameter(Mandatory = $true)]
5454
[string]$Status,

‎Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Mailbox Permissions/Push-StoreMailboxPermissions.ps1‎

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -44,15 +44,21 @@ function Push-StoreMailboxPermissions {
4444
# Results are grouped by cmdlet name due to ReturnWithCommand
4545
if ($ActualResult['Get-MailboxPermission']) {
4646
Write-Information "Adding $($ActualResult['Get-MailboxPermission'].Count) mailbox permissions"
47-
$AllMailboxPermissions.AddRange($ActualResult['Get-MailboxPermission'])
47+
foreach ($perm in $ActualResult['Get-MailboxPermission']) {
48+
$AllMailboxPermissions.Add($perm)
49+
}
4850
}
4951
if ($ActualResult['Get-RecipientPermission']) {
5052
Write-Information "Adding $($ActualResult['Get-RecipientPermission'].Count) recipient permissions"
51-
$AllRecipientPermissions.AddRange($ActualResult['Get-RecipientPermission'])
53+
foreach ($perm in $ActualResult['Get-RecipientPermission']) {
54+
$AllRecipientPermissions.Add($perm)
55+
}
5256
}
5357
if ($ActualResult['Get-MailboxFolderPermission']) {
5458
Write-Information "Adding $($ActualResult['Get-MailboxFolderPermission'].Count) calendar permissions"
55-
$AllCalendarPermissions.AddRange($ActualResult['Get-MailboxFolderPermission'])
59+
foreach ($perm in $ActualResult['Get-MailboxFolderPermission']) {
60+
$AllCalendarPermissions.Add($perm)
61+
}
5662
}
5763
} else {
5864
Write-Information "Skipping non-hashtable result: $($ActualResult.GetType().Name)"
@@ -61,30 +67,47 @@ function Push-StoreMailboxPermissions {
6167

6268
# Combine all permissions (mailbox and recipient) into a single collection
6369
$AllPermissions = [System.Collections.Generic.List[object]]::new()
64-
$AllPermissions.AddRange($AllMailboxPermissions)
65-
$AllPermissions.AddRange($AllRecipientPermissions)
70+
foreach ($perm in $AllMailboxPermissions) {
71+
$AllPermissions.Add($perm)
72+
}
73+
foreach ($perm in $AllRecipientPermissions) {
74+
$AllPermissions.Add($perm)
75+
}
6676

6777
Write-Information "Aggregated $($AllPermissions.Count) total permissions ($($AllMailboxPermissions.Count) mailbox + $($AllRecipientPermissions.Count) recipient)"
6878
Write-Information "Aggregated $($AllCalendarPermissions.Count) calendar permissions"
6979

7080
# Store all permissions together as MailboxPermissions
7181
if ($AllPermissions.Count -gt 0) {
72-
Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'MailboxPermissions' -Data $AllPermissions
73-
Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'MailboxPermissions' -Data $AllPermissions -Count
82+
Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'MailboxPermissions' -Data $AllPermissions.ToArray()
83+
Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'MailboxPermissions' -Data @{ Count = $AllPermissions.Count } -Count
7484
Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message "Cached $($AllPermissions.Count) mailbox permission records" -sev Info
7585
} else {
7686
Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message 'No mailbox permissions found to cache' -sev Info
7787
}
7888

89+
# Clear to free memory before processing calendar permissions
90+
$AllMailboxPermissions.Clear()
91+
$AllRecipientPermissions.Clear()
92+
$AllPermissions.Clear()
93+
$AllMailboxPermissions = $null
94+
$AllRecipientPermissions = $null
95+
$AllPermissions = $null
96+
7997
# Store calendar permissions separately
8098
if ($AllCalendarPermissions.Count -gt 0) {
81-
Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'CalendarPermissions' -Data $AllCalendarPermissions
82-
Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'CalendarPermissions' -Data $AllCalendarPermissions -Count
99+
Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'CalendarPermissions' -Data $AllCalendarPermissions.ToArray()
100+
Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'CalendarPermissions' -Data @{ Count = $AllCalendarPermissions.Count } -Count
83101
Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message "Cached $($AllCalendarPermissions.Count) calendar permission records" -sev Info
84102
} else {
85103
Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message 'No calendar permissions found to cache' -sev Info
86104
}
87105

106+
# Final cleanup
107+
$AllCalendarPermissions.Clear()
108+
$AllCalendarPermissions = $null
109+
[System.GC]::Collect()
110+
88111
return
89112

90113
} catch {

‎Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Push-CIPPDBCacheData.ps1‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ function Push-CIPPDBCacheData {
9999
'ExoPresetSecurityPolicy'
100100
'ExoTenantAllowBlockList'
101101
'Mailboxes'
102+
'CASMailboxes'
102103
'MailboxUsage'
103104
'OneDriveUsage'
104105
)
@@ -178,12 +179,11 @@ function Push-CIPPDBCacheData {
178179
OrchestratorName = "CIPPDBCacheTenant_$TenantFilter"
179180
Batch = @($Batch)
180181
SkipLog = $true
181-
DurableMode = 'Sequence'
182182
}
183183

184184
if ($Item.TestRun -eq $true) {
185185
$InputObject | Add-Member -NotePropertyName PostExecution -NotePropertyValue @{
186-
FunctionName = 'CIPPTestsRun'
186+
FunctionName = 'CIPPDBTestsRun'
187187
Parameters = @{
188188
TenantFilter = $TenantFilter
189189
}

‎Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Tests/Invoke-CIPPTestsRun.ps1‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
function Invoke-CIPPTestsRun {
1+
function Invoke-CIPPDBTestsRun {
22
<#
33
.FUNCTIONALITY
44
Entrypoint
@@ -13,6 +13,17 @@ function Invoke-CIPPTestsRun {
1313

1414
Write-Information "Starting tests run for tenant: $TenantFilter"
1515

16+
Write-Host 'Checking rerun protection'
17+
$RerunParams = @{
18+
TenantFilter = $TenantFilter
19+
Type = 'CippTests'
20+
API = 'CippTests'
21+
}
22+
$Rerun = Test-CIPPRerun @RerunParams
23+
if ($Rerun -eq $true) {
24+
Write-Host "rerun is true for $($TenantFilter)"
25+
return $true
26+
}
1627
try {
1728
$AllTests = Get-Command -Name 'Invoke-CippTest*' -Module CIPPCore | Select-Object -ExpandProperty Name | ForEach-Object {
1829
$_ -replace '^Invoke-CippTest', ''

‎Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Tests/Push-CIPPTest.ps1‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ function Push-CIPPTest {
2222

2323
Write-Information "Executing $FunctionName for $TenantFilter"
2424
& $FunctionName -Tenant $TenantFilter
25+
Write-Host "Returning true, test has run for $tenantFilter"
26+
return @{ testRun = $true }
2527

2628
} catch {
2729
$ErrorMessage = Get-CippException -Exception $_

‎Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Tests/Push-CIPPTestsRun.ps1‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
function Push-CIPPTestsRun {
1+
function Push-CIPPDBTestsRun {
22
<#
33
.SYNOPSIS
44
PostExecution function to run tests after data collection completes
@@ -13,7 +13,7 @@ function Push-CIPPTestsRun {
1313
Write-LogMessage -API 'Tests' -tenant $TenantFilter -message 'Starting test run after data collection' -sev Info
1414

1515
# Call the test run function
16-
$Result = Invoke-CIPPTestsRun -TenantFilter $TenantFilter
16+
$Result = Invoke-CIPPDBTestsRun -TenantFilter $TenantFilter
1717

1818
Write-LogMessage -API 'Tests' -tenant $TenantFilter -message "Test run started. Instance ID: $($Result.InstanceId)" -sev Info
1919
Write-Information "PostExecution: Tests started with Instance ID: $($Result.InstanceId)"

‎Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Email-Exchange/Reports/Invoke-ListMailboxCAS.ps1‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
Function Invoke-ListMailboxCAS {
1+
function Invoke-ListMailboxCAS {
22
<#
33
.FUNCTIONALITY
44
Entrypoint
@@ -10,7 +10,7 @@ Function Invoke-ListMailboxCAS {
1010
# Interact with query parameters or the body of the request.
1111
$TenantFilter = $Request.Query.TenantFilter
1212
try {
13-
$GraphRequest = New-GraphGetRequest -uri "https://outlook.office365.com/adminapi/beta/$($tenantfilter)/CasMailbox" -Tenantid $tenantfilter -scope ExchangeOnline | Select-Object @{ Name = 'displayName'; Expression = { $_.'DisplayName' } },
13+
$GraphRequest = New-ExoRequest -tenantid $TenantFilter -cmdlet 'Get-CasMailbox' | Select-Object @{ Name = 'displayName'; Expression = { $_.'DisplayName' } },
1414
@{ Name = 'primarySmtpAddress'; Expression = { $_.'PrimarySMTPAddress' } },
1515
@{ Name = 'ecpenabled'; Expression = { $_.'ECPEnabled' } },
1616
@{ Name = 'owaenabled'; Expression = { $_.'OWAEnabled' } },

‎Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Invoke-ListAvailableTests.ps1‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
function Invoke-ListAvailableTests {
22
<#
33
.FUNCTIONALITY
4-
Entrypoint
4+
Entrypoint,AnyTenant
55
.ROLE
66
CIPP.Dashboard.Read
77
#>

0 commit comments

Comments
 (0)