Skip to content

Commit 3780743

Browse files
SteveL-MSFTSteveL-MSFT
authored
Merge pull request #1366 from SteveL-MSFT/config-resource-discovery
Enable re-discovery of resources during deployment
2 parents 0598cbe + b936f84 commit 3780743

13 files changed

Lines changed: 302 additions & 96 deletions

File tree

‎build.helpers.psm1‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -651,7 +651,14 @@ function Install-Protobuf {
651651
} elseif ($IsWindows) {
652652
if (Test-CommandAvailable -Name 'winget') {
653653
Write-Verbose -Verbose "Using winget to install Protobuf"
654-
winget install Google.Protobuf --accept-source-agreements --accept-package-agreements --source winget --silent
654+
winget install Google.Protobuf --accept-source-agreements --accept-package-agreements --source winget --force
655+
# need to add to PATH
656+
$protocFolder = "$env:USERPROFILE\AppData\Local\Microsoft\WinGet\Packages\Google.Protobuf_Microsoft.Winget.Source_8wekyb3d8bbwe\bin"
657+
if (Test-Path $protocFolder) {
658+
$env:PATH += ";$protocFolder"
659+
} else {
660+
throw "protoc folder not found after installation: $protocFolder"
661+
}
655662
} else {
656663
Write-Warning "winget not found, please install Protobuf manually"
657664
}
@@ -665,7 +672,7 @@ function Install-Protobuf {
665672
}
666673

667674
if ($LASTEXITCODE -ne 0) {
668-
throw "Failed to install Protobuf"
675+
throw "Failed to install Protobuf: $LASTEXITCODE"
669676
}
670677
}
671678
}

‎dsc/tests/dsc_discovery.tests.ps1‎

Lines changed: 105 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@
33

44
Describe 'tests for resource discovery' {
55
BeforeAll {
6-
$env:DSC_RESOURCE_PATH = $testdrive
7-
86
$script:lookupTableFilePath = if ($IsWindows) {
97
Join-Path $env:LocalAppData "dsc\AdaptedResourcesLookupTable.json"
108
} else {
@@ -16,10 +14,6 @@ Describe 'tests for resource discovery' {
1614
Remove-Item -Path "$testdrive/test.dsc.resource.*" -ErrorAction SilentlyContinue
1715
}
1816

19-
AfterAll {
20-
$env:DSC_RESOURCE_PATH = $null
21-
}
22-
2317
It 'Use DSC_RESOURCE_PATH instead of PATH when defined' {
2418
$resourceJson = @'
2519
{
@@ -31,75 +25,85 @@ Describe 'tests for resource discovery' {
3125
}
3226
}
3327
'@
34-
35-
Set-Content -Path "$testdrive/test.dsc.resource.json" -Value $resourceJson
36-
$resources = dsc resource list | ConvertFrom-Json
37-
$resources.Count | Should -Be 1
38-
$resources.type | Should -BeExactly 'DSC/TestPathResource'
28+
try {
29+
$oldPath = $env:PATH
30+
$env:DSC_RESOURCE_PATH = $testdrive
31+
Set-Content -Path "$testdrive/test.dsc.resource.json" -Value $resourceJson
32+
$resources = dsc resource list | ConvertFrom-Json
33+
$resources.Count | Should -Be 1
34+
$resources.type | Should -BeExactly 'DSC/TestPathResource'
35+
}
36+
finally {
37+
$env:PATH = $oldPath
38+
$env:DSC_RESOURCE_PATH = $null
39+
}
3940
}
4041

41-
It 'support discovering <extension>' -TestCases @(
42-
@{ extension = 'yaml' }
43-
@{ extension = 'yml' }
44-
) {
45-
param($extension)
46-
47-
$resourceYaml = @'
48-
$schema: https://aka.ms/dsc/schemas/v3/bundled/resource/manifest.json
49-
type: DSC/TestYamlResource
50-
version: 0.1.0
51-
get:
52-
executable: dsc
42+
Context 'Forced discovery using $testdrive' {
43+
BeforeAll {
44+
$env:DSC_RESOURCE_PATH = $testdrive
45+
}
46+
47+
AfterAll {
48+
$env:DSC_RESOURCE_PATH = $null
49+
}
50+
51+
It 'support discovering <extension>' -TestCases @(
52+
@{ extension = 'yaml' }
53+
@{ extension = 'yml' }
54+
) {
55+
param($extension)
56+
57+
$resourceYaml = @'
58+
$schema: https://aka.ms/dsc/schemas/v3/bundled/resource/manifest.json
59+
type: DSC/TestYamlResource
60+
version: 0.1.0
61+
get:
62+
executable: dsc
5363
'@
5464

55-
Set-Content -Path "$testdrive/test.dsc.resource.$extension" -Value $resourceYaml
56-
$resources = dsc resource list | ConvertFrom-Json
57-
$resources.Count | Should -Be 1
58-
$resources.type | Should -BeExactly 'DSC/TestYamlResource'
59-
}
65+
Set-Content -Path "$testdrive/test.dsc.resource.$extension" -Value $resourceYaml
66+
$resources = dsc resource list | ConvertFrom-Json
67+
$resources.Count | Should -Be 1
68+
$resources.type | Should -BeExactly 'DSC/TestYamlResource'
69+
}
6070

61-
It 'does not support discovering a file with an extension that is not json or yaml' {
62-
param($extension)
71+
It 'does not support discovering a file with an extension that is not json or yaml' {
72+
param($extension)
6373

64-
$resourceInput = @'
65-
$schema: https://aka.ms/dsc/schemas/v3/bundled/resource/manifest.json
66-
type: DSC/TestYamlResource
67-
version: 0.1.0
68-
get:
69-
executable: dsc
74+
$resourceInput = @'
75+
$schema: https://aka.ms/dsc/schemas/v3/bundled/resource/manifest.json
76+
type: DSC/TestYamlResource
77+
version: 0.1.0
78+
get:
79+
executable: dsc
7080
'@
7181

72-
Set-Content -Path "$testdrive/test.dsc.resource.txt" -Value $resourceInput
73-
$resources = dsc resource list | ConvertFrom-Json
74-
$resources.Count | Should -Be 0
75-
}
82+
Set-Content -Path "$testdrive/test.dsc.resource.txt" -Value $resourceInput
83+
$resources = dsc resource list | ConvertFrom-Json
84+
$resources.Count | Should -Be 0
85+
}
7686

77-
It 'warns on invalid semver' {
78-
$manifest = @'
79-
{
80-
"$schema": "https://aka.ms/dsc/schemas/v3/bundled/resource/manifest.json",
81-
"type": "Test/InvalidSemver",
82-
"version": "1.1.0..1",
83-
"get": {
84-
"executable": "dsctest"
85-
},
86-
"schema": {
87-
"command": {
87+
It 'warns on invalid semver' {
88+
$manifest = @'
89+
{
90+
"$schema": "https://aka.ms/dsc/schemas/v3/bundled/resource/manifest.json",
91+
"type": "Test/InvalidSemver",
92+
"version": "1.1.0..1",
93+
"get": {
8894
"executable": "dsctest"
95+
},
96+
"schema": {
97+
"command": {
98+
"executable": "dsctest"
99+
}
89100
}
90101
}
91-
}
92102
'@
93-
$oldPath = $env:DSC_RESOURCE_PATH
94-
try {
95-
$env:DSC_RESOURCE_PATH = $testdrive
96103
Set-Content -Path "$testdrive/test.dsc.resource.json" -Value $manifest
97104
$null = dsc resource list 2> "$testdrive/error.txt"
98105
"$testdrive/error.txt" | Should -FileContentMatchExactly 'WARN.*?does not use semver' -Because (Get-Content -Raw "$testdrive/error.txt")
99106
}
100-
finally {
101-
$env:DSC_RESOURCE_PATH = $oldPath
102-
}
103107
}
104108

105109
It 'Ensure List operation populates adapter lookup table' {
@@ -296,4 +300,47 @@ Describe 'tests for resource discovery' {
296300
$env:DSC_RESOURCE_PATH = $null
297301
}
298302
}
303+
304+
It 'Resource discovery can be set to <mode>' -TestCases @(
305+
@{ namespace = 'Microsoft.DSC'; mode = 'preDeployment' }
306+
@{ namespace = 'Microsoft.DSC'; mode = 'duringDeployment' }
307+
@{ namespace = 'Ignore'; mode = 'ignore' }
308+
) {
309+
param($namespace, $mode)
310+
311+
$guid = (New-Guid).Guid.Replace('-', '')
312+
$manifestPath = Join-Path (Split-Path (Get-Command dscecho -ErrorAction Stop).Source -Parent) echo.dsc.resource.json
313+
314+
$config_yaml = @"
315+
`$schema: https://aka.ms/dsc/schemas/v3/bundled/config/manifest.json
316+
metadata:
317+
${namespace}:
318+
resourceDiscovery: $mode
319+
resources:
320+
- type: Test/CopyResource
321+
name: This should be found and executed
322+
properties:
323+
sourceFile: $manifestPath
324+
typeName: "Test/$guid"
325+
- type: Test/$guid
326+
name: This is the new resource
327+
properties:
328+
output: Hello World
329+
"@
330+
$out = dsc -l trace config get -i $config_yaml 2> "$testdrive/tracing.txt"
331+
$traceLog = Get-Content -Raw -Path "$testdrive/tracing.txt"
332+
if ($mode -ne 'duringDeployment') {
333+
$LASTEXITCODE | Should -Be 2
334+
$out | Should -BeNullOrEmpty
335+
$traceLog | Should -Match "ERROR.*?Resource not found: Test/$guid"
336+
$traceLog | Should -Not -Match "Invoking get for 'Test/CopyResource'"
337+
} else {
338+
$LASTEXITCODE | Should -Be 0 -Because (Get-Content -Raw -Path "$testdrive/tracing.txt")
339+
$output = $out | ConvertFrom-Json
340+
$output.results[0].result.actualState.typeName | Should -BeExactly "Test/$guid" -Because $out
341+
$output.results[1].result.actualState.output | Should -BeExactly 'Hello World' -Because $out
342+
$traceLog | Should -Match "Invoking get for 'Test/$guid'"
343+
$traceLog | Should -Match "Skipping resource discovery due to 'resourceDiscovery' mode set to 'DuringDeployment'"
344+
}
345+
}
299346
}

‎lib/dsc-lib/locales/en-us.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ skippingOutput = "Skipping output for '%{name}' due to condition evaluating to f
8484
secureOutputSkipped = "Secure output '%{name}' is skipped"
8585
outputTypeNotMatch = "Output '%{name}' type does not match expected type '%{expected_type}'"
8686
copyNotSupported = "Copy for output '%{name}' is currently not supported"
87+
skippingResourceDiscovery = "Skipping resource discovery due to 'resourceDiscovery' mode set to 'DuringDeployment'"
8788

8889
[configure.parameters]
8990
importingParametersFromComplexInput = "Importing parameters from complex input"

‎lib/dsc-lib/src/configure/config_doc.rs‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -60,34 +60,46 @@ pub enum RestartRequired {
6060
Process(Process),
6161
}
6262

63+
#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, JsonSchema, DscRepoSchema)]
64+
#[serde(rename_all = "camelCase")]
65+
#[dsc_repo_schema(base_name = "resourceDiscovery", folder_path = "metadata/Microsoft.DSC")]
66+
pub enum ResourceDiscoveryMode {
67+
PreDeployment,
68+
DuringDeployment,
69+
}
70+
6371
#[derive(Debug, Default, Clone, PartialEq, Deserialize, Serialize, JsonSchema)]
72+
#[serde(rename_all = "camelCase")]
6473
pub struct MicrosoftDscMetadata {
6574
/// The duration of the configuration operation
6675
#[serde(skip_serializing_if = "Option::is_none")]
6776
pub duration: Option<String>,
6877
/// The end time of the configuration operation
69-
#[serde(rename = "endDatetime", skip_serializing_if = "Option::is_none")]
78+
#[serde(skip_serializing_if = "Option::is_none")]
7079
pub end_datetime: Option<String>,
7180
/// The type of execution
72-
#[serde(rename = "executionType", skip_serializing_if = "Option::is_none")]
81+
#[serde(skip_serializing_if = "Option::is_none")]
7382
pub execution_type: Option<ExecutionKind>,
7483
/// The operation being performed
7584
#[serde(skip_serializing_if = "Option::is_none")]
7685
pub operation: Option<Operation>,
7786
/// Specify specific adapter type used for implicit operations
78-
#[serde(rename = "requireAdapter", skip_serializing_if = "Option::is_none")]
87+
#[serde(skip_serializing_if = "Option::is_none")]
7988
pub require_adapter: Option<String>,
89+
/// Indicates if resources are discovered pre-deployment or during deployment
90+
#[serde(skip_serializing_if = "Option::is_none")]
91+
pub resource_discovery: Option<ResourceDiscoveryMode>,
8092
/// Indicates what needs to be restarted after the configuration operation
81-
#[serde(rename = "restartRequired", skip_serializing_if = "Option::is_none")]
93+
#[serde(skip_serializing_if = "Option::is_none")]
8294
pub restart_required: Option<Vec<RestartRequired>>,
8395
/// Copy loop context for resources expanded from copy loops
8496
#[serde(rename = "copyLoops", skip_serializing_if = "Option::is_none")]
8597
pub copy_loops: Option<Map<String, Value>>,
8698
/// The security context of the configuration operation, can be specified to be required
87-
#[serde(rename = "securityContext", skip_serializing_if = "Option::is_none")]
99+
#[serde(skip_serializing_if = "Option::is_none")]
88100
pub security_context: Option<SecurityContextKind>,
89101
/// The start time of the configuration operation
90-
#[serde(rename = "startDatetime", skip_serializing_if = "Option::is_none")]
102+
#[serde(skip_serializing_if = "Option::is_none")]
91103
pub start_datetime: Option<String>,
92104
/// Version of DSC
93105
#[serde(skip_serializing_if = "Option::is_none")]

‎lib/dsc-lib/src/configure/mod.rs‎

Lines changed: 42 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
use crate::configure::context::{Context, ProcessMode};
55
use crate::configure::parameters::import_parameters;
6-
use crate::configure::{config_doc::{ExecutionKind, IntOrExpression, Metadata, Parameter, Resource, RestartRequired, ValueOrCopy}};
6+
use crate::configure::{config_doc::{ExecutionKind, IntOrExpression, Metadata, Parameter, Resource, ResourceDiscoveryMode, RestartRequired, ValueOrCopy}};
77
use crate::discovery::discovery_trait::DiscoveryFilter;
88
use crate::dscerror::DscError;
99
use crate::dscresources::{
@@ -998,6 +998,7 @@ impl Configurator {
998998
end_datetime: Some(end_datetime.to_rfc3339()),
999999
execution_type: Some(self.context.execution_type.clone()),
10001000
operation: Some(operation),
1001+
resource_discovery: None,
10011002
restart_required: self.context.restart_required.clone(),
10021003
security_context: Some(self.context.security_context.clone()),
10031004
start_datetime: Some(self.context.start_datetime.to_rfc3339()),
@@ -1013,29 +1014,52 @@ impl Configurator {
10131014
let config: Configuration = serde_json::from_str(self.json.as_str())?;
10141015
check_security_context(config.metadata.as_ref())?;
10151016

1016-
// Perform discovery of resources used in config
1017-
// create an array of DiscoveryFilter using the resource types and api_versions from the config
1018-
let mut discovery_filter: Vec<DiscoveryFilter> = Vec::new();
1019-
let config_copy = config.clone();
1020-
for resource in config_copy.resources {
1021-
let adapter = get_require_adapter_from_metadata(&resource.metadata);
1022-
let filter = DiscoveryFilter::new(&resource.resource_type, resource.api_version.as_deref(), adapter.as_deref());
1023-
if !discovery_filter.contains(&filter) {
1024-
discovery_filter.push(filter);
1017+
let mut skip_resource_validation = false;
1018+
if let Some(metadata) = &config.metadata {
1019+
if let Some(microsoft_metadata) = &metadata.microsoft {
1020+
if let Some(mode) = &microsoft_metadata.resource_discovery {
1021+
if *mode == ResourceDiscoveryMode::DuringDeployment {
1022+
debug!("{}", t!("configure.mod.skippingResourceDiscovery"));
1023+
skip_resource_validation = true;
1024+
self.discovery.refresh_cache = true;
1025+
}
1026+
}
10251027
}
1026-
// defer actual unrolling until parameters are available
1027-
if let Some(copy) = &resource.copy {
1028-
debug!("{}", t!("configure.mod.validateCopy", name = &copy.name, count = copy.count));
1029-
if copy.mode.is_some() {
1030-
return Err(DscError::Validation(t!("configure.mod.copyModeNotSupported").to_string()));
1028+
}
1029+
1030+
if !skip_resource_validation {
1031+
// Perform discovery of resources used in config
1032+
// create an array of DiscoveryFilter using the resource types and api_versions from the config
1033+
let mut discovery_filter: Vec<DiscoveryFilter> = Vec::new();
1034+
let config_copy = config.clone();
1035+
for resource in config_copy.resources {
1036+
let adapter = get_require_adapter_from_metadata(&resource.metadata);
1037+
let filter = DiscoveryFilter::new(&resource.resource_type, resource.api_version.as_deref(), adapter.as_deref());
1038+
if !discovery_filter.contains(&filter) {
1039+
discovery_filter.push(filter);
10311040
}
1032-
if copy.batch_size.is_some() {
1033-
return Err(DscError::Validation(t!("configure.mod.copyBatchSizeNotSupported").to_string()));
1041+
// defer actual unrolling until parameters are available
1042+
if let Some(copy) = &resource.copy {
1043+
debug!("{}", t!("configure.mod.validateCopy", name = &copy.name, count = copy.count));
1044+
if copy.mode.is_some() {
1045+
return Err(DscError::Validation(t!("configure.mod.copyModeNotSupported").to_string()));
1046+
}
1047+
if copy.batch_size.is_some() {
1048+
return Err(DscError::Validation(t!("configure.mod.copyBatchSizeNotSupported").to_string()));
1049+
}
10341050
}
10351051
}
1052+
self.discovery.find_resources(&discovery_filter, self.progress_format)?;
1053+
1054+
// now check that each resource in the config was found
1055+
for resource in config.resources.iter() {
1056+
let adapter = get_require_adapter_from_metadata(&resource.metadata);
1057+
let Some(_dsc_resource) = self.discovery.find_resource(&DiscoveryFilter::new(&resource.resource_type, resource.api_version.as_deref(), adapter.as_deref()))? else {
1058+
return Err(DscError::ResourceNotFound(resource.resource_type.to_string(), resource.api_version.as_deref().unwrap_or("").to_string()));
1059+
};
1060+
}
10361061
}
10371062

1038-
self.discovery.find_resources(&discovery_filter, self.progress_format)?;
10391063
self.config = config;
10401064
Ok(())
10411065
}

0 commit comments

Comments
 (0)