Skip to content

Commit f8c854d

Browse files
polac24polac24
authored
Merge pull request #214 from polac24/up-to-date-meta
Ensure up-to-date meta json in the unzipped artifact
2 parents 7fe0451 + 587840d commit f8c854d

7 files changed

Lines changed: 253 additions & 11 deletions

File tree

‎Sources/XCRemoteCache/Artifacts/ArtifactMetaUpdater.swift‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
// Copyright (c) 2023 Spotify AB.
2+
//
3+
// Licensed to the Apache Software Foundation (ASF) under one
4+
// or more contributor license agreements. See the NOTICE file
5+
// distributed with this work for additional information
6+
// regarding copyright ownership. The ASF licenses this file
7+
// to you under the Apache License, Version 2.0 (the
8+
// "License"); you may not use this file except in compliance
9+
// with the License. You may obtain a copy of the License at
10+
//
11+
// http://www.apache.org/licenses/LICENSE-2.0
12+
//
13+
// Unless required by applicable law or agreed to in writing,
14+
// software distributed under the License is distributed on an
15+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
// KIND, either express or implied. See the License for the
17+
// specific language governing permissions and limitations
18+
// under the License.
19+
20+
import Foundation
21+
22+
enum ArtifactMetaUpdaterError: Error {
23+
/// The prebuild plugin execution was called but the local
24+
/// path to the artifact directory is still unknown
25+
/// Might happen that the artifact processor didn't invoke the updater's
26+
/// .process() after downloading/activating an artifact
27+
case artifactLocationIsUnknown
28+
}
29+
30+
/// Updates the meta file in an unzipped artifact directory, by placing an up-to-date
31+
/// and remapped meta file. Updating the meta in the artifact allows reusing existing
32+
/// artifacts it a new meta.json schema has been released to the meta format, while
33+
/// artifacts are still backward-compatible
34+
class ArtifactMetaUpdater: ArtifactProcessor {
35+
private var artifactLocation: URL?
36+
private let metaWriter: MetaWriter
37+
private let fileRemapper: FileDependenciesRemapper
38+
39+
init(
40+
fileRemapper: FileDependenciesRemapper,
41+
metaWriter: MetaWriter
42+
) {
43+
self.metaWriter = metaWriter
44+
self.fileRemapper = fileRemapper
45+
}
46+
47+
/// Remembers the artifact location, used later in the plugin
48+
/// - Parameter url: artifact's root directory
49+
func process(rawArtifact url: URL) throws {
50+
// Storing the location of the just downloaded/activated artifact
51+
// Note, the `url` location already includes a meta (generated by producer
52+
// while compiling and building an artifact)
53+
artifactLocation = url
54+
}
55+
56+
func process(localArtifact url: URL) throws {
57+
// No need to do anything in the postbuild
58+
}
59+
}
60+
61+
extension ArtifactMetaUpdater: ArtifactConsumerPrebuildPlugin {
62+
63+
/// Updates the meta json file in a local, unzipped, artifact location. It also remaps
64+
/// all paths so other steps (like actool or postbuild) don't have to do it again
65+
func run(meta: MainArtifactMeta) throws {
66+
guard let artifactLocation = artifactLocation else {
67+
throw ArtifactMetaUpdaterError.artifactLocationIsUnknown
68+
}
69+
let metaURL = try metaWriter.write(meta, locationDir: artifactLocation)
70+
try fileRemapper.remap(fromGeneric: metaURL)
71+
}
72+
}

‎Sources/XCRemoteCache/Artifacts/ArtifactOrganizer.swift‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ protocol ArtifactOrganizer {
4747
}
4848

4949
class ZipArtifactOrganizer: ArtifactOrganizer {
50+
static let activeArtifactLocation = "active"
51+
5052
private let cacheDir: URL
5153
// all processors that should "prepare" the unzipped raw artifact
5254
private let artifactProcessors: [ArtifactProcessor]
@@ -63,7 +65,7 @@ class ZipArtifactOrganizer: ArtifactOrganizer {
6365
}
6466

6567
func getActiveArtifactLocation() -> URL {
66-
return cacheDir.appendingPathComponent("active")
68+
return cacheDir.appendingPathComponent(Self.self.activeArtifactLocation)
6769
}
6870

6971
func getActiveArtifactFilekey() throws -> String {
@@ -90,20 +92,27 @@ class ZipArtifactOrganizer: ArtifactOrganizer {
9092
let destinationURL = artifact.deletingPathExtension()
9193
guard fileManager.fileExists(atPath: destinationURL.path) == false else {
9294
infoLog("Skipping artifact, already existing at \(destinationURL)")
95+
try runArtifactProcessors(artifactLocation: destinationURL)
9396
return destinationURL
9497
}
95-
// Uzipping to a temp file first to never leave the uncompleted zip in the final location
98+
// Unzipping to a temp file first to never leave the uncompleted zip in the final location
9699
// when the command was interrupted (internal crash or `kill -9` signal)
97100
let tempDestination = destinationURL.appendingPathExtension("tmp")
98101
try Zip.unzipFile(artifact, destination: tempDestination, overwrite: true, password: nil)
99102

100-
try artifactProcessors.forEach { processor in
101-
try processor.process(rawArtifact: tempDestination)
102-
}
103103
try fileManager.moveItem(at: tempDestination, to: destinationURL)
104+
try runArtifactProcessors(artifactLocation: destinationURL)
104105
return destinationURL
105106
}
106107

108+
/// Iterates all processor when an artifact has been just downloaded or reused from already downloaded
109+
/// and previously processed location
110+
private func runArtifactProcessors(artifactLocation: URL) throws {
111+
try artifactProcessors.forEach { processor in
112+
try processor.process(rawArtifact: artifactLocation)
113+
}
114+
}
115+
107116
func activate(extractedArtifact: URL) throws {
108117
let activeLocationURL = getActiveArtifactLocation()
109118
try fileManager.spt_forceSymbolicLink(at: activeLocationURL, withDestinationURL: extractedArtifact)

‎Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -158,13 +158,17 @@ public class XCPrebuild {
158158
fileAccessor: fileManager
159159
)
160160
let artifactProcessor = UnzippedArtifactProcessor(fileRemapper: fileRemapper, dirScanner: fileManager)
161+
let metaUpdater = ArtifactMetaUpdater(
162+
fileRemapper: fileRemapper,
163+
metaWriter: JsonMetaWriter(fileWriter: fileManager, pretty: true)
164+
)
161165
let organizer = ZipArtifactOrganizer(
162166
targetTempDir: context.targetTempDir,
163-
artifactProcessors: [artifactProcessor],
167+
artifactProcessors: [artifactProcessor, metaUpdater],
164168
fileManager: fileManager
165169
)
166170
let metaReader = JsonMetaReader(fileAccessor: fileManager)
167-
var consumerPlugins: [ArtifactConsumerPrebuildPlugin] = []
171+
var consumerPlugins: [ArtifactConsumerPrebuildPlugin] = [metaUpdater]
168172

169173
if config.thinningEnabled {
170174
if context.moduleName == config.thinningTargetModuleName, let thinnedTarget = context.thinnedTargets {

‎Sources/XCRemoteCache/Dependencies/MarkerReader.swift‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,12 @@ import Foundation
2222
/// Reads a list of files from a marker file
2323
class FileMarkerReader: ListReader {
2424
private let file: URL
25-
private let fileManager: FileManager
25+
private let fileReader: FileReader
2626
private var cachedFiles: [URL]?
2727

28-
init(_ file: URL, fileManager: FileManager) {
28+
init(_ file: URL, fileManager: FileReader) {
2929
self.file = file
30-
self.fileManager = fileManager
30+
self.fileReader = fileManager
3131
}
3232

3333
func listFilesURLs() throws -> [URL] {
@@ -45,6 +45,6 @@ class FileMarkerReader: ListReader {
4545
}
4646

4747
func canRead() -> Bool {
48-
return fileManager.fileExists(atPath: file.path)
48+
return fileReader.fileExists(atPath: file.path)
4949
}
5050
}

‎Tests/XCRemoteCacheTests/Artifacts/ArtifactMetaUpdaterTests.swift‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
// Copyright (c) 2023 Spotify AB.
2+
//
3+
// Licensed to the Apache Software Foundation (ASF) under one
4+
// or more contributor license agreements. See the NOTICE file
5+
// distributed with this work for additional information
6+
// regarding copyright ownership. The ASF licenses this file
7+
// to you under the Apache License, Version 2.0 (the
8+
// "License"); you may not use this file except in compliance
9+
// with the License. You may obtain a copy of the License at
10+
//
11+
// http://www.apache.org/licenses/LICENSE-2.0
12+
//
13+
// Unless required by applicable law or agreed to in writing,
14+
// software distributed under the License is distributed on an
15+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
// KIND, either express or implied. See the License for the
17+
// specific language governing permissions and limitations
18+
// under the License.
19+
20+
@testable import XCRemoteCache
21+
import XCTest
22+
23+
class ArtifactMetaUpdaterTests: XCTestCase {
24+
private let accessorFake = FileAccessorFake(mode: .normal)
25+
private var metaWriter: MetaWriter!
26+
private var fileRemapper: FileDependenciesRemapper!
27+
private var updater: ArtifactMetaUpdater!
28+
private let sampleMeta = MainArtifactMeta(
29+
dependencies: [],
30+
fileKey: "abc",
31+
rawFingerprint: "",
32+
generationCommit: "",
33+
targetName: "",
34+
configuration: "",
35+
platform: "",
36+
xcode: "",
37+
inputs: ["$(BASE)/myFile.swift"],
38+
pluginsKeys: [:]
39+
)
40+
41+
override func setUp() async throws {
42+
metaWriter = JsonMetaWriter(
43+
fileWriter: accessorFake,
44+
pretty: true
45+
)
46+
fileRemapper = TextFileDependenciesRemapper(
47+
remapper: StringDependenciesRemapper(
48+
mappings: [
49+
.init(generic: "$(BASE)", local: "/base")
50+
]
51+
),
52+
fileAccessor: accessorFake
53+
)
54+
updater = ArtifactMetaUpdater(
55+
fileRemapper: fileRemapper,
56+
metaWriter: metaWriter
57+
)
58+
}
59+
60+
func testStoresInTheRawArtifact() throws {
61+
try updater.process(rawArtifact: "/artifact")
62+
try updater.run(meta: sampleMeta)
63+
64+
XCTAssertTrue(accessorFake.fileExists(atPath: "/artifact/abc.json"))
65+
}
66+
67+
func testRewirtesMetaPaths() throws {
68+
try updater.process(rawArtifact: "/artifact")
69+
try updater.run(meta: sampleMeta)
70+
71+
let diskMetaData = try XCTUnwrap(accessorFake.contents(atPath: "/artifact/abc.json"))
72+
let diskMeta = try JSONDecoder().decode(MainArtifactMeta.self, from: diskMetaData)
73+
XCTAssertEqual(diskMeta.inputs, ["/base/myFile.swift"])
74+
}
75+
76+
func testFailsIfProcessorTriggerIsNotCalledBeforeRunningAPlugin() throws {
77+
XCTAssertThrowsError(try updater.run(meta: sampleMeta)) { error in
78+
switch error {
79+
case ArtifactMetaUpdaterError.artifactLocationIsUnknown: break
80+
default:
81+
XCTFail("Not expected error")
82+
}
83+
}
84+
}
85+
}

‎Tests/XCRemoteCacheTests/Artifacts/ZipArtifactOrganizerTests.swift‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,4 +156,39 @@ class ZipArtifactOrganizerTests: XCTestCase {
156156

157157
XCTAssertEqual(fileKey, expectedFileKey)
158158
}
159+
160+
func testPrepareRunsProcessorsForAlreadyExistingArtifacts() throws {
161+
let zipURL = try prepareZipFile(content: "Magic", fileName: "content.txt")
162+
let artifactURL = zipURL.deletingPathExtension()
163+
let processor = DestroyerArtifactProcessor(fileManager)
164+
let organizer: ZipArtifactOrganizer = ZipArtifactOrganizer(
165+
targetTempDir: workingDirectory,
166+
artifactProcessors: [processor],
167+
fileManager: fileManager
168+
)
169+
try fileManager.createDirectory(
170+
at: artifactURL,
171+
withIntermediateDirectories: true
172+
)
173+
174+
let preparedArtifact = try organizer.prepare(artifact: zipURL)
175+
176+
XCTAssertFalse(fileManager.fileExists(atPath: preparedArtifact.path))
177+
178+
}
179+
180+
func testPrepareRunsProcessorsForNewlyUnzippedArtifacts() throws {
181+
let zipURL = try prepareZipFile(content: "Magic", fileName: "content.txt")
182+
let processor = DestroyerArtifactProcessor(fileManager)
183+
let organizer: ZipArtifactOrganizer = ZipArtifactOrganizer(
184+
targetTempDir: workingDirectory,
185+
artifactProcessors: [processor],
186+
fileManager: fileManager
187+
)
188+
189+
let preparedArtifact = try organizer.prepare(artifact: zipURL)
190+
191+
XCTAssertFalse(fileManager.fileExists(atPath: preparedArtifact.path))
192+
}
193+
159194
}

‎Tests/XCRemoteCacheTests/TestDoubles/DestroyerArtifactProcessor.swift‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
// Copyright (c) 2022 Spotify AB.
2+
//
3+
// Licensed to the Apache Software Foundation (ASF) under one
4+
// or more contributor license agreements. See the NOTICE file
5+
// distributed with this work for additional information
6+
// regarding copyright ownership. The ASF licenses this file
7+
// to you under the Apache License, Version 2.0 (the
8+
// "License"); you may not use this file except in compliance
9+
// with the License. You may obtain a copy of the License at
10+
//
11+
// http://www.apache.org/licenses/LICENSE-2.0
12+
//
13+
// Unless required by applicable law or agreed to in writing,
14+
// software distributed under the License is distributed on an
15+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
// KIND, either express or implied. See the License for the
17+
// specific language governing permissions and limitations
18+
// under the License.
19+
20+
21+
import Foundation
22+
@testable import XCRemoteCache
23+
24+
/// A Processor fake that deletes the artifact
25+
class DestroyerArtifactProcessor: ArtifactProcessor {
26+
private let dirAccesor: DirAccessor
27+
28+
init(_ dirAccesor: DirAccessor) {
29+
self.dirAccesor = dirAccesor
30+
}
31+
func process(rawArtifact url: URL) throws {
32+
try dirAccesor.removeItem(atPath: url.path)
33+
}
34+
func process(localArtifact url: URL) throws {
35+
try dirAccesor.removeItem(atPath: url.path)
36+
}
37+
}

0 commit comments

Comments
 (0)