Skip to content

Commit 5a382d8

Browse files
gh-64595: Touch the source file if the generated file was changed
The build system does not know that the source file depends on the files generated from it, so it did not recompile the source file if only the generated file was changed. The generated files are also kept newer than the source file.
1 parent e12ee02 commit 5a382d8

4 files changed

Lines changed: 106 additions & 9 deletions

File tree

Lib/test/test_clinic.py

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3076,6 +3076,64 @@ def test_no_change(self):
30763076
# if the content does not change
30773077
self.assertEqual(pre_mtime, post_mtime)
30783078

3079+
TOUCH_CODE = dedent("""
3080+
/*[clinic input]
3081+
module m
3082+
[clinic start generated code]*/
3083+
3084+
/*[clinic input]
3085+
output everything file
3086+
m.func
3087+
a: int
3088+
/
3089+
3090+
Docstring.
3091+
[clinic start generated code]*/
3092+
""")
3093+
3094+
def test_touch_source(self):
3095+
# gh-64595: The build system does not know that the source file
3096+
# depends on the file generated from it, so the modification
3097+
# times are updated to force the recompilation.
3098+
def mtimes():
3099+
return os.stat(fn).st_mtime_ns, os.stat(dest).st_mtime_ns
3100+
3101+
def set_mtimes(source, generated):
3102+
os.utime(fn, ns=(source, source))
3103+
os.utime(dest, ns=(generated, generated))
3104+
3105+
with os_helper.temp_dir() as tmp_dir:
3106+
fn = os.path.join(tmp_dir, "test.c")
3107+
with open(fn, "w", encoding="utf-8") as f:
3108+
f.write(self.TOUCH_CODE)
3109+
dest = self.dest_file(fn)
3110+
self.expect_success(fn)
3111+
source_mtime, generated_mtime = mtimes()
3112+
self.assertGreaterEqual(generated_mtime, source_mtime)
3113+
3114+
# The generated file is changed, so both files are touched.
3115+
os.unlink(dest)
3116+
old = source_mtime - 10**10
3117+
os.utime(fn, ns=(old, old))
3118+
self.expect_success(fn)
3119+
source_mtime, generated_mtime = mtimes()
3120+
self.assertGreater(source_mtime, old)
3121+
self.assertGreaterEqual(generated_mtime, source_mtime)
3122+
3123+
# Nothing is changed, but the source file is newer, so only
3124+
# the generated file is touched.
3125+
set_mtimes(source_mtime - 10**10, source_mtime - 2 * 10**10)
3126+
old_source_mtime = os.stat(fn).st_mtime_ns
3127+
self.expect_success(fn)
3128+
source_mtime, generated_mtime = mtimes()
3129+
self.assertEqual(source_mtime, old_source_mtime)
3130+
self.assertGreaterEqual(generated_mtime, source_mtime)
3131+
3132+
# Nothing is changed and the generated file is newer,
3133+
# so no file is touched.
3134+
self.expect_success(fn)
3135+
self.assertEqual(mtimes(), (source_mtime, generated_mtime))
3136+
30793137
def test_cli_force(self):
30803138
invalid_input = dedent("""
30813139
/*[clinic input]
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Argument Clinic now touches the source file if a file generated from it was
2+
changed, and keeps the generated files newer than the source file.
3+
The build system does not know that the source file depends on the files
4+
generated from it, so it did not recompile the source file if only the
5+
generated file was changed.

Tools/clinic/libclinic/cli.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,10 +89,15 @@ def parse_file(
8989
filename=filename,
9090
limited_capi=limited_capi,
9191
writer=writer)
92+
index = len(writer.files)
9293
cooked = clinic.parse(raw)
93-
9494
writer.write(output, cooked)
9595

96+
files = writer.files[index:]
97+
writer.update_times(output,
98+
[fn for fn, _ in files if fn != output],
99+
any(changed for _, changed in files))
100+
96101

97102
def create_cli() -> argparse.ArgumentParser:
98103
cmdline = argparse.ArgumentParser(

Tools/clinic/libclinic/utils.py

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import os
66
import re
77
import string
8+
from collections.abc import Iterable
89
from typing import Literal, Final
910

1011

@@ -17,11 +18,14 @@ def read_file(filename: str) -> str | None:
1718
return None
1819

1920

20-
def write_file(filename: str, new_contents: str) -> None:
21-
"""Write new content to file, iff the content changed."""
21+
def write_file(filename: str, new_contents: str) -> bool:
22+
"""Write new content to file, iff the content changed.
23+
24+
Return True if the file was written.
25+
"""
2226
if read_file(filename) == new_contents:
2327
# no change: avoid modifying the file modification time
24-
return
28+
return False
2529
# Atomic write using a temporary file and os.replace()
2630
filename_new = f"{filename}.new"
2731
with open(filename_new, "w", encoding="utf-8") as fp:
@@ -31,6 +35,7 @@ def write_file(filename: str, new_contents: str) -> None:
3135
except:
3236
os.unlink(filename_new)
3337
raise
38+
return True
3439

3540

3641
@dc.dataclass(slots=True, frozen=True)
@@ -50,6 +55,8 @@ class FileWriter:
5055

5156
dry_run: bool = False
5257
changes: list[FileChange] = dc.field(default_factory=list)
58+
# (filename, changed) for every file which was passed to write().
59+
files: list[tuple[str, bool]] = dc.field(default_factory=list)
5360

5461
def makedirs(self, dirname: str) -> None:
5562
if not self.dry_run:
@@ -61,12 +68,34 @@ def makedirs(self, dirname: str) -> None:
6168

6269
def write(self, filename: str, new_contents: str) -> None:
6370
if not self.dry_run:
64-
write_file(filename, new_contents)
71+
changed = write_file(filename, new_contents)
72+
else:
73+
old_contents = read_file(filename)
74+
changed = old_contents != new_contents
75+
if changed:
76+
self.changes.append(
77+
FileChange(filename, old_contents, new_contents))
78+
self.files.append((filename, changed))
79+
80+
def update_times(self, source: str, generated: Iterable[str],
81+
changed: bool) -> None:
82+
"""Keep the generated files newer than the source file.
83+
84+
The build system does not know that the source file depends on
85+
the files generated from it, so the source file is touched to
86+
force its recompilation.
87+
"""
88+
if self.dry_run:
6589
return
66-
old_contents = read_file(filename)
67-
if old_contents != new_contents:
68-
self.changes.append(
69-
FileChange(filename, old_contents, new_contents))
90+
if changed:
91+
os.utime(source)
92+
for filename in generated:
93+
os.utime(filename)
94+
else:
95+
mtime = os.stat(source).st_mtime_ns
96+
for filename in generated:
97+
if os.stat(filename).st_mtime_ns <= mtime:
98+
os.utime(filename)
7099

71100

72101
def compute_checksum(input_: str, length: int | None = None) -> str:

0 commit comments

Comments
 (0)