Skip to content

Commit ce8ea36

Browse files
dssengdsseng
committed
WIP: fs_scrub controller
Signed-off-by: Dmitry Sharshakov <dmitry.sharshakov@siderolabs.com>
1 parent 3904ef3 commit ce8ea36

3 files changed

Lines changed: 215 additions & 0 deletions

File tree

‎internal/app/machined/pkg/controllers/runtime/fs_scrub.go‎

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
// This Source Code Form is subject to the terms of the Mozilla Public
2+
// License, v. 2.0. If a copy of the MPL was not distributed with this
3+
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
4+
5+
package runtime
6+
7+
import (
8+
"context"
9+
"fmt"
10+
"math/rand/v2"
11+
"time"
12+
13+
"github.com/cosi-project/runtime/pkg/controller"
14+
"github.com/cosi-project/runtime/pkg/safe"
15+
"github.com/cosi-project/runtime/pkg/state"
16+
"go.uber.org/zap"
17+
18+
"github.com/siderolabs/talos/internal/app/machined/pkg/runtime"
19+
"github.com/siderolabs/talos/internal/app/machined/pkg/system/events"
20+
"github.com/siderolabs/talos/internal/app/machined/pkg/system/runner"
21+
"github.com/siderolabs/talos/internal/app/machined/pkg/system/runner/process"
22+
"github.com/siderolabs/talos/internal/pkg/environment"
23+
"github.com/siderolabs/talos/pkg/machinery/constants"
24+
"github.com/siderolabs/talos/pkg/machinery/resources/block"
25+
)
26+
27+
type scrubSchedule struct {
28+
period time.Duration
29+
upcoming time.Time
30+
}
31+
32+
// FSScrubController watches v1alpha1.Config and schedules filesystem online check tasks.
33+
type FSScrubController struct {
34+
Runtime runtime.Runtime
35+
schedule map[string]scrubSchedule
36+
}
37+
38+
// Name implements controller.Controller interface.
39+
func (ctrl *FSScrubController) Name() string {
40+
return "runtime.FSScrubController"
41+
}
42+
43+
// Inputs implements controller.Controller interface.
44+
func (ctrl *FSScrubController) Inputs() []controller.Input {
45+
return []controller.Input{
46+
{
47+
Namespace: block.NamespaceName,
48+
Type: block.VolumeStatusType,
49+
Kind: controller.InputWeak,
50+
},
51+
{
52+
Namespace: block.NamespaceName,
53+
Type: block.VolumeConfigType,
54+
Kind: controller.InputWeak,
55+
},
56+
}
57+
}
58+
59+
// Outputs implements controller.Controller interface.
60+
func (ctrl *FSScrubController) Outputs() []controller.Output {
61+
return []controller.Output{}
62+
}
63+
64+
// Run implements controller.Controller interface.
65+
func (ctrl *FSScrubController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error {
66+
var (
67+
ticker *time.Ticker
68+
tickerC <-chan time.Time
69+
)
70+
71+
tickerStop := func() {
72+
if ticker == nil {
73+
return
74+
}
75+
76+
ticker.Stop()
77+
78+
ticker = nil
79+
tickerC = nil
80+
}
81+
82+
defer tickerStop()
83+
84+
tickerStop()
85+
86+
ticker = time.NewTicker(15 * time.Second)
87+
tickerC = ticker.C
88+
89+
ctrl.schedule = make(map[string]scrubSchedule)
90+
91+
for {
92+
select {
93+
case <-ctx.Done():
94+
return nil
95+
case <-tickerC:
96+
if err := ctrl.scrub("/var", []string{}); err != nil {
97+
return fmt.Errorf("error running filesystem scrub: %w", err)
98+
}
99+
100+
continue
101+
case <-r.EventCh():
102+
err := ctrl.updateSchedule(ctx, r, logger)
103+
if err != nil {
104+
return err
105+
}
106+
}
107+
}
108+
}
109+
110+
func (ctrl *FSScrubController) updateSchedule(ctx context.Context, r controller.Runtime, logger *zap.Logger) error {
111+
volumesStatus, err := safe.ReaderListAll[*block.VolumeStatus](ctx, r)
112+
if err != nil && !state.IsNotFoundError(err) {
113+
return fmt.Errorf("error getting volume status: %w", err)
114+
}
115+
116+
logger.Warn("reading volume status")
117+
volumesStatus.ForEach(func(item *block.VolumeStatus) {
118+
vol := item.TypedSpec()
119+
120+
logger.Warn("volume status", zap.Reflect("item", vol))
121+
122+
if vol.Phase != block.VolumePhaseReady {
123+
logger.Warn("vol.Phase != block.VolumePhaseReady", zap.Reflect("item", vol))
124+
125+
return
126+
}
127+
128+
if vol.Filesystem != block.FilesystemTypeXFS {
129+
logger.Warn("vol.Filesystem != block.FilesystemTypeXFS", zap.Reflect("item", vol))
130+
131+
return
132+
}
133+
134+
volumeConfig, err := safe.ReaderGetByID[*block.VolumeConfig](ctx, r, item.Metadata().ID())
135+
if err != nil {
136+
logger.Warn("err", zap.Error(err))
137+
138+
return
139+
}
140+
141+
mountpoint := volumeConfig.TypedSpec().Mount.TargetPath
142+
143+
if _, ok := ctrl.schedule[mountpoint]; !ok {
144+
per := 10 * time.Second
145+
seconds := rand.Int64N(int64(per.Seconds()))
146+
147+
ctrl.schedule[mountpoint] = scrubSchedule{
148+
period: per,
149+
upcoming: time.Now().Add(time.Duration(seconds * int64(time.Second))),
150+
}
151+
152+
logger.Warn("scheduled", zap.String("path", mountpoint), zap.Reflect("upcoming", ctrl.schedule[mountpoint].upcoming))
153+
}
154+
})
155+
156+
return nil
157+
}
158+
159+
func (ctrl *FSScrubController) scrub(mountpoint string, opts []string) error {
160+
args := []string{"/usr/sbin/xfs_scrub", "-T", "-v"}
161+
args = append(args, opts...)
162+
args = append(args, mountpoint)
163+
164+
r := process.NewRunner(
165+
false,
166+
&runner.Args{
167+
ID: "fs_scrub",
168+
ProcessArgs: args,
169+
},
170+
runner.WithLoggingManager(ctrl.Runtime.Logging()),
171+
runner.WithEnv(environment.Get(ctrl.Runtime.Config())),
172+
runner.WithOOMScoreAdj(-999),
173+
runner.WithDroppedCapabilities(constants.XFSScrubDroppedCapabilities),
174+
runner.WithPriority(19),
175+
runner.WithIOPriority(runner.IoprioClassIdle, 7),
176+
runner.WithSchedulingPolicy(runner.SchedulingPolicyIdle),
177+
)
178+
179+
return r.Run(func(s events.ServiceState, msg string, args ...any) {})
180+
}

‎internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -339,6 +339,9 @@ func (ctrl *Controller) Run(ctx context.Context, drainer *runtime.Drainer) error
339339
runtimecontrollers.NewUniqueMachineTokenController(),
340340
&runtimecontrollers.WatchdogTimerConfigController{},
341341
&runtimecontrollers.WatchdogTimerController{},
342+
&runtimecontrollers.FSScrubController{
343+
Runtime: ctrl.v1alpha1Runtime,
344+
},
342345
&secrets.APICertSANsController{},
343346
&secrets.APIController{},
344347
&secrets.EtcdController{},

‎pkg/machinery/constants/constants.go‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1273,6 +1273,38 @@ var DefaultDroppedCapabilities = map[string]struct{}{
12731273
"cap_sys_module": {},
12741274
}
12751275

1276+
// XFSScrubDroppedCapabilities is the set of capabilities to drop for xfs_scrub.
1277+
// All but cap_sys_admin cap_fowner cap_dac_override cap_dac_read_search cap_sys_rawio
1278+
var XFSScrubDroppedCapabilities = map[string]struct{}{
1279+
"cap_audit_control": {},
1280+
"cap_audit_write": {},
1281+
"cap_chown": {},
1282+
"cap_fsetid": {},
1283+
"cap_ipc_lock": {},
1284+
"cap_ipc_owner": {},
1285+
"cap_kill": {},
1286+
"cap_lease": {},
1287+
"cap_linux_immutable": {},
1288+
"cap_mknod": {},
1289+
"cap_net_admin": {},
1290+
"cap_net_bind_service": {},
1291+
"cap_net_broadcast": {},
1292+
"cap_net_raw": {},
1293+
"cap_setfcap": {},
1294+
"cap_setgid": {},
1295+
"cap_setpcap": {},
1296+
"cap_setuid": {},
1297+
"cap_sys_boot": {},
1298+
"cap_sys_chroot": {},
1299+
"cap_sys_module": {},
1300+
"cap_sys_nice": {},
1301+
"cap_sys_pacct": {},
1302+
"cap_sys_ptrace": {},
1303+
"cap_sys_resource": {},
1304+
"cap_sys_time": {},
1305+
"cap_sys_tty_config": {},
1306+
}
1307+
12761308
// UdevdDroppedCapabilities is the set of capabilities to drop for udevd.
12771309
var UdevdDroppedCapabilities = map[string]struct{}{
12781310
"cap_sys_boot": {},

0 commit comments

Comments
 (0)