Skip to content

Commit 901e260

Browse files
authored
Merge pull request #193 from lets-cli/add-remote-mixins
add remote mixins (experimental)
2 parents 8c6369a + 0d09f74 commit 901e260

16 files changed

Lines changed: 286 additions & 73 deletions

File tree

.github/workflows/test.yaml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,11 @@ jobs:
2525
go-version: 1.18.x
2626
- name: Checkout code
2727
uses: actions/checkout@v2
28+
- run: go install gotest.tools/gotestsum@latest
2829
- name: Test unit
2930
env:
3031
LETS_CONFIG_DIR: ..
31-
run: go test ./... -v
32+
run: gotestsum --format testname -- ./... -coverprofile=coverage.out
3233

3334
test-bats:
3435
runs-on: ubuntu-latest

docker/Dockerfile renamed to Dockerfile

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM golang:1.18.3-bullseye
1+
FROM golang:1.18.3-bullseye as builder
22

33
ENV GOPROXY https://proxy.golang.org
44
WORKDIR /app
@@ -21,3 +21,7 @@ COPY go.mod .
2121
COPY go.sum .
2222

2323
RUN go mod download
24+
25+
FROM golangci/golangci-lint:v1.45-alpine as linter
26+
27+
RUN mkdir -p /.cache && chmod -R 777 /.cache

checksum/checksum.go

Lines changed: 12 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ import (
1515
const (
1616
DefaultChecksumKey = "__default_checksum__"
1717
DefaultChecksumFileName = "lets_default_checksum"
18-
checksumsDir = "checksums"
1918
)
2019

2120
var checksumCache = make(map[string][]byte)
@@ -144,8 +143,8 @@ func CalculateChecksumFromSources(workDir string, checksumSources map[string][]s
144143
return checksumMap, nil
145144
}
146145

147-
func ReadChecksumFromDisk(dotLetsDir, cmdName, checksumName string) (string, error) {
148-
_, checksumFilePath := getChecksumPath(dotLetsDir, cmdName, checksumName)
146+
func ReadChecksumFromDisk(checksumsDir, cmdName, checksumName string) (string, error) {
147+
_, checksumFilePath := getChecksumPath(checksumsDir, cmdName, checksumName)
149148

150149
fileData, err := os.ReadFile(checksumFilePath)
151150
if err != nil {
@@ -155,32 +154,27 @@ func ReadChecksumFromDisk(dotLetsDir, cmdName, checksumName string) (string, err
155154
return string(fileData), nil
156155
}
157156

158-
func getCmdChecksumPath(dotLetsDir string, cmdName string) string {
159-
return filepath.Join(dotLetsDir, checksumsDir, cmdName)
157+
func getCmdChecksumPath(checksumsDir string, cmdName string) string {
158+
return filepath.Join(checksumsDir, cmdName)
160159
}
161160

162161
// returns dir path and full file path to checksum
163162
// (.lets/checksums/[command_name]/, .lets/checksums/[command_name]/[checksum_name]).
164-
func getChecksumPath(dotLetsDir string, cmdName string, checksumName string) (string, string) {
165-
dirPath := getCmdChecksumPath(dotLetsDir, cmdName)
163+
func getChecksumPath(checksumsDir string, cmdName string, checksumName string) (string, string) {
164+
dirPath := getCmdChecksumPath(checksumsDir, cmdName)
166165

167166
return dirPath, filepath.Join(dirPath, checksumName)
168167
}
169168

170169
// TODO maybe checksumMap has to be separate struct ?
171-
func PersistCommandsChecksumToDisk(dotLetsDir string, checksumMap map[string]string, cmdName string) error {
172-
checksumPath := filepath.Join(dotLetsDir, checksumsDir)
173-
if err := util.SafeCreateDir(checksumPath); err != nil {
174-
return fmt.Errorf("can not create %s: %w", checksumPath, err)
175-
}
176-
170+
func PersistCommandsChecksumToDisk(checksumsDir string, checksumMap map[string]string, cmdName string) error {
177171
// TODO if at least one write failed do we have to revert all writes ???
178172
for checksumName, checksum := range checksumMap {
179173
filename := checksumName
180174
if checksumName == DefaultChecksumKey {
181175
filename = DefaultChecksumFileName
182176
}
183-
err := persistOneChecksum(dotLetsDir, cmdName, filename, checksum)
177+
err := persistOneChecksum(checksumsDir, cmdName, filename, checksum)
184178
if err != nil {
185179
return err
186180
}
@@ -189,8 +183,8 @@ func PersistCommandsChecksumToDisk(dotLetsDir string, checksumMap map[string]str
189183
return nil
190184
}
191185

192-
func persistOneChecksum(dotLetsDir string, cmdName string, checksumName string, checksum string) error {
193-
checksumDirPath, checksumFilePath := getChecksumPath(dotLetsDir, cmdName, checksumName)
186+
func persistOneChecksum(checksumsDir string, cmdName string, checksumName string, checksum string) error {
187+
checksumDirPath, checksumFilePath := getChecksumPath(checksumsDir, cmdName, checksumName)
194188
if err := util.SafeCreateDir(checksumDirPath); err != nil {
195189
return fmt.Errorf("can not create checksum dir at %s: %w", checksumDirPath, err)
196190
}
@@ -209,9 +203,9 @@ func persistOneChecksum(dotLetsDir string, cmdName string, checksumName string,
209203
}
210204

211205
// IsChecksumForCmdPersisted checks if checksums for cmd exists and persisted.
212-
func IsChecksumForCmdPersisted(dotLetsDir string, cmdName string) bool {
206+
func IsChecksumForCmdPersisted(checksumsDir string, cmdName string) bool {
213207
// check if checksums for cmd exists
214-
if _, err := os.Stat(getCmdChecksumPath(dotLetsDir, cmdName)); err != nil {
208+
if _, err := os.Stat(getCmdChecksumPath(checksumsDir, cmdName)); err != nil {
215209
return !os.IsNotExist(err)
216210
}
217211

config/config/command.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -145,15 +145,15 @@ func (cmd *Command) GetPersistedChecksums() map[string]string {
145145
}
146146

147147
// ReadChecksumsFromDisk reads all checksums for cmd into map.
148-
func (cmd *Command) ReadChecksumsFromDisk(dotLetsDir string, cmdName string, checksumMap map[string]string) error {
148+
func (cmd *Command) ReadChecksumsFromDisk(checksumsDir string, cmdName string, checksumMap map[string]string) error {
149149
checksums := make(map[string]string, len(checksumMap)+1)
150150

151151
for checksumName := range checksumMap {
152152
filename := checksumName
153153
if checksumName == checksum.DefaultChecksumKey {
154154
filename = checksum.DefaultChecksumFileName
155155
}
156-
checksumResult, err := checksum.ReadChecksumFromDisk(dotLetsDir, cmdName, filename)
156+
checksumResult, err := checksum.ReadChecksumFromDisk(checksumsDir, cmdName, filename)
157157
if err != nil {
158158
return err
159159
}

config/config/config.go

Lines changed: 38 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
package config
22

3-
import "github.com/lets-cli/lets/set"
3+
import (
4+
"fmt"
5+
"path/filepath"
6+
7+
"github.com/lets-cli/lets/set"
8+
"github.com/lets-cli/lets/util"
9+
)
410

511
var (
612
// COMMANDS is a top-level directive. Includes all commands to run.
@@ -36,21 +42,43 @@ type Config struct {
3642
isMixin bool // if true, we consider config as mixin and apply different parsing and validation
3743
// absolute path to .lets
3844
DotLetsDir string
45+
// absolute path to .lets/checksums
46+
ChecksumsDir string
47+
// absolute path to .lets/mixins
48+
MixinsDir string
3949
}
4050

4151
func NewConfig(workDir string, configAbsPath string, dotLetsDir string) *Config {
4252
return &Config{
43-
Commands: make(map[string]Command),
44-
Env: make(map[string]string),
45-
WorkDir: workDir,
46-
FilePath: configAbsPath,
47-
DotLetsDir: dotLetsDir,
53+
Commands: make(map[string]Command),
54+
Env: make(map[string]string),
55+
WorkDir: workDir,
56+
FilePath: configAbsPath,
57+
DotLetsDir: dotLetsDir,
58+
ChecksumsDir: filepath.Join(dotLetsDir, "checksums"),
59+
MixinsDir: filepath.Join(dotLetsDir, "mixins"),
60+
}
61+
}
62+
63+
func NewMixinConfig(cfg *Config, configAbsPath string) *Config {
64+
mixin := NewConfig(cfg.WorkDir, configAbsPath, cfg.DotLetsDir)
65+
mixin.isMixin = true
66+
67+
return mixin
68+
}
69+
70+
func (c *Config) CreateChecksumsDir() error {
71+
if err := util.SafeCreateDir(c.ChecksumsDir); err != nil {
72+
return fmt.Errorf("can not create %s: %w", c.ChecksumsDir, err)
4873
}
74+
75+
return nil
4976
}
5077

51-
func NewMixinConfig(workDir string, configAbsPath string, dotLetsDir string) *Config {
52-
cfg := NewConfig(workDir, configAbsPath, dotLetsDir)
53-
cfg.isMixin = true
78+
func (c *Config) CreateMixinsDir() error {
79+
if err := util.SafeCreateDir(c.MixinsDir); err != nil {
80+
return fmt.Errorf("can not create %s: %w", c.MixinsDir, err)
81+
}
5482

55-
return cfg
83+
return nil
5684
}

config/find.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"path/filepath"
66

77
"github.com/lets-cli/lets/config/path"
8+
"github.com/lets-cli/lets/util"
89
"github.com/lets-cli/lets/workdir"
910
log "github.com/sirupsen/logrus"
1011
)
@@ -71,6 +72,10 @@ func FindConfig(configName string, configDir string) (PathInfo, error) {
7172
return PathInfo{}, fmt.Errorf("can not get .lets absolute path: %w", err)
7273
}
7374

75+
if err := util.SafeCreateDir(dotLetsDir); err != nil {
76+
return PathInfo{}, fmt.Errorf("can not create .lets dir: %w", err)
77+
}
78+
7479
pathInfo := PathInfo{
7580
AbsPath: configAbsPath,
7681
WorkDir: workDir,

config/parser/parser.go

Lines changed: 146 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,16 @@ package parser
22

33
import (
44
"bytes"
5+
"context"
6+
"crypto/sha256"
57
"errors"
68
"fmt"
9+
"io"
10+
"net/http"
711
"os"
12+
"path/filepath"
813
"strings"
14+
"time"
915

1016
"github.com/lets-cli/lets/config/config"
1117
"github.com/lets-cli/lets/config/path"
@@ -193,15 +199,113 @@ func isIgnoredMixin(filename string) bool {
193199
return strings.HasPrefix(filename, "-")
194200
}
195201

202+
type RemoteMixin struct {
203+
URL string
204+
Version string
205+
206+
mixinsDir string
207+
}
208+
209+
// Filename is name of mixin file (hash from url).
210+
func (rm *RemoteMixin) Filename() string {
211+
hasher := sha256.New()
212+
hasher.Write([]byte(rm.URL))
213+
214+
if rm.Version != "" {
215+
hasher.Write([]byte(rm.Version))
216+
}
217+
218+
return fmt.Sprintf("%x", hasher.Sum(nil))
219+
}
220+
221+
// Path is abs path to mixin file (.lets/mixins/<filename>).
222+
func (rm *RemoteMixin) Path() string {
223+
return filepath.Join(rm.mixinsDir, rm.Filename())
224+
}
225+
226+
func (rm *RemoteMixin) persist(data []byte) error {
227+
f, err := os.OpenFile(rm.Path(), os.O_CREATE|os.O_WRONLY, 0o755)
228+
if err != nil {
229+
return fmt.Errorf("can not open file %s to persist mixin: %w", rm.Path(), err)
230+
}
231+
232+
_, err = f.Write(data)
233+
if err != nil {
234+
return fmt.Errorf("can not write mixin to file %s: %w", rm.Path(), err)
235+
}
236+
237+
return nil
238+
}
239+
240+
func (rm *RemoteMixin) exists() bool {
241+
return util.FileExists(rm.Path())
242+
}
243+
244+
func (rm *RemoteMixin) tryRead() ([]byte, error) {
245+
if !rm.exists() {
246+
return nil, nil
247+
}
248+
data, err := os.ReadFile(rm.Path())
249+
if err != nil {
250+
return nil, fmt.Errorf("can not read mixin config file at %s: %w", rm.Path(), err)
251+
}
252+
253+
return data, nil
254+
}
255+
256+
func (rm *RemoteMixin) download() ([]byte, error) {
257+
// TODO: maybe create a client for this?
258+
ctx, cancel := context.WithTimeout(context.Background(), 60*5*time.Second)
259+
defer cancel()
260+
261+
req, err := http.NewRequestWithContext(
262+
ctx,
263+
"GET",
264+
rm.URL,
265+
nil,
266+
)
267+
if err != nil {
268+
return nil, err
269+
}
270+
271+
client := &http.Client{
272+
Timeout: 15 * 60 * time.Second, // TODO: move to client struct
273+
}
274+
275+
resp, err := client.Do(req)
276+
if err != nil {
277+
return nil, fmt.Errorf("failed to make request: %w", err)
278+
}
279+
280+
defer resp.Body.Close()
281+
282+
if resp.StatusCode == http.StatusNotFound {
283+
return nil, fmt.Errorf("no such file at: %s", rm.URL)
284+
} else if resp.StatusCode < 200 || resp.StatusCode > 299 {
285+
return nil, fmt.Errorf("network error: %s", resp.Status)
286+
}
287+
288+
data, err := io.ReadAll(resp.Body)
289+
if err != nil {
290+
return nil, fmt.Errorf("failed to read response: %w", err)
291+
}
292+
293+
return data, nil
294+
}
295+
196296
func readAndValidateMixins(mixins []interface{}, cfg *config.Config) error {
197-
for _, filename := range mixins {
198-
if filename, ok := filename.(string); ok { //nolint:nestif
297+
if err := cfg.CreateMixinsDir(); err != nil {
298+
return err
299+
}
300+
301+
for _, mixin := range mixins {
302+
if filename, ok := mixin.(string); ok { //nolint:nestif
199303
configAbsPath, err := path.GetFullConfigPath(normalizeMixinFilename(filename), cfg.WorkDir)
200304
if err != nil {
201305
if isIgnoredMixin(filename) && errors.Is(err, path.ErrFileNotExists) {
202306
continue
203307
} else {
204-
// complain non-existed mixin only if its filename does not starts with dash `-`
308+
// complain non-existed mixin only if its filename does not start with dash `-`
205309
return fmt.Errorf("failed to read mixin config: %w", err)
206310
}
207311
}
@@ -210,14 +314,52 @@ func readAndValidateMixins(mixins []interface{}, cfg *config.Config) error {
210314
return fmt.Errorf("can not read mixin config file: %w", err)
211315
}
212316

213-
mixinCfg := config.NewMixinConfig(cfg.WorkDir, filename, cfg.DotLetsDir)
317+
mixinCfg := config.NewMixinConfig(cfg, filename)
214318
if err := parseMixinConfig(fileData, mixinCfg); err != nil {
215319
return fmt.Errorf("failed to load mixin config '%s': %w", filename, err)
216320
}
217321

218322
if err := mergeConfigs(cfg, mixinCfg); err != nil {
219323
return fmt.Errorf("failed to merge mixin config %s with main config: %w", filename, err)
220324
}
325+
} else if mixinMapping, ok := mixin.(map[string]interface{}); ok {
326+
rm := &RemoteMixin{mixinsDir: cfg.MixinsDir}
327+
if url, ok := mixinMapping["url"]; ok {
328+
// TODO check if url is valid
329+
rm.URL, _ = url.(string)
330+
}
331+
332+
if version, ok := mixinMapping["version"]; ok {
333+
rm.Version, _ = version.(string)
334+
}
335+
336+
data, err := rm.tryRead()
337+
if err != nil {
338+
return err
339+
}
340+
341+
if data == nil {
342+
data, err = rm.download()
343+
if err != nil {
344+
return err
345+
}
346+
}
347+
348+
// TODO: what if multiple mixins have same commands
349+
// 1 option - fail and suggest use to namespace all commands in remote mixin
350+
// 2 option - namespace it (this may require specifying namespace in mixin config or in main config mixin section)
351+
mixinCfg := config.NewMixinConfig(cfg, rm.Filename())
352+
if err := parseMixinConfig(data, mixinCfg); err != nil {
353+
return fmt.Errorf("failed to load remote mixin config '%s': %w", rm.URL, err)
354+
}
355+
356+
if err := mergeConfigs(cfg, mixinCfg); err != nil {
357+
return fmt.Errorf("failed to merge remote mixin config %s with main config: %w", rm.URL, err)
358+
}
359+
360+
if err := rm.persist(data); err != nil {
361+
return fmt.Errorf("failed to persist remote mixin config %s: %w", rm.URL, err)
362+
}
221363
} else {
222364
return newConfigParseError(
223365
"must be a string",

0 commit comments

Comments
 (0)