| // Copyright 2018 The LUCI Authors. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package lucicfg |
| |
| import ( |
| "context" |
| "encoding/base64" |
| "fmt" |
| "io/ioutil" |
| "os" |
| "path/filepath" |
| "sort" |
| |
| "github.com/golang/protobuf/proto" |
| "go.starlark.net/starlark" |
| |
| config "go.chromium.org/luci/common/api/luci_config/config/v1" |
| "go.chromium.org/luci/common/errors" |
| "go.chromium.org/luci/common/logging" |
| |
| "go.chromium.org/luci/starlark/builtins" |
| "go.chromium.org/luci/starlark/starlarkproto" |
| ) |
| |
| // Alias some ridiculously long type names that we round-trip in the public API. |
| type ( |
| ValidationRequest = config.LuciConfigValidateConfigRequestMessage |
| ValidationMessage = config.ComponentsConfigEndpointValidationMessage |
| ) |
| |
| // ConfigSet is an in-memory representation of a config set. |
| // |
| // Keys are slash-separated filenames, values are corresponding file bodies. |
| type ConfigSet map[string][]byte |
| |
| // ValidationResult is what we get after validating a config set. |
| type ValidationResult struct { |
| Messages []*ValidationMessage `json:"messages"` // errors, warning, infos, etc. |
| } |
| |
| // ConfigSetValidator is primarily implemented through config.Service, but can |
| // also be mocked in tests. |
| type ConfigSetValidator interface { |
| // Validate sends the validation request to the service. |
| Validate(ctx context.Context, req *ValidationRequest) (*ValidationResult, error) |
| } |
| |
| type remoteValidator struct { |
| svc *config.Service |
| } |
| |
| func (r remoteValidator) Validate(ctx context.Context, req *ValidationRequest) (*ValidationResult, error) { |
| resp, err := r.svc.ValidateConfig(req).Context(ctx).Do() |
| if err != nil { |
| return nil, err |
| } |
| return &ValidationResult{Messages: resp.Messages}, nil |
| } |
| |
| // RemoteValidator returns ConfigSetValidator that makes RPCs to LUCI Config. |
| func RemoteValidator(svc *config.Service) ConfigSetValidator { |
| return remoteValidator{svc} |
| } |
| |
| // ReadConfigSet reads all regular files in the given directory (recursively) |
| // and returns them as a ConfigSet. |
| func ReadConfigSet(dir string) (ConfigSet, error) { |
| configs := ConfigSet{} |
| err := filepath.Walk(dir, func(p string, info os.FileInfo, err error) error { |
| if err != nil || !info.Mode().IsRegular() { |
| return err |
| } |
| content, err := ioutil.ReadFile(p) |
| if err != nil { |
| return err |
| } |
| relPath, err := filepath.Rel(dir, p) |
| if err != nil { |
| return err |
| } |
| configs[filepath.ToSlash(relPath)] = content |
| return nil |
| }) |
| if err != nil { |
| return nil, errors.Annotate(err, "failed to read config files").Err() |
| } |
| return configs, nil |
| } |
| |
| // Write stores files in the config set to disk. |
| // |
| // Creates missing directories. Not atomic. All files have mode 0666. |
| func (cs ConfigSet) Write(dir string) error { |
| for name, body := range cs { |
| path := filepath.Join(dir, filepath.FromSlash(name)) |
| if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil { |
| return err |
| } |
| if err := ioutil.WriteFile(path, body, 0666); err != nil { |
| return err |
| } |
| } |
| return nil |
| } |
| |
| // Files returns a sorted list of file names in the config set. |
| func (cs ConfigSet) Files() []string { |
| f := make([]string, 0, len(cs)) |
| for k := range cs { |
| f = append(f, k) |
| } |
| sort.Strings(f) |
| return f |
| } |
| |
| // Validate sends the config set for validation to LUCI Config service. |
| // |
| // 'name' is a name of this config set from LUCI Config point of view, e.g. |
| // "projects/<something>" or "services/<something>". It tells LUCI Config how |
| // to validate files in the set. |
| // |
| // Returns an error only if the validation call itself failed (e.g. LUCI Config |
| // was unreachable). Otherwise returns ValidationResult with a list of |
| // validation message (errors, warnings, etc). The list of messages may be empty |
| // if the config set is 100% valid. |
| func (cs ConfigSet) Validate(ctx context.Context, name string, val ConfigSetValidator) (*ValidationResult, error) { |
| req := &ValidationRequest{ |
| ConfigSet: name, |
| Files: make([]*config.LuciConfigValidateConfigRequestMessageFile, len(cs)), |
| } |
| |
| logging.Debugf(ctx, "Sending for validation to LUCI Config:") |
| for idx, f := range cs.Files() { |
| logging.Debugf(ctx, " %s (%d bytes)", f, len(cs[f])) |
| req.Files[idx] = &config.LuciConfigValidateConfigRequestMessageFile{ |
| Path: f, |
| Content: base64.StdEncoding.EncodeToString(cs[f]), |
| } |
| } |
| |
| res, err := val.Validate(ctx, req) |
| if err != nil { |
| return nil, errors.Annotate(err, "error validating configs").Err() |
| } |
| return res, nil |
| } |
| |
| // Log all messages in the result to the logger at an appropriate logging level. |
| func (vr *ValidationResult) Log(ctx context.Context) { |
| for _, msg := range vr.Messages { |
| lvl := logging.Info |
| switch msg.Severity { |
| case "WARNING": |
| lvl = logging.Warning |
| case "ERROR", "CRITICAL": |
| lvl = logging.Error |
| } |
| logging.Logf(ctx, lvl, "%s: %s", msg.Path, msg.Text) |
| } |
| } |
| |
| // OverallError is nil if the validation succeeded or non-nil if failed. |
| func (vr *ValidationResult) OverallError(failOnWarnings bool) error { |
| errs, warns := 0, 0 |
| for _, msg := range vr.Messages { |
| switch msg.Severity { |
| case "WARNING": |
| warns++ |
| case "ERROR", "CRITICAL": |
| errs++ |
| } |
| } |
| |
| if errs > 0 { |
| return errors.Reason("some files were invalid").Err() |
| } |
| if warns > 0 && failOnWarnings { |
| return errors.Reason("some files had validation warnings and -fail-on-warnings is set").Err() |
| } |
| |
| return nil |
| } |
| |
| //////////////////////////////////////////////////////////////////////////////// |
| // Constructing config sets from Starlark (tested through starlark_test.go). |
| |
| // configSetValue is a map-like starlark.Value that has file names as keys and |
| // strings or protobuf messages as values. |
| // |
| // At the end of the execution all protos are serialized to strings too, using |
| // textpb encoding. |
| type configSetValue struct { |
| starlark.Dict |
| } |
| |
| func newConfigSetValue() *configSetValue { |
| return &configSetValue{} |
| } |
| |
| func (cs *configSetValue) Type() string { return "config_set" } |
| |
| func (cs *configSetValue) SetKey(k, v starlark.Value) error { |
| if _, ok := k.(starlark.String); !ok { |
| return fmt.Errorf("config set key should be a string, not %s", k.Type()) |
| } |
| |
| _, str := v.(starlark.String) |
| _, msg := v.(*starlarkproto.Message) |
| if !str && !msg { |
| return fmt.Errorf("config set value should be either a string or a proto message, not %s", v.Type()) |
| } |
| |
| return cs.Dict.SetKey(k, v) |
| } |
| |
| // renderWithTextProto returns a config set with all protos serialized to text |
| // proto format (with added header). |
| // |
| // Configs supplied as strings are serialized using UTF-8 encoding. |
| func (cs *configSetValue) renderWithTextProto(header string) (ConfigSet, error) { |
| out := make(ConfigSet, cs.Len()) |
| |
| for _, kv := range cs.Items() { |
| k, v := kv[0].(starlark.String), kv[1] |
| |
| text := "" |
| if s, ok := v.(starlark.String); ok { |
| text = s.GoString() |
| } else { |
| msg, err := v.(*starlarkproto.Message).ToProto() |
| if err != nil { |
| return nil, err |
| } |
| text = header + proto.MarshalTextString(msg) |
| } |
| |
| out[k.GoString()] = []byte(text) |
| } |
| |
| return out, nil |
| } |
| |
| // generators is a list of registered generator callbacks. |
| // |
| // It lives in State. Generators are executed sequentially after all Starlark |
| // code is loaded. They examine the state and generate configs based on it. |
| type generators struct { |
| gen []starlark.Callable |
| frozen bool // true while iterating over the list |
| } |
| |
| // add registers a new generator callback. |
| func (g *generators) add(cb starlark.Callable) error { |
| if g.frozen { |
| return fmt.Errorf("generators list is frozen during iteration") |
| } |
| g.gen = append(g.gen, cb) |
| return nil |
| } |
| |
| // call calls all registered callbacks sequentially, collecting all errors. |
| func (g *generators) call(th *starlark.Thread, ctx *genCtx) (errs errors.MultiError) { |
| if g.frozen { |
| return errors.MultiError{ |
| fmt.Errorf("generators list is frozen, nested iteration?"), |
| } |
| } |
| g.frozen = true |
| defer func() { g.frozen = false }() |
| |
| fc := builtins.GetFailureCollector(th) |
| |
| for _, cb := range g.gen { |
| if fc != nil { |
| fc.Clear() |
| } |
| if _, err := starlark.Call(th, cb, starlark.Tuple{ctx}, nil); err != nil { |
| if fc != nil && fc.LatestFailure() != nil { |
| // Prefer this error, it has custom stack trace. |
| errs = append(errs, fc.LatestFailure()) |
| } else { |
| errs = append(errs, err) |
| } |
| } |
| } |
| return |
| } |
| |
| func init() { |
| // new_config_set() makes a new empty config set, useful in tests. |
| declNative("new_config_set", func(call nativeCall) (starlark.Value, error) { |
| if err := call.unpack(0); err != nil { |
| return nil, err |
| } |
| return newConfigSetValue(), nil |
| }) |
| |
| // add_generator(cb) registers a callback that is called at the end of the |
| // execution to generate or mutate produced configs. |
| declNative("add_generator", func(call nativeCall) (starlark.Value, error) { |
| var cb starlark.Callable |
| if err := call.unpack(1, &cb); err != nil { |
| return nil, err |
| } |
| return starlark.None, call.State.generators.add(cb) |
| }) |
| |
| // call_generators(ctx) calls all registered generators, useful in tests. |
| declNative("call_generators", func(call nativeCall) (starlark.Value, error) { |
| var ctx *genCtx |
| if err := call.unpack(1, &ctx); err != nil { |
| return nil, err |
| } |
| switch errs := call.State.generators.call(call.Thread, ctx); { |
| case len(errs) == 0: |
| return starlark.None, nil |
| case len(errs) == 1: |
| return starlark.None, errs[0] |
| default: |
| return starlark.None, errs |
| } |
| }) |
| } |