blob: 14e2cc0415d414ae7ed81fd0d8e5f56e8d9998c0 [file] [log] [blame]
// Copyright 2018 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package lucicfg
import (
"context"
"encoding/base64"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"sort"
"github.com/golang/protobuf/proto"
"go.starlark.net/starlark"
config "go.chromium.org/luci/common/api/luci_config/config/v1"
"go.chromium.org/luci/common/errors"
"go.chromium.org/luci/common/logging"
"go.chromium.org/luci/starlark/builtins"
"go.chromium.org/luci/starlark/starlarkproto"
)
// Alias some ridiculously long type names that we round-trip in the public API.
type (
ValidationRequest = config.LuciConfigValidateConfigRequestMessage
ValidationMessage = config.ComponentsConfigEndpointValidationMessage
)
// ConfigSet is an in-memory representation of a config set.
//
// Keys are slash-separated filenames, values are corresponding file bodies.
type ConfigSet map[string][]byte
// ValidationResult is what we get after validating a config set.
type ValidationResult struct {
Messages []*ValidationMessage `json:"messages"` // errors, warning, infos, etc.
}
// ConfigSetValidator is primarily implemented through config.Service, but can
// also be mocked in tests.
type ConfigSetValidator interface {
// Validate sends the validation request to the service.
Validate(ctx context.Context, req *ValidationRequest) (*ValidationResult, error)
}
type remoteValidator struct {
svc *config.Service
}
func (r remoteValidator) Validate(ctx context.Context, req *ValidationRequest) (*ValidationResult, error) {
resp, err := r.svc.ValidateConfig(req).Context(ctx).Do()
if err != nil {
return nil, err
}
return &ValidationResult{Messages: resp.Messages}, nil
}
// RemoteValidator returns ConfigSetValidator that makes RPCs to LUCI Config.
func RemoteValidator(svc *config.Service) ConfigSetValidator {
return remoteValidator{svc}
}
// ReadConfigSet reads all regular files in the given directory (recursively)
// and returns them as a ConfigSet.
func ReadConfigSet(dir string) (ConfigSet, error) {
configs := ConfigSet{}
err := filepath.Walk(dir, func(p string, info os.FileInfo, err error) error {
if err != nil || !info.Mode().IsRegular() {
return err
}
content, err := ioutil.ReadFile(p)
if err != nil {
return err
}
relPath, err := filepath.Rel(dir, p)
if err != nil {
return err
}
configs[filepath.ToSlash(relPath)] = content
return nil
})
if err != nil {
return nil, errors.Annotate(err, "failed to read config files").Err()
}
return configs, nil
}
// Write stores files in the config set to disk.
//
// Creates missing directories. Not atomic. All files have mode 0666.
func (cs ConfigSet) Write(dir string) error {
for name, body := range cs {
path := filepath.Join(dir, filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
return err
}
if err := ioutil.WriteFile(path, body, 0666); err != nil {
return err
}
}
return nil
}
// Files returns a sorted list of file names in the config set.
func (cs ConfigSet) Files() []string {
f := make([]string, 0, len(cs))
for k := range cs {
f = append(f, k)
}
sort.Strings(f)
return f
}
// Validate sends the config set for validation to LUCI Config service.
//
// 'name' is a name of this config set from LUCI Config point of view, e.g.
// "projects/<something>" or "services/<something>". It tells LUCI Config how
// to validate files in the set.
//
// Returns an error only if the validation call itself failed (e.g. LUCI Config
// was unreachable). Otherwise returns ValidationResult with a list of
// validation message (errors, warnings, etc). The list of messages may be empty
// if the config set is 100% valid.
func (cs ConfigSet) Validate(ctx context.Context, name string, val ConfigSetValidator) (*ValidationResult, error) {
req := &ValidationRequest{
ConfigSet: name,
Files: make([]*config.LuciConfigValidateConfigRequestMessageFile, len(cs)),
}
logging.Debugf(ctx, "Sending for validation to LUCI Config:")
for idx, f := range cs.Files() {
logging.Debugf(ctx, " %s (%d bytes)", f, len(cs[f]))
req.Files[idx] = &config.LuciConfigValidateConfigRequestMessageFile{
Path: f,
Content: base64.StdEncoding.EncodeToString(cs[f]),
}
}
res, err := val.Validate(ctx, req)
if err != nil {
return nil, errors.Annotate(err, "error validating configs").Err()
}
return res, nil
}
// Log all messages in the result to the logger at an appropriate logging level.
func (vr *ValidationResult) Log(ctx context.Context) {
for _, msg := range vr.Messages {
lvl := logging.Info
switch msg.Severity {
case "WARNING":
lvl = logging.Warning
case "ERROR", "CRITICAL":
lvl = logging.Error
}
logging.Logf(ctx, lvl, "%s: %s", msg.Path, msg.Text)
}
}
// OverallError is nil if the validation succeeded or non-nil if failed.
func (vr *ValidationResult) OverallError(failOnWarnings bool) error {
errs, warns := 0, 0
for _, msg := range vr.Messages {
switch msg.Severity {
case "WARNING":
warns++
case "ERROR", "CRITICAL":
errs++
}
}
if errs > 0 {
return errors.Reason("some files were invalid").Err()
}
if warns > 0 && failOnWarnings {
return errors.Reason("some files had validation warnings and -fail-on-warnings is set").Err()
}
return nil
}
////////////////////////////////////////////////////////////////////////////////
// Constructing config sets from Starlark (tested through starlark_test.go).
// configSetValue is a map-like starlark.Value that has file names as keys and
// strings or protobuf messages as values.
//
// At the end of the execution all protos are serialized to strings too, using
// textpb encoding.
type configSetValue struct {
starlark.Dict
}
func newConfigSetValue() *configSetValue {
return &configSetValue{}
}
func (cs *configSetValue) Type() string { return "config_set" }
func (cs *configSetValue) SetKey(k, v starlark.Value) error {
if _, ok := k.(starlark.String); !ok {
return fmt.Errorf("config set key should be a string, not %s", k.Type())
}
_, str := v.(starlark.String)
_, msg := v.(*starlarkproto.Message)
if !str && !msg {
return fmt.Errorf("config set value should be either a string or a proto message, not %s", v.Type())
}
return cs.Dict.SetKey(k, v)
}
// renderWithTextProto returns a config set with all protos serialized to text
// proto format (with added header).
//
// Configs supplied as strings are serialized using UTF-8 encoding.
func (cs *configSetValue) renderWithTextProto(header string) (ConfigSet, error) {
out := make(ConfigSet, cs.Len())
for _, kv := range cs.Items() {
k, v := kv[0].(starlark.String), kv[1]
text := ""
if s, ok := v.(starlark.String); ok {
text = s.GoString()
} else {
msg, err := v.(*starlarkproto.Message).ToProto()
if err != nil {
return nil, err
}
text = header + proto.MarshalTextString(msg)
}
out[k.GoString()] = []byte(text)
}
return out, nil
}
// generators is a list of registered generator callbacks.
//
// It lives in State. Generators are executed sequentially after all Starlark
// code is loaded. They examine the state and generate configs based on it.
type generators struct {
gen []starlark.Callable
frozen bool // true while iterating over the list
}
// add registers a new generator callback.
func (g *generators) add(cb starlark.Callable) error {
if g.frozen {
return fmt.Errorf("generators list is frozen during iteration")
}
g.gen = append(g.gen, cb)
return nil
}
// call calls all registered callbacks sequentially, collecting all errors.
func (g *generators) call(th *starlark.Thread, ctx *genCtx) (errs errors.MultiError) {
if g.frozen {
return errors.MultiError{
fmt.Errorf("generators list is frozen, nested iteration?"),
}
}
g.frozen = true
defer func() { g.frozen = false }()
fc := builtins.GetFailureCollector(th)
for _, cb := range g.gen {
if fc != nil {
fc.Clear()
}
if _, err := starlark.Call(th, cb, starlark.Tuple{ctx}, nil); err != nil {
if fc != nil && fc.LatestFailure() != nil {
// Prefer this error, it has custom stack trace.
errs = append(errs, fc.LatestFailure())
} else {
errs = append(errs, err)
}
}
}
return
}
func init() {
// new_config_set() makes a new empty config set, useful in tests.
declNative("new_config_set", func(call nativeCall) (starlark.Value, error) {
if err := call.unpack(0); err != nil {
return nil, err
}
return newConfigSetValue(), nil
})
// add_generator(cb) registers a callback that is called at the end of the
// execution to generate or mutate produced configs.
declNative("add_generator", func(call nativeCall) (starlark.Value, error) {
var cb starlark.Callable
if err := call.unpack(1, &cb); err != nil {
return nil, err
}
return starlark.None, call.State.generators.add(cb)
})
// call_generators(ctx) calls all registered generators, useful in tests.
declNative("call_generators", func(call nativeCall) (starlark.Value, error) {
var ctx *genCtx
if err := call.unpack(1, &ctx); err != nil {
return nil, err
}
switch errs := call.State.generators.call(call.Thread, ctx); {
case len(errs) == 0:
return starlark.None, nil
case len(errs) == 1:
return starlark.None, errs[0]
default:
return starlark.None, errs
}
})
}