Merge pull request #189 from yandex/dev

v0.5.24

.changes/v0.5.24.md

@@ -0,0 +1,9 @@
+## v0.5.24 - 2024-04-27
+### Added
+* HCL scenario: add support for "locals" block in hcl
+* grpc/scenario: debug logs
+* grpc generator: reflection metadata
+### Changed
+* discard_overflow default value =true
+### Fixed
+* http generator: fix answlog error

.mapping.json

@@ -21,6 +21,7 @@
   ".changes/v0.5.21.md":"load/projects/pandora/.changes/v0.5.21.md",
   ".changes/v0.5.22.md":"load/projects/pandora/.changes/v0.5.22.md",
   ".changes/v0.5.23.md":"load/projects/pandora/.changes/v0.5.23.md",
+  ".changes/v0.5.24.md":"load/projects/pandora/.changes/v0.5.24.md",
   ".changie.yaml":"load/projects/pandora/.changie.yaml",
   ".github/workflows/release.yml":"load/projects/pandora/.github/workflows/release.yml",
   ".github/workflows/test.yml":"load/projects/pandora/.github/workflows/test.yml",
@@ -257,6 +258,7 @@
   "debian/source/format":"load/projects/pandora/debian/source/format",
   "docs/eng/architecture.md":"load/projects/pandora/docs/eng/architecture.md",
   "docs/eng/best-practices.md":"load/projects/pandora/docs/eng/best-practices.md",
+  "docs/eng/best_practices/discard-overflow.md":"load/projects/pandora/docs/eng/best_practices/discard-overflow.md",
   "docs/eng/best_practices/rps-per-instance.md":"load/projects/pandora/docs/eng/best_practices/rps-per-instance.md",
   "docs/eng/best_practices/shared-client.md":"load/projects/pandora/docs/eng/best_practices/shared-client.md",
   "docs/eng/config.md":"load/projects/pandora/docs/eng/config.md",
@@ -286,6 +288,7 @@
   "docs/rus/architecture.md":"load/projects/pandora/docs/rus/architecture.md",
   "docs/rus/best-practices.md":"load/projects/pandora/docs/rus/best-practices.md",
   "docs/rus/best_practices.md":"load/projects/pandora/docs/rus/best_practices.md",
+  "docs/rus/best_practices/discard-overflow.md":"load/projects/pandora/docs/rus/best_practices/discard-overflow.md",
   "docs/rus/best_practices/rps-per-instance.md":"load/projects/pandora/docs/rus/best_practices/rps-per-instance.md",
   "docs/rus/best_practices/shared-client.md":"load/projects/pandora/docs/rus/best_practices/shared-client.md",
   "docs/rus/config.md":"load/projects/pandora/docs/rus/config.md",

CHANGELOG.md

@@ -6,6 +6,16 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
 
+## v0.5.24 - 2024-04-27
+### Added
+* HCL scenario: add support for "locals" block in hcl
+* grpc/scenario: debug logs
+* grpc generator: reflection metadata
+### Changed
+* discard_overflow default value =true
+### Fixed
+* http generator: fix answlog error
+
 ## v0.5.23 - 2024-04-16
 ### Added
 * function for generate random values in scenario

cli/cli.go

@@ -25,7 +25,7 @@ import (
 	"go.uber.org/zap/zapcore"
 )
 
-const Version = "0.5.23"
+const Version = "0.5.24"
 const defaultConfigFile = "load"
 const stdinConfigSelector = "-"
 
@@ -218,6 +218,7 @@ func readConfig(args []string) *CliConfig {
 		}
 	}
 
+	log.Info("Pandora version", zap.String("version", Version))
 	if useStdinConfig {
 		v.SetConfigType("yaml")
 		configBuffer, err := ioutil.ReadAll(bufio.NewReader(os.Stdin))
@@ -235,6 +236,15 @@ func readConfig(args []string) *CliConfig {
 			log.Fatal("Config read failed", zap.Error(err))
 		}
 	}
+	pools := v.Get("pools").([]any)
+	for i, pool := range pools {
+		poolMap := pool.(map[string]any)
+		if _, ok := poolMap["discard_overflow"]; !ok {
+			poolMap["discard_overflow"] = true
+		}
+		pools[i] = poolMap
+	}
+	v.Set("pools", pools)
 
 	conf := DefaultConfig()
 	err = config.DecodeAndValidate(v.AllSettings(), conf)

components/guns/grpc/core.go

@@ -43,13 +43,14 @@ type GrpcDialOptions struct {
 }
 
 type GunConfig struct {
-	Target       string          `validate:"required"`
-	ReflectPort  int64           `config:"reflect_port"`
-	Timeout      time.Duration   `config:"timeout"` // grpc request timeout
-	TLS          bool            `config:"tls"`
-	DialOptions  GrpcDialOptions `config:"dial_options"`
-	AnswLog      AnswLogConfig   `config:"answlog"`
-	SharedClient struct {
+	Target          string          `validate:"required"`
+	ReflectPort     int64           `config:"reflect_port"`
+	ReflectMetadata metadata.MD     `config:"reflect_metadata"`
+	Timeout         time.Duration   `config:"timeout"` // grpc request timeout
+	TLS             bool            `config:"tls"`
+	DialOptions     GrpcDialOptions `config:"dial_options"`
+	AnswLog         AnswLogConfig   `config:"answlog"`
+	SharedClient    struct {
 		ClientNumber int  `config:"client-number,omitempty"`
 		Enabled      bool `config:"enabled"`
 	} `config:"shared-client,omitempty"`
@@ -110,8 +111,7 @@ func (g *Gun) prepareMethodList(opts *warmup.Options) (map[string]desc.MethodDes
 	}
 	defer conn.Close()
 
-	meta := make(metadata.MD)
-	refCtx := metadata.NewOutgoingContext(context.Background(), meta)
+	refCtx := metadata.NewOutgoingContext(context.Background(), g.Conf.ReflectMetadata)
 	refClient := grpcreflect.NewClientAuto(refCtx, conn)
 	listServices, err := refClient.ListServices()
 	if err != nil {
@@ -238,27 +238,35 @@ func (g *Gun) shoot(ammo *ammo.Ammo) {
 		g.GunDeps.Log.Error("response error", zap.Error(err))
 	}
 
+	g.Answ(&method, message, ammo.Metadata, out, grpcErr, code)
+}
+
+func (g *Gun) Answ(method *desc.MethodDescriptor, message *dynamic.Message, metadata map[string]string, out proto.Message, grpcErr error, code int) {
 	if g.Conf.AnswLog.Enabled {
 		switch g.Conf.AnswLog.Filter {
 		case "all":
-			g.AnswLogging(g.AnswLog, &method, message, out, grpcErr)
+			g.AnswLogging(g.AnswLog, method, message, metadata, out, grpcErr)
 
 		case "warning":
 			if code >= 400 {
-				g.AnswLogging(g.AnswLog, &method, message, out, grpcErr)
+				g.AnswLogging(g.AnswLog, method, message, metadata, out, grpcErr)
 			}
 
 		case "error":
 			if code >= 500 {
-				g.AnswLogging(g.AnswLog, &method, message, out, grpcErr)
+				g.AnswLogging(g.AnswLog, method, message, metadata, out, grpcErr)
 			}
 		}
 	}
 }
 
-func (g *Gun) AnswLogging(logger *zap.Logger, method *desc.MethodDescriptor, request proto.Message, response proto.Message, grpcErr error) {
-	logger.Debug("Request:", zap.Stringer("method", method), zap.Stringer("message", request))
-	logger.Debug("Response:", zap.Stringer("resp", response), zap.Error(grpcErr))
+func (g *Gun) AnswLogging(logger *zap.Logger, method *desc.MethodDescriptor, request proto.Message, metadata map[string]string, response proto.Message, grpcErr error) {
+	logger.Debug("Request:", zap.Stringer("method", method), zap.Stringer("message", request), zap.Any("metadata", metadata))
+	if response != nil {
+		logger.Debug("Response:", zap.Stringer("resp", response), zap.Error(grpcErr))
+	} else {
+		logger.Debug("Response:", zap.String("resp", "empty"), zap.Error(grpcErr))
+	}
 }
 
 func (g *Gun) makeConnect() (conn *grpc.ClientConn, err error) {

components/guns/grpc/scenario/core.go

@@ -21,12 +21,13 @@ import (
 const defaultTimeout = time.Second * 15
 
 type GunConfig struct {
-	Target      string          `validate:"required"`
-	ReflectPort int64           `config:"reflect_port"`
-	Timeout     time.Duration   `config:"timeout"` // grpc request timeout
-	TLS         bool            `config:"tls"`
-	DialOptions GrpcDialOptions `config:"dial_options"`
-	AnswLog     AnswLogConfig   `config:"answlog"`
+	Target          string          `validate:"required"`
+	ReflectPort     int64           `config:"reflect_port"`
+	ReflectMetadata metadata.MD     `config:"reflect_metadata"`
+	Timeout         time.Duration   `config:"timeout"` // grpc request timeout
+	TLS             bool            `config:"tls"`
+	DialOptions     GrpcDialOptions `config:"dial_options"`
+	AnswLog         AnswLogConfig   `config:"answlog"`
 }
 
 type GrpcDialOptions struct {
@@ -57,10 +58,11 @@ func NewGun(conf GunConfig) *Gun {
 	return &Gun{
 		templ: NewTextTemplater(),
 		gun: &grpcgun.Gun{Conf: grpcgun.GunConfig{
-			Target:      conf.Target,
-			ReflectPort: conf.ReflectPort,
-			Timeout:     conf.Timeout,
-			TLS:         conf.TLS,
+			Target:          conf.Target,
+			ReflectPort:     conf.ReflectPort,
+			ReflectMetadata: conf.ReflectMetadata,
+			Timeout:         conf.Timeout,
+			TLS:             conf.TLS,
 			DialOptions: grpcgun.GrpcDialOptions{
 				Authority: conf.DialOptions.Authority,
 				Timeout:   conf.DialOptions.Timeout,
@@ -116,6 +118,9 @@ func (g *Gun) shoot(ammo *Scenario, templateVars map[string]any) error {
 
 	requestVars := map[string]any{}
 	templateVars["request"] = requestVars
+	if g.gun.DebugLog {
+		g.gun.GunDeps.Log.Debug("Source variables", zap.Any("variables", templateVars))
+	}
 
 	startAt := time.Now()
 	for _, call := range ammo.Calls {
@@ -154,7 +159,7 @@ func (g *Gun) shootStep(step *Call, sample *netsample.Sample, ammoName string, t
 		}
 		preprocVars = mergeMaps(preprocVars, pp)
 		if g.gun.DebugLog {
-			g.gun.GunDeps.Log.Debug("PreparePreprocessor variables", zap.Any(fmt.Sprintf(".resuest.%s.preprocessor", step.Name), pp))
+			g.gun.GunDeps.Log.Debug("PreparePreprocessor variables", zap.Any(fmt.Sprintf(".request.%s.preprocessor", step.Name), pp))
 		}
 	}
 	stepVars["preprocessor"] = preprocVars
@@ -198,22 +203,7 @@ func (g *Gun) shootStep(step *Call, sample *netsample.Sample, ammoName string, t
 		g.gun.GunDeps.Log.Error("response error", zap.Error(err))
 	}
 
-	if g.gun.Conf.AnswLog.Enabled {
-		switch g.gun.Conf.AnswLog.Filter {
-		case "all":
-			g.gun.AnswLogging(g.gun.AnswLog, &method, message, out, grpcErr)
-
-		case "warning":
-			if code >= 400 {
-				g.gun.AnswLogging(g.gun.AnswLog, &method, message, out, grpcErr)
-			}
-
-		case "error":
-			if code >= 500 {
-				g.gun.AnswLogging(g.gun.AnswLog, &method, message, out, grpcErr)
-			}
-		}
-	}
+	g.gun.Answ(&method, message, step.Metadata, out, grpcErr, code)
 
 	for _, postProcessor := range step.Postprocessors {
 		pp, err := postProcessor.Process(out, code)

components/guns/http/base.go

@@ -25,16 +25,6 @@ const (
 	EmptyTag = "__EMPTY__"
 )
 
-type BaseGunConfig struct {
-	AutoTag      AutoTagConfig   `config:"auto-tag"`
-	AnswLog      AnswLogConfig   `config:"answlog"`
-	HTTPTrace    HTTPTraceConfig `config:"httptrace"`
-	SharedClient struct {
-		ClientNumber int  `config:"client-number,omitempty"`
-		Enabled      bool `config:"enabled"`
-	} `config:"shared-client,omitempty"`
-}
-
 // AutoTagConfig configure automatic tags generation based on ammo URI. First AutoTag URI path elements becomes tag.
 // Example: /my/very/deep/page?id=23&param=33 -> /my/very when uri-elements: 2.
 type AutoTagConfig struct {
@@ -54,29 +44,10 @@ type HTTPTraceConfig struct {
 	TraceEnabled bool `config:"trace"`
 }
 
-func DefaultBaseGunConfig() BaseGunConfig {
-	return BaseGunConfig{
-		AutoTag: AutoTagConfig{
-			Enabled:     false,
-			URIElements: 2,
-			NoTagOnly:   true,
-		},
-		AnswLog: AnswLogConfig{
-			Enabled: false,
-			Path:    "answ.log",
-			Filter:  "error",
-		},
-		HTTPTrace: HTTPTraceConfig{
-			DumpEnabled:  false,
-			TraceEnabled: false,
-		},
-	}
-}
-
-func NewBaseGun(clientConstructor ClientConstructor, cfg HTTPGunConfig, answLog *zap.Logger) *BaseGun {
+func NewBaseGun(clientConstructor ClientConstructor, cfg GunConfig, answLog *zap.Logger) *BaseGun {
 	client := clientConstructor(cfg.Client, cfg.Target)
 	return &BaseGun{
-		Config: cfg.Base,
+		Config: cfg,
 		OnClose: func() error {
 			client.CloseIdleConnections()
 			return nil
@@ -91,7 +62,7 @@ func NewBaseGun(clientConstructor ClientConstructor, cfg HTTPGunConfig, answLog
 
 type BaseGun struct {
 	DebugLog          bool // Automaticaly set in Bind if Log accepts debug messages.
-	Config            BaseGunConfig
+	Config            GunConfig
 	Connect           func(ctx context.Context) error // Optional hook.
 	OnClose           func() error                    // Optional. Called on Close().
 	Aggregator        netsample.Aggregator            // Lazy set via BindResultTo.
@@ -183,6 +154,17 @@ func (b *BaseGun) Shoot(ammo Ammo) {
 		b.Log.Warn("Invalid ammo", zap.Uint64("request", ammo.ID()))
 		return
 	}
+
+	if b.Config.SSL {
+		req.URL.Scheme = "https"
+	} else {
+		req.URL.Scheme = "http"
+	}
+	if req.Host == "" {
+		req.Host = getHostWithoutPort(b.Config.Target)
+	}
+	req.URL.Host = b.Config.TargetResolved
+
 	if b.DebugLog {
 		b.Log.Debug("Prepared ammo to shoot", zap.Stringer("url", req.URL))
 	}

components/guns/http/base_test.go

@@ -63,7 +63,7 @@ func (s *BaseGunSuite) SetupSuite() {
 }
 
 func (s *BaseGunSuite) SetupTest() {
-	s.base = BaseGun{Config: DefaultBaseGunConfig()}
+	s.base = BaseGun{Config: DefaultHTTPGunConfig()}
 }
 
 func (s *BaseGunSuite) Test_BindResultTo_Panics() {
@@ -335,8 +335,9 @@ func Test_Autotag(t *testing.T) {
 }
 
 func Test_ConfigDecode(t *testing.T) {
-	var conf BaseGunConfig
+	var conf GunConfig
 	coretest.DecodeAndValidateT(t, `
+target: localhost:80
 auto-tag:
   enabled: true
   uri-elements: 3

components/guns/http/client.go

@@ -173,43 +173,6 @@ func (c *panicOnHTTP1Client) Do(req *http.Request) (*http.Response, error) {
 	return res, nil
 }
 
-func WrapClientHostResolving(client Client, cfg HTTPGunConfig, targetResolved string) Client {
-	hostname := getHostWithoutPort(cfg.Target)
-	scheme := "http"
-	if cfg.SSL {
-		scheme = "https"
-	}
-	return &httpDecoratedClient{
-		client:         client,
-		scheme:         scheme,
-		hostname:       hostname,
-		targetResolved: targetResolved,
-	}
-}
-
-type httpDecoratedClient struct {
-	client         Client
-	scheme         string
-	hostname       string
-	targetResolved string
-}
-
-func (c *httpDecoratedClient) Do(req *http.Request) (*http.Response, error) {
-	if req.Host == "" {
-		req.Host = c.hostname
-	}
-
-	if c.targetResolved != "" {
-		req.URL.Host = c.targetResolved
-	}
-	req.URL.Scheme = c.scheme
-	return c.client.Do(req)
-}
-
-func (c *httpDecoratedClient) CloseIdleConnections() {
-	c.client.CloseIdleConnections()
-}
-
 func checkHTTP2(state *tls.ConnectionState) error {
 	if state == nil {
 		return errors.New("http2: non TLS connection")