Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use custom logger to downgrade canceled context errors to warnings #2936

Draft
wants to merge 7 commits into
base: main
Choose a base branch
from
+700 −575
Draft

Use custom logger to downgrade canceled context errors to warnings #2936

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a18ca59
add custom logger
orouz Jan 20, 2025
e30b256
clog changes
orouz Jan 21, 2025
f75f241
log changes
orouz Jan 21, 2025
12c1716
cleanup
orouz Jan 21, 2025
525a043
headers
orouz Jan 21, 2025
336f180
formatting
orouz Jan 21, 2025
e8a9105
apply mockery ci patch
orouz Jan 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions internal/beater/beater.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,15 @@ import (

"github.com/elastic/beats/v7/libbeat/beat"
agentconfig "github.com/elastic/elastic-agent-libs/config"
"github.com/elastic/elastic-agent-libs/logp"

"github.com/elastic/cloudbeat/internal/config"
"github.com/elastic/cloudbeat/internal/flavors"
"github.com/elastic/cloudbeat/internal/launcher"
"github.com/elastic/cloudbeat/internal/resources/utils/clog"
)

func New(b *beat.Beat, cfg *agentconfig.C) (beat.Beater, error) {
log := logp.NewLogger("launcher")
log := clog.NewLogger("launcher")
reloader := launcher.NewListener(log)
validator := &validator{}

Expand Down
7 changes: 4 additions & 3 deletions internal/dataprovider/providers/k8s/client_provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,20 +21,21 @@ import (
"os"

"github.com/elastic/elastic-agent-autodiscover/kubernetes"
"github.com/elastic/elastic-agent-libs/logp"
"github.com/go-logr/zapr"
"go.uber.org/zap"
k8s "k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"

"github.com/elastic/cloudbeat/internal/resources/utils/clog"
)

type ClientGetterAPI interface {
GetClient(log *logp.Logger, kubeConfig string, options kubernetes.KubeClientOptions) (k8s.Interface, error)
GetClient(log *clog.Logger, kubeConfig string, options kubernetes.KubeClientOptions) (k8s.Interface, error)
}

type ClientGetter struct{}

func (ClientGetter) GetClient(log *logp.Logger, kubeConfig string, options kubernetes.KubeClientOptions) (k8s.Interface, error) {
func (ClientGetter) GetClient(log *clog.Logger, kubeConfig string, options kubernetes.KubeClientOptions) (k8s.Interface, error) {
// Prevent klog from writing anything other than errors to stderr
if replacementLogger, err := zap.NewProduction(); err != nil {
replacementLogger = replacementLogger.WithOptions(zap.IncreaseLevel(zap.ErrorLevel))
Expand Down
18 changes: 9 additions & 9 deletions internal/dataprovider/providers/k8s/mock_client_getter_api.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 4 additions & 2 deletions internal/evaluator/logger.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,12 @@ import (
"github.com/open-policy-agent/opa/v1/logging"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"

"github.com/elastic/cloudbeat/internal/resources/utils/clog"
)

type logger struct {
log *logp.Logger
log *clog.Logger
lvl zap.AtomicLevel
}

Expand Down Expand Up @@ -84,7 +86,7 @@ func mapToArray(m map[string]any) []any {

func newLogger() logging.Logger {
lvl := zap.NewAtomicLevelAt(logp.GetLevel())
log := logp.NewLogger("opa").WithOptions(
log := clog.NewLogger("opa").WithOptions(
zap.IncreaseLevel(lvl),
zap.AddCallerSkip(1),
)
Expand Down
6 changes: 3 additions & 3 deletions internal/evaluator/opa.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,20 +23,20 @@ import (
"fmt"
"time"

"github.com/elastic/elastic-agent-libs/logp"
"github.com/mitchellh/mapstructure"
"github.com/open-policy-agent/opa/v1/plugins"
"github.com/open-policy-agent/opa/v1/sdk"

"github.com/elastic/cloudbeat/internal/config"
dlogger "github.com/elastic/cloudbeat/internal/evaluator/debug_logger"
"github.com/elastic/cloudbeat/internal/resources/fetching"
"github.com/elastic/cloudbeat/internal/resources/utils/clog"
)

var now = func() time.Time { return time.Now().UTC() }

type OpaEvaluator struct {
log *logp.Logger
log *clog.Logger
opa *sdk.OPA
benchmark string
}
Expand All @@ -63,7 +63,7 @@ var logPlugin = `
"%s": {}
}`

func NewOpaEvaluator(ctx context.Context, log *logp.Logger, cfg *config.Config) (*OpaEvaluator, error) {
func NewOpaEvaluator(ctx context.Context, log *clog.Logger, cfg *config.Config) (*OpaEvaluator, error) {
// provide the OPA configuration which specifies
// fetching policy bundle and logging decisions locally to the console

Expand Down
3 changes: 2 additions & 1 deletion internal/evaluator/opa_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
"github.com/elastic/cloudbeat/internal/config"
"github.com/elastic/cloudbeat/internal/resources/fetching"
"github.com/elastic/cloudbeat/internal/resources/fetching/cycle"
"github.com/elastic/cloudbeat/internal/resources/utils/clog"
"github.com/elastic/cloudbeat/internal/resources/utils/testhelper"
)

Expand All @@ -56,7 +57,7 @@ func (d *DummyResource) GetElasticCommonData() (map[string]any, error) {

type OpaTestSuite struct {
suite.Suite
log *logp.Logger
log *clog.Logger
}

func TestOpaTestSuite(t *testing.T) {
Expand Down
4 changes: 2 additions & 2 deletions internal/flavors/asset_inventory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,11 @@ import (

"github.com/elastic/beats/v7/libbeat/beat"
agentconfig "github.com/elastic/elastic-agent-libs/config"
"github.com/elastic/elastic-agent-libs/logp"

"github.com/elastic/cloudbeat/internal/config"
"github.com/elastic/cloudbeat/internal/flavors/assetinventory"
"github.com/elastic/cloudbeat/internal/inventory"
"github.com/elastic/cloudbeat/internal/resources/utils/clog"
)

type assetInventory struct {
Expand All @@ -44,7 +44,7 @@ func NewAssetInventory(b *beat.Beat, agentConfig *agentconfig.C) (beat.Beater, e
}

func newAssetInventoryFromCfg(b *beat.Beat, cfg *config.Config) (*assetInventory, error) {
logger := logp.NewLogger("asset_inventory")
logger := clog.NewLogger("asset_inventory")
ctx, cancel := context.WithCancel(context.Background())

beatClient, err := NewClient(b.Publisher, cfg.Processors)
Expand Down
6 changes: 3 additions & 3 deletions internal/flavors/assetinventory/strategy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
"time"

"github.com/elastic/beats/v7/libbeat/beat"
"github.com/elastic/elastic-agent-libs/logp"

"github.com/elastic/cloudbeat/internal/config"
"github.com/elastic/cloudbeat/internal/inventory"
Expand All @@ -35,14 +34,15 @@ import (
gcp_auth "github.com/elastic/cloudbeat/internal/resources/providers/gcplib/auth"
gcp_inventory "github.com/elastic/cloudbeat/internal/resources/providers/gcplib/inventory"
"github.com/elastic/cloudbeat/internal/resources/providers/msgraph"
"github.com/elastic/cloudbeat/internal/resources/utils/clog"
)

type Strategy interface {
NewAssetInventory(ctx context.Context, client beat.Client) (inventory.AssetInventory, error)
}

type strategy struct {
logger *logp.Logger
logger *clog.Logger
cfg *config.Config
}

Expand Down Expand Up @@ -110,7 +110,7 @@ func (s *strategy) initGcpFetchers(ctx context.Context) ([]inventory.AssetFetche
return gcpfetcher.New(s.logger, provider), nil
}

func GetStrategy(logger *logp.Logger, cfg *config.Config) Strategy {
func GetStrategy(logger *clog.Logger, cfg *config.Config) Strategy {
return &strategy{
logger: logger,
cfg: cfg,
Expand Down
4 changes: 2 additions & 2 deletions internal/flavors/assetinventory/strategy_aws.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,12 @@ import (
"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/sts"
"github.com/elastic/elastic-agent-libs/logp"

"github.com/elastic/cloudbeat/internal/config"
"github.com/elastic/cloudbeat/internal/inventory"
"github.com/elastic/cloudbeat/internal/inventory/awsfetcher"
"github.com/elastic/cloudbeat/internal/resources/providers/awslib"
"github.com/elastic/cloudbeat/internal/resources/utils/clog"
"github.com/elastic/cloudbeat/internal/resources/utils/pointers"
)

Expand Down Expand Up @@ -88,7 +88,7 @@ func (s *strategy) initAwsFetchers(ctx context.Context) ([]inventory.AssetFetche
return fetchers, nil
}

func tryListingBuckets(ctx context.Context, log *logp.Logger, roleConfig awssdk.Config) bool {
func tryListingBuckets(ctx context.Context, log *clog.Logger, roleConfig awssdk.Config) bool {
s3Client := s3.NewFromConfig(roleConfig)
_, err := s3Client.ListBuckets(ctx, &s3.ListBucketsInput{MaxBuckets: pointers.Ref(int32(1))})
if err == nil {
Expand Down
4 changes: 2 additions & 2 deletions internal/flavors/assetinventory/strategy_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,12 @@ import (
"time"

"github.com/elastic/beats/v7/x-pack/libbeat/common/aws"
"github.com/elastic/elastic-agent-libs/logp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/elastic/cloudbeat/internal/config"
"github.com/elastic/cloudbeat/internal/inventory"
"github.com/elastic/cloudbeat/internal/resources/utils/clog"
)

func TestStrategyPicks(t *testing.T) {
Expand Down Expand Up @@ -113,7 +113,7 @@ func TestStrategyPicks(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
s := strategy{
logger: logp.NewLogger("strategy_test"),
logger: clog.NewLogger("strategy_test"),
cfg: tc.cfg,
}
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
Expand Down
6 changes: 3 additions & 3 deletions internal/flavors/benchmark/aws.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (
"fmt"

awssdk "github.com/aws/aws-sdk-go-v2/aws"
"github.com/elastic/elastic-agent-libs/logp"

"github.com/elastic/cloudbeat/internal/config"
"github.com/elastic/cloudbeat/internal/dataprovider"
Expand All @@ -33,6 +32,7 @@ import (
"github.com/elastic/cloudbeat/internal/resources/fetching/preset"
"github.com/elastic/cloudbeat/internal/resources/fetching/registry"
"github.com/elastic/cloudbeat/internal/resources/providers/awslib"
"github.com/elastic/cloudbeat/internal/resources/utils/clog"
)

const resourceChBufferSize = 10000
Expand All @@ -41,7 +41,7 @@ type AWS struct {
IdentityProvider awslib.IdentityProviderGetter
}

func (a *AWS) NewBenchmark(ctx context.Context, log *logp.Logger, cfg *config.Config) (builder.Benchmark, error) {
func (a *AWS) NewBenchmark(ctx context.Context, log *clog.Logger, cfg *config.Config) (builder.Benchmark, error) {
resourceCh := make(chan fetching.ResourceInfo, resourceChBufferSize)
reg, bdp, _, err := a.initialize(ctx, log, cfg, resourceCh)
if err != nil {
Expand All @@ -54,7 +54,7 @@ func (a *AWS) NewBenchmark(ctx context.Context, log *logp.Logger, cfg *config.Co
}

//revive:disable-next-line:function-result-limit
func (a *AWS) initialize(ctx context.Context, log *logp.Logger, cfg *config.Config, ch chan fetching.ResourceInfo) (registry.Registry, dataprovider.CommonDataProvider, dataprovider.IdProvider, error) {
func (a *AWS) initialize(ctx context.Context, log *clog.Logger, cfg *config.Config, ch chan fetching.ResourceInfo) (registry.Registry, dataprovider.CommonDataProvider, dataprovider.IdProvider, error) {
if err := a.checkDependencies(); err != nil {
return nil, nil, nil, err
}
Expand Down
10 changes: 5 additions & 5 deletions internal/flavors/benchmark/aws_org.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import (
awssdk "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
"github.com/aws/aws-sdk-go-v2/service/sts"
"github.com/elastic/elastic-agent-libs/logp"

"github.com/elastic/cloudbeat/internal/config"
"github.com/elastic/cloudbeat/internal/dataprovider"
Expand All @@ -36,6 +35,7 @@ import (
"github.com/elastic/cloudbeat/internal/resources/fetching/registry"
"github.com/elastic/cloudbeat/internal/resources/providers/awslib"
"github.com/elastic/cloudbeat/internal/resources/providers/awslib/iam"
"github.com/elastic/cloudbeat/internal/resources/utils/clog"
"github.com/elastic/cloudbeat/internal/resources/utils/pointers"
)

Expand All @@ -52,7 +52,7 @@ type AWSOrg struct {
AccountProvider awslib.AccountProviderAPI
}

func (a *AWSOrg) NewBenchmark(ctx context.Context, log *logp.Logger, cfg *config.Config) (builder.Benchmark, error) {
func (a *AWSOrg) NewBenchmark(ctx context.Context, log *clog.Logger, cfg *config.Config) (builder.Benchmark, error) {
resourceCh := make(chan fetching.ResourceInfo, resourceChBufferSize)
reg, bdp, _, err := a.initialize(ctx, log, cfg, resourceCh)
if err != nil {
Expand All @@ -65,7 +65,7 @@ func (a *AWSOrg) NewBenchmark(ctx context.Context, log *logp.Logger, cfg *config
}

//revive:disable-next-line:function-result-limit
func (a *AWSOrg) initialize(ctx context.Context, log *logp.Logger, cfg *config.Config, ch chan fetching.ResourceInfo) (registry.Registry, dataprovider.CommonDataProvider, dataprovider.IdProvider, error) {
func (a *AWSOrg) initialize(ctx context.Context, log *clog.Logger, cfg *config.Config, ch chan fetching.ResourceInfo) (registry.Registry, dataprovider.CommonDataProvider, dataprovider.IdProvider, error) {
if err := a.checkDependencies(); err != nil {
return nil, nil, nil, err
}
Expand Down Expand Up @@ -112,7 +112,7 @@ func (a *AWSOrg) initialize(ctx context.Context, log *logp.Logger, cfg *config.C
return reg, cloud.NewDataProvider(cloud.WithAccount(*awsIdentity)), nil, nil
}

func (a *AWSOrg) getAwsAccounts(ctx context.Context, log *logp.Logger, initialCfg awssdk.Config, rootIdentity *cloud.Identity) ([]preset.AwsAccount, error) {
func (a *AWSOrg) getAwsAccounts(ctx context.Context, log *clog.Logger, initialCfg awssdk.Config, rootIdentity *cloud.Identity) ([]preset.AwsAccount, error) {
rootCfg := assumeRole(
sts.NewFromConfig(initialCfg),
initialCfg,
Expand Down Expand Up @@ -165,7 +165,7 @@ func (a *AWSOrg) getAwsAccounts(ctx context.Context, log *logp.Logger, initialCf

// pickManagementAccountRole selects role used to fetch resources from the
// Management Account (and decides if they should be fetched at all).
func (a *AWSOrg) pickManagementAccountRole(ctx context.Context, log *logp.Logger, stsClient stscreds.AssumeRoleAPIClient, rootCfg awssdk.Config, identity cloud.Identity) (awssdk.Config, error) {
func (a *AWSOrg) pickManagementAccountRole(ctx context.Context, log *clog.Logger, stsClient stscreds.AssumeRoleAPIClient, rootCfg awssdk.Config, identity cloud.Identity) (awssdk.Config, error) {
// We will check for a tag on 'cloudbeat-root' role. If it is missing, we
// will try to be backward compatible and use the "cloudbeat-root" role to
// scan the Management Account. In previous CF templates, "cloudbeat-root"
Expand Down
Loading
Loading