-
Notifications
You must be signed in to change notification settings - Fork 2.1k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Fix for FullStatus gRPC connection pooling #15520
Closed
GuptaManan100
wants to merge
14
commits into
vitessio:main
from
planetscale:full-status-connection-pooling
Closed
Changes from 5 commits
Commits
Show all changes
14 commits
Select commit
Hold shift + click to select a range
92c3641
test: add failing test for full status connection pooling
GuptaManan100 6b09b84
TabletManagetClient: dialPool invalidator
shlomi-noach bb78973
empty commit to kick CI
shlomi-noach 13f5824
Merge branch 'main' into full-status-connection-pooling
shlomi-noach 395a07e
fix ExecuteMultiFetchAsDba
shlomi-noach 853517f
test: add some comments
GuptaManan100 3b6acfd
close all clients for address
shlomi-noach 284e757
Merge branch 'full-status-connection-pooling' of github.com:planetsca…
shlomi-noach 113d3f1
shorter form
shlomi-noach 2ff52e2
Do not close connection if error is SQLErr
shlomi-noach 62ce4ed
do not close connections on error
shlomi-noach 80a8a1b
only invalidating cache in FullStatus and CheckThrottler
shlomi-noach b34e2c2
do close when invalidating
shlomi-noach 7da6c01
only close the specific client
shlomi-noach File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -25,6 +25,7 @@ import ( | |
|
||
"github.com/spf13/pflag" | ||
"google.golang.org/grpc" | ||
"google.golang.org/grpc/status" | ||
|
||
"vitess.io/vitess/go/netutil" | ||
"vitess.io/vitess/go/vt/callerid" | ||
|
@@ -108,7 +109,7 @@ type dialer interface { | |
} | ||
|
||
type poolDialer interface { | ||
dialPool(ctx context.Context, tablet *topodatapb.Tablet) (tabletmanagerservicepb.TabletManagerClient, error) | ||
dialPool(ctx context.Context, tablet *topodatapb.Tablet) (c tabletmanagerservicepb.TabletManagerClient, invalidator func(error), err error) | ||
} | ||
|
||
// Client implements tmclient.TabletManagerClient. | ||
|
@@ -152,11 +153,11 @@ func (client *grpcClient) dial(ctx context.Context, tablet *topodatapb.Tablet) ( | |
return tabletmanagerservicepb.NewTabletManagerClient(cc), cc, nil | ||
} | ||
|
||
func (client *grpcClient) dialPool(ctx context.Context, tablet *topodatapb.Tablet) (tabletmanagerservicepb.TabletManagerClient, error) { | ||
func (client *grpcClient) dialPool(ctx context.Context, tablet *topodatapb.Tablet) (tabletmanagerservicepb.TabletManagerClient, func(error) /* map invalidator */, error) { | ||
addr := netutil.JoinHostPort(tablet.Hostname, int32(tablet.PortMap["grpc"])) | ||
opt, err := grpcclient.SecureDialOption(cert, key, ca, crl, name) | ||
if err != nil { | ||
return nil, err | ||
return nil, nil, err | ||
} | ||
|
||
client.mu.Lock() | ||
|
@@ -172,7 +173,7 @@ func (client *grpcClient) dialPool(ctx context.Context, tablet *topodatapb.Table | |
for i := 0; i < cap(c); i++ { | ||
cc, err := grpcclient.Dial(addr, grpcclient.FailFast(false), opt) | ||
if err != nil { | ||
return nil, err | ||
return nil, nil, err | ||
} | ||
c <- &tmc{ | ||
cc: cc, | ||
|
@@ -185,7 +186,19 @@ func (client *grpcClient) dialPool(ctx context.Context, tablet *topodatapb.Table | |
|
||
result := <-c | ||
c <- result | ||
return result.client, nil | ||
invalidator := func(err error) { | ||
if err == nil { | ||
return | ||
} | ||
if _, ok := status.FromError(err); !ok { | ||
// Not a gRPC error | ||
return | ||
} | ||
client.mu.Lock() | ||
defer client.mu.Unlock() | ||
delete(client.rpcClientMap, addr) | ||
GuptaManan100 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
} | ||
return result.client, invalidator, nil | ||
} | ||
|
||
// Close is part of the tmclient.TabletManagerClient interface. | ||
|
@@ -470,9 +483,10 @@ func (client *Client) ExecuteQuery(ctx context.Context, tablet *topodatapb.Table | |
func (client *Client) ExecuteFetchAsDba(ctx context.Context, tablet *topodatapb.Tablet, usePool bool, req *tabletmanagerdatapb.ExecuteFetchAsDbaRequest) (*querypb.QueryResult, error) { | ||
var c tabletmanagerservicepb.TabletManagerClient | ||
var err error | ||
var invalidator func(error) | ||
if usePool { | ||
if poolDialer, ok := client.dialer.(poolDialer); ok { | ||
c, err = poolDialer.dialPool(ctx, tablet) | ||
c, invalidator, err = poolDialer.dialPool(ctx, tablet) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
@@ -497,6 +511,9 @@ func (client *Client) ExecuteFetchAsDba(ctx context.Context, tablet *topodatapb. | |
DisableForeignKeyChecks: req.DisableForeignKeyChecks, | ||
}) | ||
if err != nil { | ||
if invalidator != nil { | ||
invalidator(err) | ||
} | ||
return nil, err | ||
} | ||
return response.Result, nil | ||
|
@@ -506,9 +523,10 @@ func (client *Client) ExecuteFetchAsDba(ctx context.Context, tablet *topodatapb. | |
func (client *Client) ExecuteMultiFetchAsDba(ctx context.Context, tablet *topodatapb.Tablet, usePool bool, req *tabletmanagerdatapb.ExecuteMultiFetchAsDbaRequest) ([]*querypb.QueryResult, error) { | ||
var c tabletmanagerservicepb.TabletManagerClient | ||
var err error | ||
var invalidator func(error) | ||
if usePool { | ||
if poolDialer, ok := client.dialer.(poolDialer); ok { | ||
c, err = poolDialer.dialPool(ctx, tablet) | ||
c, invalidator, err = poolDialer.dialPool(ctx, tablet) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
@@ -533,6 +551,9 @@ func (client *Client) ExecuteMultiFetchAsDba(ctx context.Context, tablet *topoda | |
DisableForeignKeyChecks: req.DisableForeignKeyChecks, | ||
}) | ||
if err != nil { | ||
if invalidator != nil { | ||
invalidator(err) | ||
} | ||
return nil, err | ||
} | ||
return response.Results, err | ||
|
@@ -562,9 +583,10 @@ func (client *Client) ExecuteFetchAsAllPrivs(ctx context.Context, tablet *topoda | |
func (client *Client) ExecuteFetchAsApp(ctx context.Context, tablet *topodatapb.Tablet, usePool bool, req *tabletmanagerdatapb.ExecuteFetchAsAppRequest) (*querypb.QueryResult, error) { | ||
var c tabletmanagerservicepb.TabletManagerClient | ||
var err error | ||
var invalidator func(error) | ||
if usePool { | ||
if poolDialer, ok := client.dialer.(poolDialer); ok { | ||
c, err = poolDialer.dialPool(ctx, tablet) | ||
c, invalidator, err = poolDialer.dialPool(ctx, tablet) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
@@ -582,6 +604,9 @@ func (client *Client) ExecuteFetchAsApp(ctx context.Context, tablet *topodatapb. | |
|
||
response, err := c.ExecuteFetchAsApp(ctx, req) | ||
if err != nil { | ||
if invalidator != nil { | ||
invalidator(err) | ||
} | ||
return nil, err | ||
} | ||
return response.Result, nil | ||
|
@@ -612,8 +637,9 @@ func (client *Client) ReplicationStatus(ctx context.Context, tablet *topodatapb. | |
func (client *Client) FullStatus(ctx context.Context, tablet *topodatapb.Tablet) (*replicationdatapb.FullStatus, error) { | ||
var c tabletmanagerservicepb.TabletManagerClient | ||
var err error | ||
var invalidator func(error) | ||
if poolDialer, ok := client.dialer.(poolDialer); ok { | ||
c, err = poolDialer.dialPool(ctx, tablet) | ||
c, invalidator, err = poolDialer.dialPool(ctx, tablet) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
@@ -630,6 +656,9 @@ func (client *Client) FullStatus(ctx context.Context, tablet *topodatapb.Tablet) | |
|
||
response, err := c.FullStatus(ctx, &tabletmanagerdatapb.FullStatusRequest{}) | ||
if err != nil { | ||
if invalidator != nil { | ||
invalidator(err) | ||
} | ||
return nil, err | ||
} | ||
return response.Status, nil | ||
|
@@ -1102,8 +1131,9 @@ func (client *Client) Backup(ctx context.Context, tablet *topodatapb.Tablet, req | |
func (client *Client) CheckThrottler(ctx context.Context, tablet *topodatapb.Tablet, req *tabletmanagerdatapb.CheckThrottlerRequest) (*tabletmanagerdatapb.CheckThrottlerResponse, error) { | ||
var c tabletmanagerservicepb.TabletManagerClient | ||
var err error | ||
var invalidator func(error) | ||
if poolDialer, ok := client.dialer.(poolDialer); ok { | ||
c, err = poolDialer.dialPool(ctx, tablet) | ||
c, invalidator, err = poolDialer.dialPool(ctx, tablet) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
@@ -1120,6 +1150,9 @@ func (client *Client) CheckThrottler(ctx context.Context, tablet *topodatapb.Tab | |
|
||
response, err := c.CheckThrottler(ctx, req) | ||
if err != nil { | ||
if invalidator != nil { | ||
invalidator(err) | ||
} | ||
return nil, err | ||
} | ||
Comment on lines
1140
to
1145
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: can remove passing |
||
return response, nil | ||
|
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@harshit-gangal Would some error codes from MySQL also pass this check and end up causing us to invalidate the grpc connection?