Permit skipping metadata for MariaDB (#1301)

Since MariaDB 10.6 (with https://jira.mariadb.org/browse/MDEV-19237), binary result-set skips sending metadata when they haven't changed. This avoids network data and parsing client-side.

Signed-off-by: rusher <[email protected]>

src/MySqlConnector/Core/CommandListPosition.cs

@@ -8,6 +8,7 @@ internal struct CommandListPosition
 	public CommandListPosition(IReadOnlyList<IMySqlCommand> commands)
 	{
 		Commands = commands;
+		PreparedStatements = null;
 		CommandIndex = 0;
 		PreparedStatementIndex = 0;
 	}
@@ -17,6 +18,11 @@ public CommandListPosition(IReadOnlyList<IMySqlCommand> commands)
 	/// </summary>
 	public IReadOnlyList<IMySqlCommand> Commands { get; }
 
+	/// <summary>
+	/// Associated prepared statements of commands
+	/// </summary>
+	public PreparedStatements? PreparedStatements;
+
 	/// <summary>
 	/// The index of the current command.
 	/// </summary>
@@ -26,4 +32,9 @@ public CommandListPosition(IReadOnlyList<IMySqlCommand> commands)
 	/// If the current command is a prepared statement, the index of the current prepared statement for that command.
 	/// </summary>
 	public int PreparedStatementIndex;
+
+	/// <summary>
+	/// Retrieve the last used prepared statement
+	/// </summary>
+	public PreparedStatement? LastUsedPreparedStatement;
 }

src/MySqlConnector/Core/PreparedStatement.cs

@@ -17,6 +17,6 @@ public PreparedStatement(int statementId, ParsedStatement statement, ColumnDefin
 
 	public int StatementId { get; }
 	public ParsedStatement Statement { get; }
-	public ColumnDefinitionPayload[]? Columns { get; }
+	public ColumnDefinitionPayload[]? Columns { get; internal set; }
 	public ColumnDefinitionPayload[]? Parameters { get; }
 }

src/MySqlConnector/Core/ResultSet.cs

@@ -116,42 +116,55 @@ public async Task ReadResultSetHeaderAsync(IOBehavior ioBehavior)
 				}
 				else
 				{
-					static int ReadColumnCount(ReadOnlySpan<byte> span)
+					var columnCountPacket = ColumnCountPayload.Create(payload.Span, Session.SupportsMetaSkip);
+					if (!columnCountPacket.MetadataFollows)
 					{
-						var reader = new ByteArrayReader(span);
-						var columnCount_ = (int) reader.ReadLengthEncodedInteger();
-						if (reader.BytesRemaining != 0)
-							throw new MySqlException("Unexpected data at end of column_count packet; see https://github.com/mysql-net/MySqlConnector/issues/324");
-						return columnCount_;
+						// reuse previous metadata
+						var preparedStatement = DataReader.CurrentPrepareStmt()!;
+						ColumnDefinitions = preparedStatement.Columns!;
+						ColumnTypes = new MySqlDbType[columnCountPacket.ColumnCount];
+						for (var column = 0; column < columnCountPacket.ColumnCount; column++)
+						{
+							ColumnTypes[column] = TypeMapper.ConvertToMySqlDbType(ColumnDefinitions[column],
+								Connection.TreatTinyAsBoolean, Connection.GuidFormat);
+						}
 					}
-					var columnCount = ReadColumnCount(payload.Span);
-
-					// reserve adequate space to hold a copy of all column definitions (but note that this can be resized below if we guess too small)
-					Utility.Resize(ref m_columnDefinitionPayloads, columnCount * 96);
+					else
+					{
+						// parse columns
+						// reserve adequate space to hold a copy of all column definitions (but note that this can be resized below if we guess too small)
+						Utility.Resize(ref m_columnDefinitionPayloads, columnCountPacket.ColumnCount * 96);
 
-					ColumnDefinitions = new ColumnDefinitionPayload[columnCount];
-					ColumnTypes = new MySqlDbType[columnCount];
+						ColumnDefinitions = new ColumnDefinitionPayload[columnCountPacket.ColumnCount];
+						ColumnTypes = new MySqlDbType[columnCountPacket.ColumnCount];
+						for (var column = 0; column < ColumnDefinitions.Length; column++)
+						{
+							payload = await Session.ReceiveReplyAsync(ioBehavior, CancellationToken.None).ConfigureAwait(false);
+							var payloadLength = payload.Span.Length;
+
+							// 'Session.ReceiveReplyAsync' reuses a shared buffer; make a copy so that the column definitions can always be safely read at any future point
+							if (m_columnDefinitionPayloadUsedBytes + payloadLength > m_columnDefinitionPayloads.Count)
+								Utility.Resize(ref m_columnDefinitionPayloads, m_columnDefinitionPayloadUsedBytes + payloadLength);
+							payload.Span.CopyTo(m_columnDefinitionPayloads.AsSpan(m_columnDefinitionPayloadUsedBytes));
+
+							var columnDefinition = ColumnDefinitionPayload.Create(new ResizableArraySegment<byte>(m_columnDefinitionPayloads, m_columnDefinitionPayloadUsedBytes, payloadLength));
+							ColumnDefinitions[column] = columnDefinition;
+							ColumnTypes[column] = TypeMapper.ConvertToMySqlDbType(columnDefinition, Connection.TreatTinyAsBoolean, Connection.GuidFormat);
+							m_columnDefinitionPayloadUsedBytes += payloadLength;
+						}
 
-					for (var column = 0; column < ColumnDefinitions.Length; column++)
-					{
-						payload = await Session.ReceiveReplyAsync(ioBehavior, CancellationToken.None).ConfigureAwait(false);
-						var payloadLength = payload.Span.Length;
-
-						// 'Session.ReceiveReplyAsync' reuses a shared buffer; make a copy so that the column definitions can always be safely read at any future point
-						if (m_columnDefinitionPayloadUsedBytes + payloadLength > m_columnDefinitionPayloads.Count)
-							Utility.Resize(ref m_columnDefinitionPayloads, m_columnDefinitionPayloadUsedBytes + payloadLength);
-						payload.Span.CopyTo(m_columnDefinitionPayloads.AsSpan(m_columnDefinitionPayloadUsedBytes));
-
-						var columnDefinition = ColumnDefinitionPayload.Create(new ResizableArraySegment<byte>(m_columnDefinitionPayloads, m_columnDefinitionPayloadUsedBytes, payloadLength));
-						ColumnDefinitions[column] = columnDefinition;
-						ColumnTypes[column] = TypeMapper.ConvertToMySqlDbType(columnDefinition, treatTinyAsBoolean: Connection.TreatTinyAsBoolean, guidFormat: Connection.GuidFormat);
-						m_columnDefinitionPayloadUsedBytes += payloadLength;
-					}
+						if (Session.SupportsMetaSkip)
+						{
+							// server support metadata skipping, but has resend them, so something has change since last prepare/execution
+							var preparedStatement = DataReader.CurrentPrepareStmt();
+							if (preparedStatement != null) preparedStatement.Columns = ColumnDefinitions;
+						}
 
-					if (!Session.SupportsDeprecateEof)
-					{
-						payload = await Session.ReceiveReplyAsync(ioBehavior, CancellationToken.None).ConfigureAwait(false);
-						EofPayload.Create(payload.Span);
+						if (!Session.SupportsDeprecateEof)
+						{
+							payload = await Session.ReceiveReplyAsync(ioBehavior, CancellationToken.None).ConfigureAwait(false);
+							EofPayload.Create(payload.Span);
+						}
 					}
 
 					if (ColumnDefinitions.Length == (Command?.OutParameters?.Count + 1) && ColumnDefinitions[0].Name == SingleCommandPayloadCreator.OutParameterSentinelColumnName)

src/MySqlConnector/Core/ServerSession.cs

@@ -63,6 +63,7 @@ public ServerSession(ILogger logger, ConnectionPool? pool, int poolGeneration, i
 	public WeakReference<MySqlConnection>? OwningConnection { get; set; }
 	public bool SupportsComMulti => m_supportsComMulti;
 	public bool SupportsDeprecateEof => m_supportsDeprecateEof;
+	public bool SupportsMetaSkip => m_supportsMetaSkip;
 	public bool SupportsQueryAttributes { get; private set; }
 	public bool SupportsSessionTrack => m_supportsSessionTrack;
 	public bool ProcAccessDenied { get; set; }
@@ -485,6 +486,7 @@ public async Task DisposeAsync(IOBehavior ioBehavior, CancellationToken cancella
 				m_supportsComMulti = (initialHandshake.ProtocolCapabilities & ProtocolCapabilities.MariaDbComMulti) != 0;
 				m_supportsConnectionAttributes = (initialHandshake.ProtocolCapabilities & ProtocolCapabilities.ConnectionAttributes) != 0;
 				m_supportsDeprecateEof = (initialHandshake.ProtocolCapabilities & ProtocolCapabilities.DeprecateEof) != 0;
+				m_supportsMetaSkip = (initialHandshake.ProtocolCapabilities & ProtocolCapabilities.MariaDbCacheMetadata) != 0;
 				SupportsQueryAttributes = (initialHandshake.ProtocolCapabilities & ProtocolCapabilities.QueryAttributes) != 0;
 				m_supportsSessionTrack = (initialHandshake.ProtocolCapabilities & ProtocolCapabilities.SessionTrack) != 0;
 				var serverSupportsSsl = (initialHandshake.ProtocolCapabilities & ProtocolCapabilities.Ssl) != 0;
@@ -519,7 +521,7 @@ public async Task DisposeAsync(IOBehavior ioBehavior, CancellationToken cancella
 					}
 				}
 
-				Log.SessionMadeConnection(m_logger, Id, ServerVersion.OriginalString, ConnectionId, m_useCompression, m_supportsConnectionAttributes, m_supportsDeprecateEof, serverSupportsSsl, m_supportsSessionTrack, m_supportsPipelining, SupportsQueryAttributes);
+				Log.SessionMadeConnection(m_logger, Id, ServerVersion.OriginalString, ConnectionId, m_useCompression, m_supportsConnectionAttributes, m_supportsDeprecateEof, m_supportsMetaSkip, serverSupportsSsl, m_supportsSessionTrack, m_supportsPipelining, SupportsQueryAttributes);
 
 				if (cs.SslMode != MySqlSslMode.None && (cs.SslMode != MySqlSslMode.Preferred || serverSupportsSsl))
 				{
@@ -1969,6 +1971,7 @@ protected override void OnStatementBegin(int index)
 	private bool m_supportsComMulti;
 	private bool m_supportsConnectionAttributes;
 	private bool m_supportsDeprecateEof;
+	private bool m_supportsMetaSkip;
 	private bool m_supportsSessionTrack;
 	private bool m_supportsPipelining;
 	private CharacterSet m_characterSet;

src/MySqlConnector/Core/SingleCommandPayloadCreator.cs

@@ -1,7 +1,6 @@
 using MySqlConnector.Logging;
 using MySqlConnector.Protocol;
 using MySqlConnector.Protocol.Serialization;
-using MySqlConnector.Utilities;
 
 namespace MySqlConnector.Core;
 
@@ -19,8 +18,8 @@ public bool WriteQueryCommand(ref CommandListPosition commandListPosition, IDict
 			return false;
 
 		var command = commandListPosition.Commands[commandListPosition.CommandIndex];
-		var preparedStatements = command.TryGetPreparedStatements();
-		if (preparedStatements is null)
+		commandListPosition.PreparedStatements = command.TryGetPreparedStatements();
+		if (commandListPosition.PreparedStatements is null)
 		{
 			Log.PreparingCommandPayload(command.Logger, command.Connection!.Session.Id, command.CommandText!);
 
@@ -44,16 +43,18 @@ public bool WriteQueryCommand(ref CommandListPosition commandListPosition, IDict
 			}
 
 			WriteQueryPayload(command, cachedProcedures, writer, appendSemicolon);
-
+			commandListPosition.LastUsedPreparedStatement = null;
 			commandListPosition.CommandIndex++;
 		}
 		else
 		{
 			writer.Write((byte) CommandKind.StatementExecute);
-			WritePreparedStatement(command, preparedStatements.Statements[commandListPosition.PreparedStatementIndex], writer);
+			commandListPosition.LastUsedPreparedStatement =
+				commandListPosition.PreparedStatements.Statements[commandListPosition.PreparedStatementIndex];
+			WritePreparedStatement(command, commandListPosition.LastUsedPreparedStatement, writer);
 
 			// advance to next prepared statement or next command
-			if (++commandListPosition.PreparedStatementIndex == preparedStatements.Statements.Count)
+			if (++commandListPosition.PreparedStatementIndex == commandListPosition.PreparedStatements.Statements.Count)
 			{
 				commandListPosition.CommandIndex++;
 				commandListPosition.PreparedStatementIndex = 0;

src/MySqlConnector/Logging/Log.cs

@@ -40,8 +40,8 @@ internal static partial class Log
 	[LoggerMessage(EventIds.AutoDetectedAurora57, LogLevel.Debug, "Session {SessionId} auto-detected Aurora 5.7 at '{HostName}'; disabling pipelining")]
 	public static partial void AutoDetectedAurora57(ILogger logger, string sessionId, string hostName);
 
-	[LoggerMessage(EventIds.SessionMadeConnection, LogLevel.Debug, "Session {SessionId} made connection; server version {ServerVersion}; connection ID {ConnectionId}; supports: compression {SupportsCompression}, attributes {SupportsAttributes}, deprecate EOF {SupportsDeprecateEof}, SSL {SupportsSsl}, session track {SupportsSessionTrack}, pipelining {SupportsPipelining}, query attributes {SupportsQueryAttributes}")]
-	public static partial void SessionMadeConnection(ILogger logger, string sessionId, string serverVersion, int connectionId, bool supportsCompression, bool supportsAttributes, bool supportsDeprecateEof, bool supportsSsl, bool supportsSessionTrack, bool supportsPipelining, bool supportsQueryAttributes);
+	[LoggerMessage(EventIds.SessionMadeConnection, LogLevel.Debug, "Session {SessionId} made connection; server version {ServerVersion}; connection ID {ConnectionId}; supports: compression {SupportsCompression}, attributes {SupportsAttributes}, deprecate EOF {SupportsDeprecateEof}, metadata skip {supportsMetaSkip}, SSL {SupportsSsl}, session track {SupportsSessionTrack}, pipelining {SupportsPipelining}, query attributes {SupportsQueryAttributes}")]
+	public static partial void SessionMadeConnection(ILogger logger, string sessionId, string serverVersion, int connectionId, bool supportsCompression, bool supportsAttributes, bool supportsDeprecateEof, bool supportsMetaSkip, bool supportsSsl, bool supportsSessionTrack, bool supportsPipelining, bool supportsQueryAttributes);
 
 	[LoggerMessage(EventIds.ServerDoesNotSupportSsl, LogLevel.Error, "Session {SessionId} requires SSL but server doesn't support it")]
 	public static partial void ServerDoesNotSupportSsl(ILogger logger, string sessionId);

src/MySqlConnector/MySqlDataReader.cs

@@ -691,6 +691,8 @@ private void VerifyNotDisposed()
 			throw new InvalidOperationException("Can't call this method when MySqlDataReader is closed.");
 	}
 
+	internal PreparedStatement? CurrentPrepareStmt() => m_commandListPosition.LastUsedPreparedStatement;
+
 	private ResultSet GetResultSet()
 	{
 		VerifyNotDisposed();

src/MySqlConnector/Protocol/Payloads/ColumnCountPayload.cs

@@ -0,0 +1,28 @@
+using MySqlConnector.Protocol.Serialization;
+
+namespace MySqlConnector.Protocol.Payloads;
+
+/// <summary>
+///     Helper class to parse Column count packet.
+///     https://mariadb.com/kb/en/result-set-packets/#column-count-packet
+///     Packet contains a columnCount, and - if capability MARIADB_CLIENT_CACHE_METADATA is set - a flag to indicate if metadata follows
+/// </summary>
+internal sealed class ColumnCountPayload
+{
+	private ColumnCountPayload(int columnCount, bool metadataFollows)
+	{
+		ColumnCount = columnCount;
+		MetadataFollows = metadataFollows;
+	}
+
+	public int ColumnCount { get; }
+	public bool MetadataFollows { get; }
+
+	public static ColumnCountPayload Create(ReadOnlySpan<byte> span, bool supportsMetaSkip)
+	{
+		var reader = new ByteArrayReader(span);
+		var columnCount = (int) reader.ReadLengthEncodedInteger();
+		var metadataFollows = supportsMetaSkip ? reader.ReadByte() == 1 : true;
+		return new ColumnCountPayload(columnCount, metadataFollows);
+	}
+}

src/MySqlConnector/Protocol/Payloads/HandshakeResponse41Payload.cs

@@ -9,25 +9,30 @@ private static ByteBufferWriter CreateCapabilitiesPayload(ProtocolCapabilities s
 	{
 		var writer = new ByteBufferWriter();
 
-		writer.Write((int) (
-			ProtocolCapabilities.Protocol41 |
-			(cs.InteractiveSession ? (serverCapabilities & ProtocolCapabilities.Interactive) : 0) |
-			(serverCapabilities & ProtocolCapabilities.LongPassword) |
-			(serverCapabilities & ProtocolCapabilities.Transactions) |
-			ProtocolCapabilities.SecureConnection |
-			(serverCapabilities & ProtocolCapabilities.PluginAuth) |
-			(serverCapabilities & ProtocolCapabilities.PluginAuthLengthEncodedClientData) |
-			ProtocolCapabilities.MultiStatements |
-			ProtocolCapabilities.MultiResults |
-			(cs.AllowLoadLocalInfile ? ProtocolCapabilities.LocalFiles : 0) |
-			(string.IsNullOrWhiteSpace(cs.Database) ? 0 : ProtocolCapabilities.ConnectWithDatabase) |
-			(cs.UseAffectedRows ? 0 : ProtocolCapabilities.FoundRows) |
-			(useCompression ? ProtocolCapabilities.Compress : ProtocolCapabilities.None) |
-			(serverCapabilities & ProtocolCapabilities.ConnectionAttributes) |
-			(serverCapabilities & ProtocolCapabilities.SessionTrack) |
-			(serverCapabilities & ProtocolCapabilities.DeprecateEof) |
-			(serverCapabilities & ProtocolCapabilities.QueryAttributes) |
-			additionalCapabilities));
+		var clientCapabilities = (ProtocolCapabilities.Protocol41 |
+		                          (cs.InteractiveSession ? ProtocolCapabilities.Interactive : 0) |
+		                          ProtocolCapabilities.LongPassword |
+		                          ProtocolCapabilities.Transactions |
+		                          ProtocolCapabilities.SecureConnection |
+		                          ProtocolCapabilities.PluginAuth |
+		                          ProtocolCapabilities.PluginAuthLengthEncodedClientData |
+		                          ProtocolCapabilities.MultiStatements |
+		                          ProtocolCapabilities.MultiResults |
+		                          (cs.AllowLoadLocalInfile ? ProtocolCapabilities.LocalFiles : 0) |
+		                          (string.IsNullOrWhiteSpace(cs.Database)
+			                          ? 0
+			                          : ProtocolCapabilities.ConnectWithDatabase) |
+		                          (cs.UseAffectedRows ? 0 : ProtocolCapabilities.FoundRows) |
+		                          (useCompression ? ProtocolCapabilities.Compress : ProtocolCapabilities.None) |
+		                          ProtocolCapabilities.ConnectionAttributes |
+		                          ProtocolCapabilities.SessionTrack |
+		                          ProtocolCapabilities.DeprecateEof |
+		                          ProtocolCapabilities.QueryAttributes |
+		                          ProtocolCapabilities.MariaDbComMulti |
+		                          ProtocolCapabilities.MariaDbCacheMetadata |
+		                          additionalCapabilities) & serverCapabilities;
+
+		writer.Write((int) clientCapabilities);
 		writer.Write(0x4000_0000);
 		writer.Write((byte) characterSet);
 
@@ -38,7 +43,7 @@ private static ByteBufferWriter CreateCapabilitiesPayload(ProtocolCapabilities s
 		if ((serverCapabilities & ProtocolCapabilities.LongPassword) == 0)
 		{
 			// MariaDB writes extended capabilities at the end of the padding
-			writer.Write((int) (((long) (serverCapabilities & ProtocolCapabilities.MariaDbComMulti)) >> 32));
+			writer.Write((int) ((ulong) clientCapabilities >> 32));
 		}
 		else
 		{

src/MySqlConnector/Protocol/ProtocolCapabilities.cs

@@ -147,4 +147,14 @@ internal enum ProtocolCapabilities : ulong
 	/// Support of array binding.
 	/// </summary>
 	MariaDbStatementBulkOperations = 0x4_0000_0000,
+
+	/// <summary>
+	/// metadata extended information
+	/// </summary>
+	MariaDbExtendedTypeInfo = 0x8_0000_0000,
+
+	/// <summary>
+	/// permit metadata caching
+	/// </summary>
+	MariaDbCacheMetadata = 0x10_0000_0000,
 }