diff --git a/packages/block-editor/src/components/block-controls/use-has-block-controls.js b/packages/block-editor/src/components/block-controls/use-has-block-controls.js
new file mode 100644
index 00000000000000..f7884cc1882ed5
--- /dev/null
+++ b/packages/block-editor/src/components/block-controls/use-has-block-controls.js
@@ -0,0 +1,35 @@
+/**
+ * WordPress dependencies
+ */
+import { __experimentalUseSlotFills as useSlotFills } from '@wordpress/components';
+import warning from '@wordpress/warning';
+
+/**
+ * Internal dependencies
+ */
+import groups from './groups';
+
+export function useHasAnyBlockControls() {
+	let hasAnyBlockControls = false;
+	for ( const group in groups ) {
+		// It is safe to violate the rules of hooks here as the `groups` object
+		// is static and will not change length between renders. Do not return
+		// early as that will cause the hook to be called a different number of
+		// times between renders.
+		// eslint-disable-next-line react-hooks/rules-of-hooks
+		if ( useHasBlockControls( group ) ) {
+			hasAnyBlockControls = true;
+		}
+	}
+	return hasAnyBlockControls;
+}
+
+export function useHasBlockControls( group = 'default' ) {
+	const Slot = groups[ group ]?.Slot;
+	const fills = useSlotFills( Slot?.__unstableName );
+	if ( ! Slot ) {
+		warning( `Unknown BlockControls group "${ group }" provided.` );
+		return null;
+	}
+	return !! fills?.length;
+}
diff --git a/packages/block-editor/src/components/block-tools/block-contextual-toolbar.js b/packages/block-editor/src/components/block-tools/block-contextual-toolbar.js
index 7dcd88d1cfd10f..9667cb21d99b74 100644
--- a/packages/block-editor/src/components/block-tools/block-contextual-toolbar.js
+++ b/packages/block-editor/src/components/block-tools/block-contextual-toolbar.js
@@ -25,6 +25,7 @@ import NavigableToolbar from '../navigable-toolbar';
 import BlockToolbar from '../block-toolbar';
 import { store as blockEditorStore } from '../../store';
 import { unlock } from '../../lock-unlock';
+import { useHasAnyBlockControls } from '../block-controls/use-has-block-controls';
 
 function BlockContextualToolbar( { focusOnMount, isFixed, ...props } ) {
 	// When the toolbar is fixed it can be collapsed
@@ -34,10 +35,10 @@ function BlockContextualToolbar( { focusOnMount, isFixed, ...props } ) {
 	const isLargeViewport = useViewportMatch( 'medium' );
 	const {
 		blockType,
+		blockEditingMode,
 		hasParents,
 		showParentSelector,
 		selectedBlockClientId,
-		isContentOnly,
 	} = useSelect( ( select ) => {
 		const {
 			getBlockName,
@@ -58,9 +59,8 @@ function BlockContextualToolbar( { focusOnMount, isFixed, ...props } ) {
 			blockType:
 				_selectedBlockClientId &&
 				getBlockType( getBlockName( _selectedBlockClientId ) ),
+			blockEditingMode: getBlockEditingMode( _selectedBlockClientId ),
 			hasParents: parents.length,
-			isContentOnly:
-				getBlockEditingMode( _selectedBlockClientId ) === 'contentOnly',
 			showParentSelector:
 				parentBlockType &&
 				getBlockEditingMode( firstParentClientId ) === 'default' &&
@@ -78,10 +78,13 @@ function BlockContextualToolbar( { focusOnMount, isFixed, ...props } ) {
 		setIsCollapsed( false );
 	}, [ selectedBlockClientId ] );
 
+	const isToolbarEnabled =
+		! blockType ||
+		hasBlockSupport( blockType, '__experimentalToolbar', true );
+	const hasAnyBlockControls = useHasAnyBlockControls();
 	if (
-		isContentOnly ||
-		( blockType &&
-			! hasBlockSupport( blockType, '__experimentalToolbar', true ) )
+		! isToolbarEnabled ||
+		( blockEditingMode !== 'default' && ! hasAnyBlockControls )
 	) {
 		return null;
 	}
diff --git a/packages/block-editor/src/components/block-tools/style.scss b/packages/block-editor/src/components/block-tools/style.scss
index 7c1d305c4b6d20..87481c2bc09d49 100644
--- a/packages/block-editor/src/components/block-tools/style.scss
+++ b/packages/block-editor/src/components/block-tools/style.scss
@@ -121,10 +121,6 @@
 		}
 	}
 
-	&:has(.block-editor-block-toolbar:empty) {
-		display: none;
-	}
-
 	// Add a scrim to the right of the collapsed button.
 	&.is-collapsed::after {
 		content: "";