[JEWEL-1285] Update JDialogRenderer to Support Custom Window Shape

platform/jewel/ide-laf-bridge/src/main/kotlin/org/jetbrains/jewel/bridge/component/JBPopupRenderer.kt

@@ -68,6 +68,7 @@ internal object JBPopupRenderer : PopupRenderer {
         onPreviewKeyEvent: ((KeyEvent) -> Boolean)?,
         onKeyEvent: ((KeyEvent) -> Boolean)?,
         cornerSize: CornerSize,
+        windowShape: ((IntSize) -> java.awt.Shape)?,
         content: @Composable () -> Unit,
     ) {
         JBPopup(

platform/jewel/int-ui/int-ui-standalone/src/main/kotlin/org/jetbrains/jewel/intui/standalone/popup/JDialogRenderer.kt

@@ -35,6 +35,7 @@ import androidx.compose.ui.platform.LocalDensity
 import androidx.compose.ui.semantics.popup
 import androidx.compose.ui.semantics.semantics
 import androidx.compose.ui.unit.Constraints
+import androidx.compose.ui.unit.Density
 import androidx.compose.ui.unit.IntOffset
 import androidx.compose.ui.unit.IntRect
 import androidx.compose.ui.unit.IntSize
@@ -102,6 +103,7 @@ internal object JDialogRenderer : PopupRenderer {
         onPreviewKeyEvent: ((KeyEvent) -> Boolean)?,
         onKeyEvent: ((KeyEvent) -> Boolean)?,
         cornerSize: CornerSize,
+        windowShape: ((IntSize) -> java.awt.Shape)?,
         content: @Composable () -> Unit,
     ) {
         val isJBREnvironment = remember { JBR.isAvailable() && JBR.isRoundedCornersManagerSupported() }
@@ -148,6 +150,7 @@ internal object JDialogRenderer : PopupRenderer {
                 onKeyEvent = onKeyEvent,
                 cornerSize = cornerSize,
                 blendingEnabled = supportBlending,
+                windowShape = windowShape,
                 content = content,
             )
         }
@@ -164,6 +167,7 @@ private fun JPopupImpl(
     onKeyEvent: ((KeyEvent) -> Boolean)?,
     cornerSize: CornerSize,
     blendingEnabled: Boolean,
+    windowShape: ((IntSize) -> java.awt.Shape)?,
     content: @Composable () -> Unit,
 ) {
     val popupDensity = LocalDensity.current
@@ -175,6 +179,7 @@ private fun JPopupImpl(
     val currentOnKeyEvent by rememberUpdatedState(onKeyEvent)
     val currentOnPreviewKeyEvent by rememberUpdatedState(onPreviewKeyEvent)
     val currentProperties by rememberUpdatedState(properties)
+    val windowShapeState = rememberUpdatedState(windowShape)
 
     val compositionLocalContext by rememberUpdatedState(currentCompositionLocalContext)
 
@@ -242,19 +247,40 @@ private fun JPopupImpl(
                             JPopupMeasurePolicy(dialog, currentPopupPositionProvider, parentBounds) { position, size ->
                                 popupRectangle = Rectangle(position.x, position.y, size.width, size.height)
 
+                                val currentWindowShape = windowShapeState.value
+                                if (currentWindowShape != null) {
+                                    if (blendingEnabled) {
+                                        // When blending is active (via compose.interop.blending), the window is
+                                        // already in java.awt.GraphicsDevice.WindowTranslucency.PERPIXEL_TRANSLUCENT
+                                        // mode (per-pixel alpha). Calling Window.setShape() would switch it to
+                                        // PERPIXEL_TRANSPARENT (hard pixel clip), breaking antialiasing at the edges.
+                                        // Compose's own drawing + transparent background is sufficient.
+                                        return@JPopupMeasurePolicy
+                                    }
+                                    // Without blending, fall back to Window.setShape() to at least
+                                    // clip the rectangular window boundary to the balloon outline.
+                                    // Note: this uses PERPIXEL_TRANSPARENT mode, which has known
+                                    // antialiasing limitations at concave corners (e.g. arrow junction).
+                                    val logicalSize =
+                                        IntSize(
+                                            floor(size.width / popupDensity.density).toInt(),
+                                            floor(size.height / popupDensity.density).toInt(),
+                                        )
+                                    try {
+                                        dialog.shape = currentWindowShape(logicalSize)
+                                    } catch (_: UnsupportedOperationException) {
+                                        applyRoundedCorners(dialog, cornerSize, size, popupDensity)
+                                    }
+                                    return@JPopupMeasurePolicy
+                                }
+
                                 if (blendingEnabled) {
                                     // If any of the blending logic is enabled, we don't need to use JBR APIs
                                     // to set the rounded corners and fix the background.
                                     return@JPopupMeasurePolicy
                                 }
 
-                                if (cornerSize != ZeroCornerSize) {
-                                    JBR.getRoundedCornersManager()
-                                        .setRoundedCorners(
-                                            dialog,
-                                            cornerSize.toPx(size.toSize(), popupDensity) / dialog.density(),
-                                        )
-                                }
+                                applyRoundedCorners(dialog, cornerSize, size, popupDensity)
                             }
                         },
                 )
@@ -390,6 +416,11 @@ private class JPopupMeasurePolicy(
     }
 }
 
+private fun applyRoundedCorners(dialog: Window, cornerSize: CornerSize, size: IntSize, density: Density) {
+    if (cornerSize == ZeroCornerSize) return
+    JBR.getRoundedCornersManager().setRoundedCorners(dialog, cornerSize.toPx(size.toSize(), density) / dialog.density())
+}
+
 // Based on implementation from JBUIScale and ScreenUtil
 private fun IntSize.Companion.screenSize(window: Component): IntSize {
     val windowConfiguration = window.graphicsConfiguration.device.defaultConfiguration

platform/jewel/ui/api-dump.txt

@@ -573,8 +573,10 @@
 - bsf:PopupContainer(kotlin.jvm.functions.Function0,androidx.compose.ui.Alignment$Horizontal,androidx.compose.ui.Modifier,org.jetbrains.jewel.ui.component.styling.PopupContainerStyle,androidx.compose.ui.window.PopupProperties,androidx.compose.ui.window.PopupPositionProvider,kotlin.jvm.functions.Function2,androidx.compose.runtime.Composer,I,I):V
 - sf:PopupContainer(kotlin.jvm.functions.Function0,androidx.compose.ui.Alignment$Horizontal,androidx.compose.ui.Modifier,org.jetbrains.jewel.ui.component.styling.PopupContainerStyle,androidx.compose.ui.window.PopupProperties,androidx.compose.ui.window.PopupPositionProvider,kotlin.jvm.functions.Function2,kotlin.jvm.functions.Function2,androidx.compose.runtime.Composer,I,I):V
 f:org.jetbrains.jewel.ui.component.PopupKt
-- sf:Popup(androidx.compose.ui.window.PopupPositionProvider,androidx.compose.foundation.shape.CornerSize,kotlin.jvm.functions.Function0,androidx.compose.ui.window.PopupProperties,kotlin.jvm.functions.Function1,kotlin.jvm.functions.Function1,kotlin.jvm.functions.Function2,androidx.compose.runtime.Composer,I,I):V
-- sf:Popup(androidx.compose.ui.window.PopupPositionProvider,kotlin.jvm.functions.Function0,androidx.compose.ui.window.PopupProperties,kotlin.jvm.functions.Function1,kotlin.jvm.functions.Function1,kotlin.jvm.functions.Function2,androidx.compose.runtime.Composer,I,I):V
+- sf:Popup(androidx.compose.ui.window.PopupPositionProvider,androidx.compose.foundation.shape.CornerSize,kotlin.jvm.functions.Function0,androidx.compose.ui.window.PopupProperties,kotlin.jvm.functions.Function1,kotlin.jvm.functions.Function1,kotlin.jvm.functions.Function1,kotlin.jvm.functions.Function2,androidx.compose.runtime.Composer,I,I):V
+- bsf:Popup(androidx.compose.ui.window.PopupPositionProvider,androidx.compose.foundation.shape.CornerSize,kotlin.jvm.functions.Function0,androidx.compose.ui.window.PopupProperties,kotlin.jvm.functions.Function1,kotlin.jvm.functions.Function1,kotlin.jvm.functions.Function2,androidx.compose.runtime.Composer,I,I):V
+- sf:Popup(androidx.compose.ui.window.PopupPositionProvider,kotlin.jvm.functions.Function0,androidx.compose.ui.window.PopupProperties,kotlin.jvm.functions.Function1,kotlin.jvm.functions.Function1,kotlin.jvm.functions.Function1,kotlin.jvm.functions.Function2,androidx.compose.runtime.Composer,I,I):V
+- bsf:Popup(androidx.compose.ui.window.PopupPositionProvider,kotlin.jvm.functions.Function0,androidx.compose.ui.window.PopupProperties,kotlin.jvm.functions.Function1,kotlin.jvm.functions.Function1,kotlin.jvm.functions.Function2,androidx.compose.runtime.Composer,I,I):V
 - sf:getLocalPopupRenderer():androidx.compose.runtime.ProvidableCompositionLocal
 f:org.jetbrains.jewel.ui.component.PopupManager
 - sf:$stable:I
@@ -590,8 +592,10 @@
 - f:togglePopupVisibility():V
 org.jetbrains.jewel.ui.component.PopupRenderer
 - sf:Companion:org.jetbrains.jewel.ui.component.PopupRenderer$Companion
-- a:Popup(androidx.compose.ui.window.PopupPositionProvider,androidx.compose.ui.window.PopupProperties,kotlin.jvm.functions.Function0,kotlin.jvm.functions.Function1,kotlin.jvm.functions.Function1,androidx.compose.foundation.shape.CornerSize,kotlin.jvm.functions.Function2,androidx.compose.runtime.Composer,I):V
+- a:Popup(androidx.compose.ui.window.PopupPositionProvider,androidx.compose.ui.window.PopupProperties,kotlin.jvm.functions.Function0,kotlin.jvm.functions.Function1,kotlin.jvm.functions.Function1,androidx.compose.foundation.shape.CornerSize,kotlin.jvm.functions.Function1,kotlin.jvm.functions.Function2,androidx.compose.runtime.Composer,I):V
 f:org.jetbrains.jewel.ui.component.PopupRenderer$Companion
+f:org.jetbrains.jewel.ui.component.PopupRenderer$ComposeDefaultImpls
+- sf:Popup$default(androidx.compose.ui.window.PopupPositionProvider,androidx.compose.ui.window.PopupProperties,kotlin.jvm.functions.Function0,kotlin.jvm.functions.Function1,kotlin.jvm.functions.Function1,androidx.compose.foundation.shape.CornerSize,kotlin.jvm.functions.Function1,kotlin.jvm.functions.Function2,org.jetbrains.jewel.ui.component.PopupRenderer,androidx.compose.runtime.Composer,I,I):V
 f:org.jetbrains.jewel.ui.component.RadioButtonKt
 - sf:RadioButton(Z,kotlin.jvm.functions.Function0,androidx.compose.ui.Modifier,Z,org.jetbrains.jewel.ui.Outline,androidx.compose.foundation.interaction.MutableInteractionSource,org.jetbrains.jewel.ui.component.styling.RadioButtonStyle,androidx.compose.ui.text.TextStyle,androidx.compose.ui.Alignment$Vertical,androidx.compose.runtime.Composer,I,I):V
 - bsf:RadioButtonRow(java.lang.String,Z,kotlin.jvm.functions.Function0,androidx.compose.ui.Modifier,Z,org.jetbrains.jewel.ui.Outline,androidx.compose.foundation.interaction.MutableInteractionSource,org.jetbrains.jewel.ui.component.styling.RadioButtonStyle,androidx.compose.ui.text.TextStyle,androidx.compose.ui.Alignment$Vertical,androidx.compose.runtime.Composer,I,I):V