diff --git a/android/app/src/main/java/com/paarrot/app/ImageKeyboardContentReceiver.kt b/android/app/src/main/java/com/paarrot/app/ImageKeyboardContentReceiver.kt new file mode 100644 index 0000000..43db7d4 --- /dev/null +++ b/android/app/src/main/java/com/paarrot/app/ImageKeyboardContentReceiver.kt @@ -0,0 +1,32 @@ +package com.paarrot.app + +import android.net.Uri +import android.view.View +import androidx.core.view.ContentInfoCompat +import androidx.core.view.OnReceiveContentListener + +/** + * Receives rich content from the IME (Gboard GIF), clipboard paste, and drag-drop, + * then forwards file URIs through [ShareIntentStore] so JS can attach them in-room. + */ +class ImageKeyboardContentReceiver : OnReceiveContentListener { + + override fun onReceiveContent(view: View, payload: ContentInfoCompat): ContentInfoCompat? { + val split = payload.partition { item -> item.uri != null } + val uriContent = split.first + val remaining = split.second + + if (uriContent != null) { + val uris = mutableListOf() + val clip = uriContent.clip + for (i in 0 until clip.itemCount) { + clip.getItemAt(i).uri?.let { uris.add(it) } + } + if (uris.isNotEmpty()) { + ShareIntentStore.handleContentUris(view.context, uris) + } + } + + return remaining + } +} diff --git a/android/app/src/main/java/com/paarrot/app/ImageKeyboardWebView.kt b/android/app/src/main/java/com/paarrot/app/ImageKeyboardWebView.kt new file mode 100644 index 0000000..d54ba95 --- /dev/null +++ b/android/app/src/main/java/com/paarrot/app/ImageKeyboardWebView.kt @@ -0,0 +1,38 @@ +package com.paarrot.app + +import android.content.Context +import android.util.AttributeSet +import android.view.inputmethod.EditorInfo +import android.view.inputmethod.InputConnection +import androidx.core.view.inputmethod.EditorInfoCompat +import androidx.core.view.inputmethod.InputConnectionCompat +import com.getcapacitor.CapacitorWebView + +/** + * Capacitor WebView that advertises image/video MIME types to the IME + * (Gboard GIF/sticker keyboard) and routes commit-content through + * [androidx.core.view.ViewCompat.setOnReceiveContentListener]. + */ +class ImageKeyboardWebView( + context: Context, + attrs: AttributeSet?, +) : CapacitorWebView(context, attrs) { + + override fun onCreateInputConnection(outAttrs: EditorInfo): InputConnection? { + val inputConnection = super.onCreateInputConnection(outAttrs) ?: return null + EditorInfoCompat.setContentMimeTypes(outAttrs, MIME_TYPES) + return InputConnectionCompat.createWrapper(this, inputConnection, outAttrs) + } + + companion object { + @JvmField + val MIME_TYPES: Array = arrayOf( + "image/*", + "image/png", + "image/gif", + "image/jpeg", + "image/webp", + "video/*", + ) + } +} diff --git a/android/app/src/main/java/com/paarrot/app/MainActivity.java b/android/app/src/main/java/com/paarrot/app/MainActivity.java index 23d598d..629c605 100644 --- a/android/app/src/main/java/com/paarrot/app/MainActivity.java +++ b/android/app/src/main/java/com/paarrot/app/MainActivity.java @@ -1,6 +1,8 @@ package com.paarrot.app; import android.os.Bundle; +import android.webkit.WebView; +import androidx.core.view.ViewCompat; import com.getcapacitor.BridgeActivity; public class MainActivity extends BridgeActivity { @@ -10,5 +12,18 @@ public class MainActivity extends BridgeActivity { registerPlugin(ShareHandlerPlugin.class); registerPlugin(PluginStoragePlugin.class); super.onCreate(savedInstanceState); + setupImageKeyboardSupport(); } -} \ No newline at end of file + + /** Advertise image MIME types and receive Gboard / paste / drag-drop media. */ + private void setupImageKeyboardSupport() { + if (bridge == null) return; + WebView webView = bridge.getWebView(); + if (webView == null) return; + ViewCompat.setOnReceiveContentListener( + webView, + ImageKeyboardWebView.MIME_TYPES, + new ImageKeyboardContentReceiver() + ); + } +} diff --git a/android/app/src/main/java/com/paarrot/app/ShareHandlerPlugin.kt b/android/app/src/main/java/com/paarrot/app/ShareHandlerPlugin.kt index 0ac7975..8eb7f57 100644 --- a/android/app/src/main/java/com/paarrot/app/ShareHandlerPlugin.kt +++ b/android/app/src/main/java/com/paarrot/app/ShareHandlerPlugin.kt @@ -62,7 +62,6 @@ object ShareIntentStore { if (action != Intent.ACTION_SEND && action != Intent.ACTION_SEND_MULTIPLE) return - val receivedAt = System.currentTimeMillis() val text = intent.getStringExtra(Intent.EXTRA_TEXT) val subject = intent.getStringExtra(Intent.EXTRA_SUBJECT) @@ -86,8 +85,29 @@ object ShareIntentStore { uri?.let { uris.add(it) } } + deliverShare(context, uris, text, subject) + } + + /** + * Deliver content URIs from the IME (Gboard GIF), clipboard paste, or drag-drop + * into the same share pipeline used by ACTION_SEND. + */ + fun handleContentUris(context: Context, uris: List) { + if (uris.isEmpty()) return + deliverShare(context, uris, text = null, subject = null) + } + + private fun deliverShare( + context: Context, + uris: List, + text: String?, + subject: String?, + ) { + val receivedAt = System.currentTimeMillis() val copiedFiles = uris.mapNotNull { copySharedUri(context, it, receivedAt) } + if (copiedFiles.isEmpty() && text.isNullOrBlank() && subject.isNullOrBlank()) return + val payload = SharePayload( text = text, subject = subject, diff --git a/android/app/src/main/res/layout/capacitor_bridge_layout_main.xml b/android/app/src/main/res/layout/capacitor_bridge_layout_main.xml new file mode 100644 index 0000000..becd324 --- /dev/null +++ b/android/app/src/main/res/layout/capacitor_bridge_layout_main.xml @@ -0,0 +1,13 @@ + + + + + + diff --git a/overlay/src/app/hooks/useFilePasteHandler.ts b/overlay/src/app/hooks/useFilePasteHandler.ts index 4b929a6..ef5a664 100644 --- a/overlay/src/app/hooks/useFilePasteHandler.ts +++ b/overlay/src/app/hooks/useFilePasteHandler.ts @@ -1,6 +1,6 @@ import { useCallback, ClipboardEventHandler } from 'react'; import { getDataTransferFiles } from '../utils/dom'; -import { readClipboardImage, isTauri, isLinux } from '../utils/tauri'; +import { readClipboardImage, isTauri, isLinux, isCapacitorNative } from '../utils/tauri'; const filesFromClipboardItems = (data: DataTransfer | null): File[] => { if (!data?.items) return []; @@ -50,9 +50,30 @@ const getClipboardFiles = (data: DataTransfer | null): File[] => { return filesFromHtmlImage(data); }; +/** Async Clipboard API fallback (when paste event has no file payload). */ +const filesFromClipboardApi = async (): Promise => { + if (!navigator.clipboard?.read) return []; + try { + const items = await navigator.clipboard.read(); + const files: File[] = []; + for (const item of items) { + for (const type of item.types) { + if (!type.startsWith('image/')) continue; + const blob = await item.getType(type); + const ext = type.split('/')[1]?.replace('+xml', '') || 'png'; + files.push(new File([blob], `clipboard-image.${ext}`, { type })); + } + } + return files; + } catch { + return []; + } +}; + /** * Paste handler for RoomInput. Supports clipboard images via files, DataTransferItem - * list, and HTML data-URL fallbacks (common on mobile WebViews). + * list, HTML data-URL fallbacks, and Clipboard API (common on mobile WebViews). + * On Capacitor Android, native OnReceiveContent also routes images into the share pipeline. */ export const useFilePasteHandler = (onPaste: (file: File[]) => void): ClipboardEventHandler => useCallback( @@ -72,6 +93,17 @@ export const useFilePasteHandler = (onPaste: (file: File[]) => void): ClipboardE evt.preventDefault(); evt.stopPropagation(); onPaste([clipboardImage]); + return; + } + } + + // Mobile WebView / Capacitor: paste events often omit file payloads. + if (isCapacitorNative()) { + const apiFiles = await filesFromClipboardApi(); + if (apiFiles.length > 0) { + evt.preventDefault(); + evt.stopPropagation(); + onPaste(apiFiles); } } }, diff --git a/overlay/src/app/pages/client/ClientNonUIFeatures.tsx b/overlay/src/app/pages/client/ClientNonUIFeatures.tsx index 1387ffa..b9745e9 100644 --- a/overlay/src/app/pages/client/ClientNonUIFeatures.tsx +++ b/overlay/src/app/pages/client/ClientNonUIFeatures.tsx @@ -1,5 +1,4 @@ -import { useAtomValue } from 'jotai'; -import { useAtom, useSetAtom } from 'jotai'; +import { useAtomValue, useStore } from 'jotai'; import React, { ReactNode, useCallback, useEffect, useRef, useState, Fragment } from 'react'; import { useNavigate } from 'react-router-dom'; import { RoomEvent, RoomEventHandlerMap } from 'matrix-js-sdk'; @@ -23,13 +22,12 @@ import { guessPerfectParent, } from '../../utils/room'; import { NotificationType, UnreadInfo } from '../../../types/matrix/room'; -import { getMxIdLocalPart, mxcUrlToHttp, getCanonicalAliasOrRoomId } from '../../utils/matrix'; +import { getMxIdLocalPart, mxcUrlToHttp, getCanonicalAliasOrRoomId, encryptFile } from '../../utils/matrix'; import { mDirectAtom } from '../../state/mDirectList'; import { roomToParentsAtom } from '../../state/room/roomToParents'; import { useSelectedRoom } from '../../hooks/router/useSelectedRoom'; import { useInboxNotificationsSelected } from '../../hooks/router/useInbox'; import { useMediaAuthentication } from '../../hooks/useMediaAuthentication'; -import { plainToEditorInput, toPlainText } from '../../components/editor'; import { ShareRoomPicker } from '../../features/ShareRoomPicker'; import { isTauri, @@ -46,10 +44,10 @@ import { } from '../../utils/backgroundSync'; import { TUploadItem, - roomIdToMsgDraftAtomFamily, roomIdToUploadItemsAtomFamily, } from '../../state/room/roomInputDrafts'; -import { encryptFile } from '../../utils/matrix'; +import { fulfilledPromiseSettledResult } from '../../utils/common'; +import { safeFile } from '../../utils/mimeTypes'; import { AndroidSharePayload, clearPendingAndroidShare, @@ -517,10 +515,71 @@ function TaskbarFlashStopper() { function AndroidShareIntentHandler() { const mx = useMatrixClient(); + const store = useStore(); const navigate = useNavigate(); const [pendingShare, setPendingShare] = useState(null); const mDirects = useAtomValue(mDirectAtom); const roomToParents = useAtomValue(roomToParentsAtom); + const selectedRoomId = useSelectedRoom(); + const selectedRoomIdRef = useRef(selectedRoomId); + selectedRoomIdRef.current = selectedRoomId; + + /** Queue shared files into the open room's composer upload board (Gboard / paste / share). */ + const queueShareAsDrafts = useCallback( + async (share: AndroidSharePayload, roomId: string) => { + const room = mx.getRoom(roomId); + if (!room) return false; + + if (share.files.length > 0) { + const originalFiles: File[] = []; + for (const sharedFile of share.files) { + originalFiles.push(await materializeSharedFile(sharedFile, share.receivedAt)); + } + const safeFiles = originalFiles.map(safeFile); + const fileItems: TUploadItem[] = []; + + if (room.hasEncryptionStateEvent()) { + const encryptFiles = fulfilledPromiseSettledResult( + await Promise.allSettled(safeFiles.map((f) => encryptFile(f))) + ); + encryptFiles.forEach((ef) => + fileItems.push({ + ...ef, + metadata: { markedAsSpoiler: false }, + }) + ); + } else { + safeFiles.forEach((f) => + fileItems.push({ + file: f, + originalFile: f, + encInfo: undefined, + metadata: { markedAsSpoiler: false }, + }) + ); + } + + store.set(roomIdToUploadItemsAtomFamily(roomId), { + type: 'PUT', + item: fileItems, + }); + } + + const nextParts = [share.subject?.trim(), share.text?.trim()].filter( + (value): value is string => !!value + ); + if (nextParts.length > 0) { + await mx.sendMessage(roomId, { + msgtype: 'm.text' as const, + body: nextParts.join('\n'), + }); + } + + await clearPendingAndroidShare(); + return true; + }, + [mx, store] + ); const applyPendingShare = useCallback( async (share: AndroidSharePayload, roomId: string) => { @@ -543,7 +602,7 @@ function AndroidShareIntentHandler() { if (share.files.length > 0) { for (const sharedFile of share.files) { const originalFile = await materializeSharedFile(sharedFile, share.receivedAt); - + let fileToUpload = originalFile; let encInfo: any = undefined; @@ -594,6 +653,22 @@ function AndroidShareIntentHandler() { [mx] ); + const ingestShare = useCallback( + (share: AndroidSharePayload) => { + const roomId = selectedRoomIdRef.current; + // Already in a room: attach to composer (Gboard GIF / paste / in-app share). + if (roomId && share.files.length > 0) { + queueShareAsDrafts(share, roomId).catch((err) => { + console.error('[AndroidShare] Failed to queue share into room:', err); + setPendingShare(share); + }); + return; + } + setPendingShare(share); + }, + [queueShareAsDrafts] + ); + const handlePickRoom = useCallback( (roomId: string) => { if (!pendingShare) return; @@ -602,7 +677,7 @@ function AndroidShareIntentHandler() { .then((applied) => { if (applied) { setPendingShare(null); - + // Navigate to the selected room const isDirect = mDirects.has(roomId); if (isDirect) { @@ -640,7 +715,7 @@ function AndroidShareIntentHandler() { getPendingAndroidShare() .then((share) => { if (mounted && share) { - setPendingShare(share); + ingestShare(share); } }) .catch((err) => { @@ -649,7 +724,7 @@ function AndroidShareIntentHandler() { listenForAndroidShares((share) => { if (mounted) { - setPendingShare(share); + ingestShare(share); } }) .then((handle) => { @@ -663,7 +738,7 @@ function AndroidShareIntentHandler() { mounted = false; void listenerHandle?.remove(); }; - }, []); + }, [ingestShare]); if (!pendingShare) return null;