perf: improve performance of clipboard history

This commit completely revamps the clipboard history feature to resolve major performance bottlenecks. The backend now uses a more intelligent data model, storing previews and content size in the database to allow for lazy-loading of large clipboard items. The frontend is updated to use paginated, infinite-scrolling lists, and it fetches full item content on-demand, which makes the initial load instantaneous and keeps the UI responsive even with a very large history.
This commit is contained in:
ByteAtATime 2025-06-22 22:14:13 -07:00
parent 9256457f74
commit 73c1e43f76
No known key found for this signature in database
3 changed files with 310 additions and 145 deletions

View file

@ -14,6 +14,8 @@ use tauri::{AppHandle, Manager};
const KEYRING_SERVICE: &str = "dev.byteatatime.raycast";
const KEYRING_USERNAME: &str = "clipboard_history_key";
const INLINE_CONTENT_THRESHOLD_BYTES: i64 = 10_000; // 10 KB
const PREVIEW_LENGTH_CHARS: usize = 500;
static COLOR_REGEX: Lazy<Regex> =
Lazy::new(|| Regex::new(r"^#([A-Fa-f0-9]{6}|[A-Fa-f0-9]{3})$").unwrap());
@ -25,7 +27,9 @@ pub struct ClipboardItem {
id: i64,
hash: String,
content_type: ContentType,
content_value: String,
content_value: Option<String>,
preview: Option<String>,
content_size_bytes: i64,
source_app_name: Option<String>,
first_copied_at: DateTime<Utc>,
last_copied_at: DateTime<Utc>,
@ -146,6 +150,8 @@ impl ClipboardHistoryManager {
hash TEXT UNIQUE NOT NULL,
content_type TEXT NOT NULL,
encrypted_content TEXT NOT NULL,
encrypted_preview TEXT,
content_size_bytes INTEGER,
source_app_name TEXT,
first_copied_at INTEGER NOT NULL,
last_copied_at INTEGER NOT NULL,
@ -179,59 +185,110 @@ impl ClipboardHistoryManager {
params![now.timestamp(), &hash],
)?;
} else {
let content_size_bytes = content_value.len() as i64;
let mut preview_text = content_value.chars().take(PREVIEW_LENGTH_CHARS).collect::<String>();
if content_value.chars().count() > PREVIEW_LENGTH_CHARS {
preview_text.push_str("...");
}
let encrypted_preview = encrypt(&preview_text, &self.key)?;
let encrypted_content = encrypt(&content_value, &self.key)?;
db.execute(
"INSERT INTO clipboard_history (hash, content_type, encrypted_content, source_app_name, first_copied_at, last_copied_at)
VALUES (?, ?, ?, ?, ?, ?)",
params![hash, content_type.as_str(), encrypted_content, source_app_name, now.timestamp(), now.timestamp()],
"INSERT INTO clipboard_history (hash, content_type, encrypted_content, encrypted_preview, content_size_bytes, source_app_name, first_copied_at, last_copied_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
params![hash, content_type.as_str(), encrypted_content, encrypted_preview, content_size_bytes, source_app_name, now.timestamp(), now.timestamp()],
)?;
}
Ok(())
}
fn get_items(&self, filter: String, limit: u32) -> Result<Vec<ClipboardItem>, AppError> {
fn get_items(
&self,
filter: String,
search_term: Option<String>,
limit: u32,
offset: u32,
) -> Result<Vec<ClipboardItem>, AppError> {
let db = self.db.lock().unwrap();
let mut query = "SELECT id, hash, content_type, encrypted_content, source_app_name, first_copied_at, last_copied_at, times_copied, is_pinned FROM clipboard_history".to_string();
let mut query = "SELECT id, hash, content_type, source_app_name, first_copied_at, last_copied_at, times_copied, is_pinned, content_size_bytes, encrypted_preview, CASE WHEN content_size_bytes <= ? THEN encrypted_content ELSE NULL END as conditional_encrypted_content FROM clipboard_history".to_string();
let mut where_clauses: Vec<String> = Vec::new();
let mut params_vec: Vec<Box<dyn rusqlite::ToSql>> = vec![Box::new(INLINE_CONTENT_THRESHOLD_BYTES)];
match filter.as_str() {
"all" => {}
"pinned" => query.push_str(" WHERE is_pinned = 1"),
"text" => query.push_str(" WHERE content_type = 'text'"),
"image" => query.push_str(" WHERE content_type = 'image'"),
"link" => query.push_str(" WHERE content_type = 'link'"),
"color" => query.push_str(" WHERE content_type = 'color'"),
"pinned" => where_clauses.push("is_pinned = 1".to_string()),
"text" => where_clauses.push("content_type = 'text'".to_string()),
"image" => where_clauses.push("content_type = 'image'".to_string()),
"link" => where_clauses.push("content_type = 'link'".to_string()),
"color" => where_clauses.push("content_type = 'color'".to_string()),
_ => {}
}
query.push_str(" ORDER BY last_copied_at DESC LIMIT ?");
if !where_clauses.is_empty() {
query.push_str(" WHERE ");
query.push_str(&where_clauses.join(" AND "));
}
query.push_str(" ORDER BY last_copied_at DESC LIMIT ? OFFSET ?");
params_vec.push(Box::new(limit));
params_vec.push(Box::new(offset));
let params_ref: Vec<&dyn rusqlite::ToSql> = params_vec.iter().map(|b| b.as_ref()).collect();
let mut stmt = db.prepare(&query)?;
let items = stmt.query_map(params![limit], |row| {
let encrypted_content: String = row.get(3)?;
let first_ts: i64 = row.get(5)?;
let last_ts: i64 = row.get(6)?;
let items_iter = stmt.query_map(&params_ref[..], |row| {
let conditional_encrypted_content: Option<String> = row.get(10)?;
let content_value = conditional_encrypted_content
.and_then(|cec| decrypt(&cec, &self.key).ok());
let encrypted_preview: Option<String> = row.get(9)?;
let preview = encrypted_preview.and_then(|ep| decrypt(&ep, &self.key).ok());
let first_ts: i64 = row.get(4)?;
let last_ts: i64 = row.get(5)?;
Ok(ClipboardItem {
id: row.get(0)?,
hash: row.get(1)?,
content_type: ContentType::from_str(&row.get::<_, String>(2)?)
.unwrap_or(ContentType::Text),
content_value: decrypt(&encrypted_content, &self.key).unwrap_or_default(),
source_app_name: row.get(4)?,
first_copied_at: DateTime::from_naive_utc_and_offset(
NaiveDateTime::from_timestamp_opt(first_ts, 0).unwrap_or_default(),
Utc,
),
last_copied_at: DateTime::from_naive_utc_and_offset(
NaiveDateTime::from_timestamp_opt(last_ts, 0).unwrap_or_default(),
Utc,
),
times_copied: row.get(7)?,
is_pinned: row.get::<_, i32>(8)? == 1,
content_type: ContentType::from_str(&row.get::<_, String>(2)?).unwrap_or(ContentType::Text),
content_value,
preview,
content_size_bytes: row.get(8)?,
source_app_name: row.get(3)?,
first_copied_at: DateTime::from_naive_utc_and_offset(NaiveDateTime::from_timestamp_opt(first_ts, 0).unwrap_or_default(), Utc),
last_copied_at: DateTime::from_naive_utc_and_offset(NaiveDateTime::from_timestamp_opt(last_ts, 0).unwrap_or_default(), Utc),
times_copied: row.get(6)?,
is_pinned: row.get::<_, i32>(7)? == 1,
})
})?;
let mut all_items = items_iter.collect::<Result<Vec<_>, _>>()?;
items.collect::<Result<Vec<_>, _>>().map_err(|e| e.into())
if let Some(term) = search_term {
if !term.is_empty() {
let lower_term = term.to_lowercase();
all_items.retain(|item| {
if let Some(preview) = &item.preview {
preview.to_lowercase().contains(&lower_term)
} else if let Some(value) = &item.content_value {
value.to_lowercase().contains(&lower_term)
} else {
false
}
});
}
}
Ok(all_items)
}
fn get_item_content(&self, id: i64) -> Result<String, AppError> {
let db = self.db.lock().unwrap();
let encrypted_content: String = db.query_row(
"SELECT encrypted_content FROM clipboard_history WHERE id = ?",
params![id],
|row| row.get(0),
)?;
decrypt(&encrypted_content, &self.key)
}
fn item_was_copied(&self, id: i64) -> RusqliteResult<usize> {
@ -355,9 +412,24 @@ fn start_monitoring(_app_handle: AppHandle) {
}
#[tauri::command]
pub fn history_get_items(filter: String, limit: u32) -> Result<Vec<ClipboardItem>, String> {
pub fn history_get_items(
filter: String,
search_term: Option<String>,
limit: u32,
offset: u32,
) -> Result<Vec<ClipboardItem>, String> {
if let Some(manager) = MANAGER.lock().unwrap().as_ref() {
manager.get_items(filter, limit).map_err(|e| e.to_string())
manager.get_items(filter, search_term, limit, offset).map_err(|e| e.to_string())
} else {
Err("Clipboard history manager not initialized".to_string())
}
}
#[tauri::command]
pub fn history_get_item_content(id: i64) -> Result<String, String> {
if let Some(manager) = MANAGER.lock().unwrap().as_ref() {
manager.get_item_content(id).map_err(|e| e.to_string())
} else {
Err("Clipboard history manager not initialized".to_string())
}

View file

@ -180,6 +180,7 @@ pub fn run() {
oauth::oauth_get_tokens,
oauth::oauth_remove_tokens,
clipboard_history::history_get_items,
clipboard_history::history_get_item_content,
clipboard_history::history_delete_item,
clipboard_history::history_toggle_pin,
clipboard_history::history_clear_all,

View file

@ -1,13 +1,13 @@
<script lang="ts">
import { invoke } from '@tauri-apps/api/core';
import { onMount } from 'svelte';
import { onMount, tick } from 'svelte';
import { VList } from 'virtua/svelte';
import { Input } from '$lib/components/ui/input';
import { Button } from '$lib/components/ui/button';
import { ArrowLeft, Pin, Trash } from '@lucide/svelte';
import { ArrowLeft, Pin, Trash, Loader2 } from '@lucide/svelte';
import ListItemBase from './nodes/shared/ListItemBase.svelte';
import { convertFileSrc } from '@tauri-apps/api/core';
import { writeText } from '@tauri-apps/plugin-clipboard-manager';
import Fuse from 'fuse.js';
import { Kbd } from './ui/kbd';
import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
import { shortcutToText } from '$lib/renderKey';
@ -20,13 +20,13 @@
onBack: () => void;
};
let { onBack }: Props = $props();
type ClipboardItem = {
id: number;
hash: string;
contentType: 'text' | 'image' | 'color' | 'link' | 'file';
contentValue: string;
contentValue: string | null;
preview: string | null;
contentSizeBytes: number;
sourceAppName: string | null;
firstCopiedAt: string;
lastCopiedAt: string;
@ -40,35 +40,35 @@
data: ClipboardItem | string;
};
let { onBack }: Props = $props();
let allItems = $state<ClipboardItem[]>([]);
let selectedIndex = $state(0);
let searchText = $state('');
let filter = $state('all');
let listContainerEl = $state<HTMLElement | null>(null);
let isInitialMount = $state(true);
const fuse = $derived(
new Fuse(
allItems.filter((item) => !item.isPinned),
{
keys: ['contentValue'],
threshold: 0.3
}
)
);
let currentPage = $state(0);
let hasMore = $state(true);
let isFetching = $state(false);
let selectedItemContent = $state<string | null>(null);
let virtualizedLines = $state<string[]>([]);
let isContentLoading = $state(false);
const displayedItems = $derived.by(() => {
const items: DisplayItem[] = [];
const pinned = allItems.filter((item) => item.isPinned);
const recent = searchText
? fuse.search(searchText).map((r) => r.item)
: allItems.filter((item) => !item.isPinned);
const recent = allItems.filter((item) => !item.isPinned);
if (pinned.length > 0) {
items.push({ id: 'header-pinned', itemType: 'header', data: 'Pinned' });
items.push(...pinned.map((item) => ({ id: item.id, itemType: 'item' as const, data: item })));
pinned.forEach((item) => items.push({ id: item.id, itemType: 'item', data: item }));
}
if (recent.length > 0) {
items.push({ id: 'header-recent', itemType: 'header', data: 'Most Recent' });
items.push(...recent.map((item) => ({ id: item.id, itemType: 'item' as const, data: item })));
recent.forEach((item) => items.push({ id: item.id, itemType: 'item', data: item }));
}
return items;
});
@ -79,73 +79,76 @@
: null
);
async function fetchHistory() {
const iconMap = new Map<ClipboardItem['contentType'], string>([
['text', 'text-16'],
['image', 'image-16'],
['color', 'swatch-16'],
['link', 'link-16'],
['file', 'blank-document-16']
]);
const PAGE_SIZE = 50;
const loadMoreItems = async () => {
if (isFetching || !hasMore) return;
isFetching = true;
try {
const items = await invoke<ClipboardItem[]>('history_get_items', {
const newItems = await invoke<ClipboardItem[]>('history_get_items', {
filter,
limit: 200
limit: PAGE_SIZE,
offset: currentPage * PAGE_SIZE,
searchTerm: searchText || null
});
allItems = items;
selectedIndex = Math.min(selectedIndex, displayedItems.length - 1);
if (newItems.length < PAGE_SIZE) hasMore = false;
allItems = currentPage === 0 ? newItems : [...allItems, ...newItems];
currentPage += 1;
} catch (e) {
console.error('Failed to fetch clipboard history:', e);
} finally {
isFetching = false;
}
}
};
onMount(() => {
fetchHistory();
});
const resetAndFetch = () => {
allItems = [];
currentPage = 0;
hasMore = true;
if (isFetching) return;
selectedIndex = 0;
tick().then(loadMoreItems);
};
$effect(() => {
fetchHistory();
});
const formatDateTime = (dateString: string) =>
`Today at ${new Date(dateString).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit', second: '2-digit' })}`;
function getIconForType(type: ClipboardItem['contentType']): string {
switch (type) {
case 'text':
return 'text-16';
case 'image':
return 'image-16';
case 'color':
return 'swatch-16';
case 'link':
return 'link-16';
case 'file':
return 'blank-document-16';
default:
return 'question-mark-circle-16';
}
}
function formatDateTime(dateString: string) {
const date = new Date(dateString);
return `Today at ${date.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit', second: '2-digit' })}`;
}
async function handleCopy(item: ClipboardItem) {
await writeText(item.contentValue);
const handleCopy = async (item: ClipboardItem) => {
const content =
item.contentValue ?? (await invoke<string>('history_get_item_content', { id: item.id }));
await writeText(content);
await invoke('history_item_was_copied', { id: item.id });
await fetchHistory();
}
const updatedItems = allItems.map((i) =>
i.id === item.id ? { ...i, timesCopied: i.timesCopied + 1 } : i
);
allItems = updatedItems;
};
async function handlePin(item: ClipboardItem) {
const handlePin = async (item: ClipboardItem) => {
await invoke('history_toggle_pin', { id: item.id });
await fetchHistory();
}
resetAndFetch();
};
async function handleDelete(item: ClipboardItem) {
const handleDelete = async (item: ClipboardItem) => {
await invoke('history_delete_item', { id: item.id });
await fetchHistory();
}
resetAndFetch();
};
function handleKeydown(e: KeyboardEvent) {
const handleKeydown = (e: KeyboardEvent) => {
if (e.key === 'Escape') {
e.preventDefault();
onBack();
return;
}
if (!selectedItem) return;
if (e.metaKey && e.shiftKey && e.key.toLowerCase() === 'p') {
e.preventDefault();
handlePin(selectedItem);
@ -154,7 +157,73 @@
e.preventDefault();
handleDelete(selectedItem);
}
}
};
onMount(() => {
const container = listContainerEl;
if (!container) return;
const onScroll = () => {
if (
container.scrollHeight > container.clientHeight &&
container.scrollHeight - container.scrollTop - container.clientHeight < 200
) {
loadMoreItems();
}
};
container.addEventListener('scroll', onScroll);
resetAndFetch();
isInitialMount = false;
return () => container.removeEventListener('scroll', onScroll);
});
$effect(() => {
const term = searchText;
const currentFilter = filter;
if (isInitialMount) return;
const id = setTimeout(() => {
resetAndFetch();
}, 300);
return () => clearTimeout(id);
});
$effect(() => {
const item = selectedItem;
virtualizedLines = [];
selectedItemContent = null;
if (!item) return;
const processContent = async () => {
if (item.contentValue !== null) {
selectedItemContent = item.contentValue;
isContentLoading = false;
return;
}
isContentLoading = true;
try {
const fullContent = await invoke<string>('history_get_item_content', { id: item.id });
if (selectedItem?.id !== item.id) return;
if (item.contentType === 'text' && item.contentSizeBytes > 10000) {
selectedItemContent = fullContent;
await tick();
virtualizedLines = fullContent.split('\n');
} else {
selectedItemContent = fullContent;
}
} catch (err) {
console.error('Failed to load content', err);
if (selectedItem?.id === item.id) selectedItemContent = 'Error: Could not load content.';
} finally {
if (selectedItem?.id === item.id) isContentLoading = false;
}
};
processContent();
});
</script>
<svelte:window onkeydown={handleKeydown} />
@ -172,7 +241,7 @@
/>
<Select.Root bind:value={filter} type="single">
<Select.Trigger class="w-32">
{filter ?? 'All Types'}
{filter === 'all' ? 'All Types' : filter.charAt(0).toUpperCase() + filter.slice(1) + 's'}
</Select.Trigger>
<Select.Content>
<Select.Item value="all">All Types</Select.Item>
@ -184,77 +253,100 @@
</Select.Root>
</header>
<div class="grid grow grid-cols-[minmax(0,_1.5fr)_minmax(0,_2.5fr)] overflow-y-hidden">
<div class="flex-grow overflow-y-auto border-r">
<div class="flex-grow overflow-y-auto border-r" bind:this={listContainerEl}>
<BaseList
items={displayedItems}
bind:selectedIndex
onenter={(item) => handleCopy(item.data as ClipboardItem)}
isItemSelectable={(item) => item.itemType === 'item'}
>
{#snippet itemSnippet({ item, isSelected, onclick })}
{#snippet itemSnippet({ item, isSelected, onclick: itemOnClick })}
{#if item.itemType === 'header'}
<h3 class="text-muted-foreground px-4 pt-2.5 pb-1 text-xs font-semibold uppercase">
{item.data as string}
</h3>
{:else if item.itemType === 'item'}
{@const clipboardItem = item.data as ClipboardItem}
<button class="w-full" {onclick}>
<button class="w-full" onclick={itemOnClick}>
<ListItemBase
icon={getIconForType(clipboardItem.contentType)}
title={clipboardItem.contentValue}
icon={iconMap.get(clipboardItem.contentType) ?? 'question-mark-circle-16'}
title={clipboardItem.preview ?? clipboardItem.contentValue ?? ''}
{isSelected}
/>
</button>
{/if}
{/snippet}
</BaseList>
{#if isFetching && allItems.length > 0}
<div class="text-muted-foreground flex h-10 items-center justify-center">
<Loader2 class="size-4 animate-spin" />
</div>
{/if}
</div>
<div class="flex flex-col overflow-y-hidden">
{#if selectedItem}
<div class="flex-grow overflow-y-auto p-4">
{#if selectedItem.contentType === 'color'}
<div class="flex flex-col items-center justify-center gap-4 py-8">
<div
class="size-24 rounded-full border"
style:background-color={selectedItem.contentValue}
></div>
<p class="font-mono text-lg">{selectedItem.contentValue}</p>
</div>
{:else if selectedItem.contentType === 'image'}
<img
src={convertFileSrc(selectedItem.contentValue)}
alt="Clipboard content"
class="max-h-60 w-full rounded-lg object-contain"
/>
{:else}
<div class="rounded bg-black/10 p-4 font-mono text-sm whitespace-pre-wrap">
{selectedItem.contentValue}
<div class="relative flex-grow overflow-y-auto p-4">
{#if isContentLoading}
<div
class="bg-background/50 absolute inset-0 z-10 flex items-center justify-center backdrop-blur-sm"
>
<Loader2 class="text-muted-foreground size-6 animate-spin" />
</div>
{/if}
<div class="mt-4 border-t py-4">
<h3 class="text-muted-foreground mb-2 text-xs font-semibold uppercase">Information</h3>
<div class="flex flex-col gap-3 text-sm">
<div class="flex justify-between">
<span class="text-muted-foreground">Application</span>
<span>{selectedItem.sourceAppName ?? 'Unknown'}</span>
{#if selectedItemContent}
{#if selectedItem.contentType === 'color'}
<div class="flex flex-col items-center justify-center gap-4 py-8">
<div
class="size-24 rounded-full border"
style:background-color={selectedItemContent}
></div>
<p class="font-mono text-lg">{selectedItemContent}</p>
</div>
<div class="flex justify-between">
<span class="text-muted-foreground">Content type</span>
<span class="capitalize">{selectedItem.contentType}</span>
{:else if selectedItem.contentType === 'image'}
<img
src={convertFileSrc(selectedItemContent)}
alt="Clipboard content"
class="mx-auto max-h-full max-w-full rounded-lg object-contain"
/>
{:else if virtualizedLines.length > 0}
<div class="h-full font-mono text-sm">
<VList data={virtualizedLines}>
{#snippet children(item)}
<div class="whitespace-pre">{item}</div>
{/snippet}
</VList>
</div>
<div class="flex justify-between">
<span class="text-muted-foreground">Times copied</span>
<span>{selectedItem.timesCopied}</span>
</div>
<div class="flex justify-between">
<span class="text-muted-foreground">Last copied</span>
<span>{formatDateTime(selectedItem.lastCopiedAt)}</span>
</div>
<div class="flex justify-between">
<span class="text-muted-foreground">First copied</span>
<span>{formatDateTime(selectedItem.firstCopiedAt)}</span>
{:else if selectedItem.contentType === 'text'}
<div class="rounded bg-black/10 p-4 font-mono text-sm whitespace-pre-wrap">
{selectedItemContent}
</div>
{/if}
{/if}
</div>
<div class="border-t p-4">
<h3 class="text-muted-foreground mb-2 text-xs font-semibold uppercase">Information</h3>
<div class="flex flex-col gap-3 text-sm">
<div class="flex justify-between">
<span class="text-muted-foreground">Application</span>
<span>{selectedItem.sourceAppName ?? 'Unknown'}</span>
</div>
<div class="flex justify-between">
<span class="text-muted-foreground">Content type</span>
<span class="capitalize">{selectedItem.contentType}</span>
</div>
<div class="flex justify-between">
<span class="text-muted-foreground">Times copied</span>
<span>{selectedItem.timesCopied}</span>
</div>
<div class="flex justify-between">
<span class="text-muted-foreground">Last copied</span>
<span>{formatDateTime(selectedItem.lastCopiedAt)}</span>
</div>
<div class="flex justify-between">
<span class="text-muted-foreground">First copied</span>
<span>{formatDateTime(selectedItem.firstCopiedAt)}</span>
</div>
</div>
</div>