From f161fc4c1f29395d73c1412383317a8a24a0ce08 Mon Sep 17 00:00:00 2001 From: Vito Sansevero Date: Mon, 4 Aug 2025 06:56:05 -0700 Subject: [PATCH] fix(database): resolve SQLite ORDER BY syntax error in GROUP_CONCAT Fix compatibility issue with older SQLite versions that don't support ORDER BY clause inside GROUP_CONCAT function. - Remove ORDER BY from GROUP_CONCAT in SQL queries - Add GROUP_CONCAT for created_at dates - Sort IDs manually in Python after fetching results - Maintain same behavior of keeping oldest prompt as primary This resolves the startup error: "sqlite3.OperationalError: near 'ORDER': syntax error" The fix ensures compatibility with all SQLite versions while preserving the original functionality of keeping the oldest duplicate. --- database/operations.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/database/operations.py b/database/operations.py index a139a8c..cf0eec6 100644 --- a/database/operations.py +++ b/database/operations.py @@ -459,9 +459,12 @@ class PromptDatabase: try: with self.model.get_connection() as conn: # Find duplicates by text content (case-insensitive) + # Note: Removed ORDER BY from GROUP_CONCAT for SQLite compatibility + # We'll sort the IDs manually after fetching cursor = conn.execute(""" SELECT LOWER(TRIM(text)) as normalized_text, COUNT(*) as count, - GROUP_CONCAT(id ORDER BY created_at ASC) as ids + GROUP_CONCAT(id) as ids, + GROUP_CONCAT(created_at) as created_dates FROM prompts GROUP BY LOWER(TRIM(text)) HAVING COUNT(*) > 1 @@ -474,6 +477,12 @@ class PromptDatabase: for group in duplicate_groups: ids = group['ids'].split(',') + created_dates = group['created_dates'].split(',') + + # Sort IDs by created_at date + id_date_pairs = list(zip(ids, created_dates)) + id_date_pairs.sort(key=lambda x: x[1]) # Sort by date + ids = [pair[0] for pair in id_date_pairs] # Get full details for all prompts in this duplicate group prompts = [] @@ -523,9 +532,12 @@ class PromptDatabase: try: with self.model.get_connection() as conn: # Find duplicates by text content (case-insensitive) + # Note: Removed ORDER BY from GROUP_CONCAT for SQLite compatibility + # We'll sort the IDs manually after fetching cursor = conn.execute(""" SELECT LOWER(TRIM(text)) as normalized_text, COUNT(*) as count, - GROUP_CONCAT(id ORDER BY created_at ASC) as ids + GROUP_CONCAT(id) as ids, + GROUP_CONCAT(created_at) as created_dates FROM prompts GROUP BY LOWER(TRIM(text)) HAVING COUNT(*) > 1 @@ -538,9 +550,16 @@ class PromptDatabase: for duplicate in duplicates: ids = duplicate['ids'].split(',') + created_dates = duplicate['created_dates'].split(',') + + # Sort IDs by created_at date to keep the oldest + id_date_pairs = list(zip(ids, created_dates)) + id_date_pairs.sort(key=lambda x: x[1]) # Sort by date + sorted_ids = [int(pair[0]) for pair in id_date_pairs] + # Keep the oldest one (first), merge and delete the rest - primary_id = int(ids[0]) # Keep the oldest - duplicate_ids = [int(id_str) for id_str in ids[1:]] + primary_id = sorted_ids[0] # Keep the oldest + duplicate_ids = sorted_ids[1:] self.logger.debug(f"Merging duplicates: keeping {primary_id}, removing {duplicate_ids}")