diff --git a/pkg/cli/cmd.go b/pkg/cli/cmd.go
index 93e8d62..5cf311f 100644
--- a/pkg/cli/cmd.go
+++ b/pkg/cli/cmd.go
@@ -472,6 +472,14 @@ var retryCmd = &cobra.Command{
 				lastUserMessageIndex = i
 				break
 			}
+
+			if lastUserMessageIndex == 0 {
+				// haven't found the the last user message yet, delete this one
+				err = store.DeleteMessage(&messages[i])
+				if err != nil {
+					Warn("Could not delete previous reply: %v\n", err)
+				}
+			}
 		}
 
 		messages = messages[:lastUserMessageIndex+1]
diff --git a/pkg/cli/store.go b/pkg/cli/store.go
index 41f9795..af9ed55 100644
--- a/pkg/cli/store.go
+++ b/pkg/cli/store.go
@@ -97,6 +97,10 @@ func (s *Store) SaveMessage(message *Message) error {
 	return s.db.Create(message).Error
 }
 
+func (s *Store) DeleteMessage(message *Message) error {
+	return s.db.Delete(&message).Error
+}
+
 func (s *Store) Conversations() ([]Conversation, error) {
 	var conversations []Conversation
 	err := s.db.Find(&conversations).Error