Recovering from ASR Errors: Fallback Strategies for Voice Agents

Automatic Speech Recognition (ASR) errors are inevitable in real-world voice agents. Background noise, accents, domain-specific vocabulary, and ambiguous utterances all contribute to misheard transcripts. The key is not preventing all errors—impossible—but detecting them early and recovering gracefully.

The ASR Error Landscape

Common Failure Modes

class ASRFailureModes:
    """Categories of ASR errors in voice agents."""
    
    FAILURE_MODES = {
        'noise_omissions': {
            'description': 'Background noise causes entities to drop',
            'examples': ['dates, amounts, names missing from transcript'],
            'detection': 'Entity presence checks'
        },
        'substituted_intents': {
            'description': 'Misheard words change user intent',
            'examples': ['Cancel becomes schedule, buy becomes sell'],
            'detection': 'Intent confidence scoring'
        },
        'formatting_drift': {
            'description': 'Format changes without meaning change',
            'examples': ['120 becomes one two zero, 555-1234 becomes 5551234'],
            'detection': 'Format validation'
        },
        'truncation': {
            'description': 'Agent responds to incomplete utterance',
            'examples': ['User interrupted but agent already responded'],
            'detection': 'Endpointing analysis'
        },
        'hallucinations': {
            'description': 'ASR generates text during silence',
            'examples': ['Fluent text appears when no one spoke'],
            'detection': 'Audio presence verification'
        }
    }

Detection Strategies

Confidence-Based Detection

from dataclasses import dataclass
from typing import List, Optional

@dataclass
class ASRResult:
    transcript: str
    confidence: float
    alternatives: List[dict]
    word_confidences: List[dict]
    
class ConfidenceDetector:
    def __init__(self, confidence_threshold: float = 0.7):
        self.threshold = confidence_threshold
        
    def detect_uncertainty(self, result: ASRResult) -> dict:
        """Detect if ASR result is uncertain."""
        signals = {
            'low_overall_confidence': result.confidence < self.threshold,
            'close_alternatives': self._has_close_alternatives(result),
            'low_word_confidence': self._has_low_word_confidence(result),
            'ambiguous_intent': self._detect_ambiguous_intent(result)
        }
        
        return {
            'is_uncertain': any(signals.values()),
            'signals': signals,
            'recommended_action': self._recommend_action(signals)
        }
    
    def _has_close_alternatives(self, result: ASRResult) -> bool:
        """Check if alternatives are close in confidence."""
        if len(result.alternatives) < 2:
            return False
            
        top_two = sorted(
            [a['confidence'] for a in result.alternatives[:2]],
            reverse=True
        )
        return (top_two[0] - top_two[1]) < 0.1  # Within 10%
    
    def _has_low_word_confidence(self, result: ASRResult) -> bool:
        """Check for low-confidence words."""
        return any(
            word['confidence'] < 0.6 
            for word in result.word_confidences
        )
    
    def _detect_ambiguous_intent(self, result: ASRResult) -> bool:
        """Detect ambiguous intent from alternatives."""
        intents = [a.get('intent') for a in result.alternatives]
        return len(set(intents)) > 1  # Multiple different intents
    
    def _recommend_action(self, signals: dict) -> str:
        """Recommend recovery action based on signals."""
        if signals['ambiguous_intent']:
            return 'explicit_confirmation'
        elif signals['low_overall_confidence']:
            return 'repetition_request'
        elif signals['close_alternatives']:
            return 'disambiguation'
        else:
            return 'implicit_confirmation'

Entity Presence Checks

class EntityPresenceChecker:
    def __init__(self, required_entities: List[str]):
        self.required_entities = required_entities
        
    def check_entities(self, transcript: str, 
                      extracted_entities: dict) -> dict:
        """Verify all required entities are present."""
        missing = []
        weak = []
        
        for entity in self.required_entities:
            if entity not in extracted_entities:
                missing.append(entity)
            elif extracted_entities[entity].get('confidence', 1.0) < 0.7:
                weak.append(entity)
        
        return {
            'all_present': len(missing) == 0,
            'missing_entities': missing,
            'weak_entities': weak,
            'can_proceed': len(missing) == 0 and len(weak) == 0
        }

Recovery Strategies

1. Implicit Confirmation

Confirm without explicitly asking:

class ImplicitConfirmation:
    def __init__(self, agent):
        self.agent = agent
        
    async def confirm_entity(self, entity: str, value: str) -> str:
        """Implicitly confirm by incorporating into next question."""
        # Instead of: "Did you say March 14th?"
        # Say: "And for March 14th, what time works for you?"
        
        prompt = f"""
        The user mentioned {entity}: {value}
        Create a follow-up question that implicitly confirms this value.
        The user can object if it's wrong.
        """
        
        return await self.agent.generate_response(prompt)

2. Explicit Confirmation

Directly verify uncertain values:

class ExplicitConfirmation:
    def __init__(self, agent):
        self.agent = agent
        
    async def confirm_value(self, entity: str, value: str, 
                           alternatives: List[str]) -> dict:
        """Explicitly confirm a critical value."""
        if len(alternatives) > 1:
            # Present alternatives
            options = " or ".join([value] + alternatives[:2])
            prompt = f"I heard {options}. Which is correct?"
        else:
            # Confirm single value
            prompt = f"I have {value} for {entity}. Is that right?"
        
        response = await self.agent.get_user_response(prompt)
        
        return {
            'confirmed': response.get('affirmative', False),
            'final_value': response.get('correction', value)
        }

3. Targeted Reprompting

Ask for specific uncertain information:

class TargetedReprompt:
    def __init__(self, agent):
        self.agent = agent
        
    async def reprompt_for_entity(self, entity: str, 
                                 context: dict) -> str:
        """Reprompt for a specific entity."""
        # Instead of: "Sorry, I didn't catch that. Please repeat everything."
        # Say: "I got the date, but could you tell me the time again?"
        
        known_info = ", ".join([
            f"{k}: {v}" for k, v in context.items() 
            if k != entity
        ])
        
        prompt = f"""
        I understood: {known_info}
        But I'm unsure about the {entity}.
        Ask specifically for the {entity} while showing what you do know.
        """
        
        return await self.agent.generate_response(prompt)

4. Progressive Prompting

Vary the recovery strategy:

class ProgressivePrompting:
    def __init__(self, agent):
        self.agent = agent
        self.attempt_strategies = [
            'simple_repetition',
            'rephrase_question',
            'offer_alternatives',
            'spell_or_slow_down',
            'escalate_to_human'
        ]
        
    async def recover(self, entity: str, attempt: int) -> str:
        """Progressive recovery with different strategies."""
        if attempt >= len(self.attempt_strategies):
            return await self.escalate_to_human(entity)
        
        strategy = self.attempt_strategies[attempt]
        
        if strategy == 'simple_repetition':
            return f"Could you repeat the {entity}?"
            
        elif strategy == 'rephrase_question':
            return await self.rephrase_question(entity)
            
        elif strategy == 'offer_alternatives':
            return await self.offer_alternatives(entity)
            
        elif strategy == 'spell_or_slow_down':
            return f"You can spell the {entity} or say it slowly."
            
        elif strategy == 'escalate_to_human':
            return await self.escalate_to_human(entity)
    
    async def escalate_to_human(self, entity: str) -> str:
        """Transfer to human agent."""
        return f"I'm having trouble understanding the {entity}. Let me connect you with someone who can help."

Fallback Patterns

Multi-Vendor ASR Fallback

class MultiVendorASR:
    def __init__(self):
        self.primary = WhisperASR()
        self.fallback = GoogleASR()
        self.emergency = DeepgramASR()
        
    async def transcribe(self, audio: bytes) -> ASRResult:
        """Try multiple ASR vendors."""
        # Try primary
        try:
            result = await self.primary.transcribe(audio)
            if result.confidence > 0.7:
                return result
        except Exception as e:
            print(f"Primary ASR failed: {e}")
        
        # Try fallback
        try:
            result = await self.fallback.transcribe(audio)
            if result.confidence > 0.6:
                return result
        except Exception as e:
            print(f"Fallback ASR failed: {e}")
        
        # Try emergency
        try:
            return await self.emergency.transcribe(audio)
        except Exception as e:
            raise ASRFallbackExhausted("All ASR vendors failed")

Confidence-Based Routing

class ConfidenceBasedRouter:
    def __init__(self):
        self.routes = {
            'high_confidence': self.proceed_with_action,
            'medium_confidence': self.implicit_confirmation,
            'low_confidence': self.explicit_confirmation,
            'very_low_confidence': self.reprompt_or_escalate
        }
        
    def route_by_confidence(self, result: ASRResult, 
                           action_risk: str) -> str:
        """Route based on confidence and action risk."""
        if action_risk == 'high':
            # Critical actions need higher confidence
            if result.confidence > 0.9:
                return 'high_confidence'
            elif result.confidence > 0.7:
                return 'medium_confidence'
            else:
                return 'explicit_confirmation'
        else:
            # Lower risk actions can proceed with lower confidence
            if result.confidence > 0.7:
                return 'high_confidence'
            elif result.confidence > 0.5:
                return 'medium_confidence'
            else:
                return 'low_confidence'

Production Implementation

Complete Error Recovery Pipeline

class VoiceAgentWithRecovery:
    def __init__(self):
        self.confidence_detector = ConfidenceDetector()
        self.entity_checker = EntityPresenceChecker([
            'date', 'time', 'location', 'amount'
        ])
        self.implicit_confirm = ImplicitConfirmation(self)
        self.explicit_confirm = ExplicitConfirmation(self)
        self.progressive_prompt = ProgressivePrompting(self)
        
    async def process_utterance(self, audio: bytes) -> dict:
        """Process utterance with error recovery."""
        # Step 1: Transcribe
        asr_result = await self.transcribe(audio)
        
        # Step 2: Detect uncertainty
        uncertainty = self.confidence_detector.detect_uncertainty(asr_result)
        
        # Step 3: Extract entities
        entities = await self.extract_entities(asr_result.transcript)
        
        # Step 4: Check entity presence
        entity_check = self.entity_checker.check_entities(
            asr_result.transcript, entities
        )
        
        # Step 5: Determine action
        if not uncertainty['is_uncertain'] and entity_check['can_proceed']:
            # High confidence, all entities present
            return await self.proceed_with_action(entities)
            
        elif uncertainty['recommended_action'] == 'explicit_confirmation':
            # Need explicit confirmation
            return await self.handle_explicit_confirmation(
                entities, uncertainty['signals']
            )
            
        elif not entity_check['all_present']:
            # Missing entities
            return await self.handle_missing_entities(
                entity_check['missing_entities'], entities
            )
            
        else:
            # General uncertainty
            return await self.handle_general_uncertainty(
                asr_result, uncertainty
            )

Monitoring and Alerting

class ASRMonitoring:
    def __init__(self):
        self.metrics = {
            'total_utterances': 0,
            'uncertain_utterances': 0,
            'recovery_attempts': 0,
            'successful_recoveries': 0,
            'escalations_to_human': 0
        }
        
    def record_utterance(self, uncertainty: dict, 
                        recovery_action: str, success: bool):
        """Record ASR performance metrics."""
        self.metrics['total_utterances'] += 1
        
        if uncertainty['is_uncertain']:
            self.metrics['uncertain_utterances'] += 1
            
        if recovery_action != 'none':
            self.metrics['recovery_attempts'] += 1
            
        if success:
            self.metrics['successful_recoveries'] += 1
            
        if recovery_action == 'escalate_to_human':
            self.metrics['escalations_to_human'] += 1
            
        # Calculate rates
        total = self.metrics['total_utterances']
        self.metrics['uncertainty_rate'] = (
            self.metrics['uncertain_utterances'] / total
        )
        self.metrics['recovery_success_rate'] = (
            self.metrics['successful_recoveries'] / 
            max(self.metrics['recovery_attempts'], 1)
        )

Best Practices

1. Selective Confirmation

Don’t confirm everything—balance accuracy with user experience:

class SelectiveConfirmationStrategy:
    def should_confirm(self, entity: str, value: str, 
                      confidence: float, risk_level: str) -> str:
        """Decide whether and how to confirm."""
        
        # Always confirm high-risk, low-confidence values
        if risk_level == 'high' and confidence < 0.8:
            return 'explicit'
        
        # Implicitly confirm medium-risk values
        elif risk_level == 'medium' and confidence < 0.7:
            return 'implicit'
        
        # Skip confirmation for low-risk, high-confidence values
        elif risk_level == 'low' and confidence > 0.6:
            return 'none'
        
        # Default to implicit for ambiguous cases
        else:
            return 'implicit'

2. Limit Retry Attempts

class RetryLimiter:
    def __init__(self, max_retries: int = 3):
        self.max_retries = max_retries
        
    def can_retry(self, entity: str, attempt: int) -> bool:
        """Check if we should retry or escalate."""
        if attempt >= self.max_retries:
            return False
            
        # Don't retry certain critical entities
        critical_entities = ['payment_amount', 'social_security']
        if entity in critical_entities and attempt >= 2:
            return False
            
        return True

3. Test with Noise

class NoiseTesting:
    def __init__(self):
        self.noise_levels = ['quiet', 'moderate', 'noisy', 'very_noisy']
        
    async def test_recovery_strategies(self, agent):
        """Test recovery with various noise levels."""
        results = {}
        
        for noise_level in self.noise_levels:
            test_audio = self.generate_noisy_audio(noise_level)
            result = await agent.process_utterance(test_audio)
            
            results[noise_level] = {
                'transcription_accuracy': result.get('accuracy'),
                'recovery_success': result.get('recovered'),
                'user_effort': result.get('user_retries', 0)
            }
        
        return results

Conclusion

ASR errors are inevitable, but operational failures are preventable:

1. Detect early: Use confidence scoring and entity presence checks
2. Recover gracefully: Choose confirmation strategy based on risk and confidence
3. Progressive recovery: Vary strategies across retry attempts
4. Escalate when needed: Don’t trap users in endless retry loops
5. Monitor continuously: Track uncertainty rates and recovery success

The key insight: a voice agent that acknowledges uncertainty and recovers gracefully builds more trust than one that pretends to never make mistakes. Selective confirmation—confirming critical values while letting low-risk turns pass—balances accuracy with conversational flow.

Remember: the goal isn’t zero errors, it’s zero silent failures that slip through without checks.