# Understanding Spot vs On-Demand

## What We're Building

A decision engine that automatically chooses between spot and on-demand rentals based on job characteristics, plus a spot manager that handles preemption gracefully.

## Prerequisites

* Clore.ai API key
* Python 3.10+
* Understanding of [automation basics](https://docs.clore.ai/dev/getting-started/automation-basics)

## Spot vs On-Demand: Quick Comparison

| Feature      | On-Demand                     | Spot                                       |
| ------------ | ----------------------------- | ------------------------------------------ |
| Price        | Fixed hourly rate             | Bid-based (usually 30-70% cheaper)         |
| Availability | Guaranteed once rented        | Can be outbid/preempted                    |
| Platform Fee | 10%                           | 2.5%                                       |
| Best For     | Production, long-running jobs | Batch processing, fault-tolerant workloads |
| Risk         | None                          | Preemption possible                        |

## Step 1: Analyzing the Spot Market

```python
# spot_analyzer.py
"""Analyze spot market to find optimal bidding strategies."""

import requests
from typing import Dict, List, Optional
from dataclasses import dataclass
from datetime import datetime
import statistics

@dataclass
class SpotAnalysis:
    """Spot market analysis for a server."""
    server_id: int
    gpu_type: str
    on_demand_price_usd: float
    min_spot_price_usd: float
    current_winning_bid_usd: Optional[float]
    recommended_bid_usd: float
    savings_percent: float
    competition_level: str  # low, medium, high

class SpotAnalyzer:
    """Analyze spot market opportunities."""
    
    BASE_URL = "https://api.clore.ai"
    
    def __init__(self, api_key: str):
        self.headers = {"auth": api_key}
    
    def _request(self, endpoint: str, **kwargs) -> dict:
        response = requests.get(f"{self.BASE_URL}{endpoint}", 
                               headers=self.headers, **kwargs)
        return response.json()
    
    def analyze_server(self, server_id: int) -> SpotAnalysis:
        """Analyze spot market for a specific server."""
        
        # Get marketplace data
        marketplace = self._request("/v1/marketplace")
        server = next((s for s in marketplace["servers"] if s["id"] == server_id), None)
        
        if not server:
            raise ValueError(f"Server {server_id} not found")
        
        # Get spot market data
        spot_data = self._request("/v1/spot_marketplace", params={"market": server_id})
        
        # Extract prices
        on_demand = server["price"]["usd"].get("on_demand_clore", 0)
        min_spot = server["price"]["usd"].get("spot", 0)
        
        # Get current bids
        offers = spot_data.get("market", {}).get("offers", [])
        active_bids = [o for o in offers if o.get("active")]
        
        # Find winning bid (if any)
        winning_bid = None
        if active_bids:
            # Convert to USD
            rates = spot_data.get("market", {}).get("currency_rates_in_usd", {})
            winning_bid = max(o["bid"] * rates.get(o["currency"], 1) for o in active_bids)
        
        # Calculate recommended bid
        if winning_bid:
            # Bid 10% above current winner
            recommended = winning_bid * 1.10
        else:
            # Start at minimum + 20%
            recommended = min_spot * 1.20
        
        # Ensure recommended is reasonable
        recommended = max(recommended, min_spot)
        recommended = min(recommended, on_demand * 0.8)  # Don't bid more than 80% of on-demand
        
        # Calculate savings
        savings = ((on_demand - recommended) / on_demand * 100) if on_demand else 0
        
        # Assess competition
        if len(active_bids) == 0:
            competition = "low"
        elif len(active_bids) < 3:
            competition = "medium"
        else:
            competition = "high"
        
        return SpotAnalysis(
            server_id=server_id,
            gpu_type=str(server.get("gpu_array", [])),
            on_demand_price_usd=on_demand,
            min_spot_price_usd=min_spot,
            current_winning_bid_usd=winning_bid,
            recommended_bid_usd=recommended,
            savings_percent=savings,
            competition_level=competition
        )
    
    def find_best_spot_deals(self, gpu_type: str = None, 
                             max_price: float = 1.0,
                             min_savings: float = 30.0) -> List[SpotAnalysis]:
        """Find the best spot market deals."""
        
        marketplace = self._request("/v1/marketplace")
        servers = marketplace["servers"]
        
        deals = []
        for server in servers:
            if server.get("rented"):
                continue
            
            # Filter by GPU type
            if gpu_type:
                gpus = server.get("gpu_array", [])
                if not any(gpu_type in g for g in gpus):
                    continue
            
            try:
                analysis = self.analyze_server(server["id"])
                
                # Filter by price and savings
                if analysis.recommended_bid_usd <= max_price and \
                   analysis.savings_percent >= min_savings:
                    deals.append(analysis)
                    
            except Exception as e:
                continue  # Skip servers with issues
        
        # Sort by savings
        deals.sort(key=lambda x: -x.savings_percent)
        return deals
    
    def print_analysis(self, analysis: SpotAnalysis):
        """Print formatted analysis."""
        print(f"\n{'='*50}")
        print(f"📊 Server {analysis.server_id} ({analysis.gpu_type})")
        print(f"{'='*50}")
        print(f"On-Demand Price: ${analysis.on_demand_price_usd:.2f}/hr")
        print(f"Min Spot Price:  ${analysis.min_spot_price_usd:.2f}/hr")
        
        if analysis.current_winning_bid_usd:
            print(f"Current Winner:  ${analysis.current_winning_bid_usd:.2f}/hr")
        else:
            print(f"Current Winner:  No active bids")
        
        print(f"\n💡 Recommended Bid: ${analysis.recommended_bid_usd:.2f}/hr")
        print(f"💰 Potential Savings: {analysis.savings_percent:.0f}%")
        print(f"📈 Competition: {analysis.competition_level}")


if __name__ == "__main__":
    analyzer = SpotAnalyzer("YOUR_API_KEY")
    
    # Find best RTX 4090 spot deals
    print("🔍 Finding best RTX 4090 spot deals...")
    deals = analyzer.find_best_spot_deals(
        gpu_type="RTX 4090",
        max_price=0.40,
        min_savings=40.0
    )
    
    print(f"\nFound {len(deals)} great deals!")
    for deal in deals[:5]:
        analyzer.print_analysis(deal)
```

## Step 2: Smart Rental Decision Engine

```python
# rental_decision.py
"""Decide between spot and on-demand based on job requirements."""

from dataclasses import dataclass
from typing import Optional
from enum import Enum

class RentalStrategy(Enum):
    ON_DEMAND = "on-demand"
    SPOT = "spot"
    SPOT_WITH_FALLBACK = "spot-with-fallback"

@dataclass
class JobRequirements:
    """Job requirements for rental decision."""
    
    # Time constraints
    max_duration_hours: float = 24.0
    deadline_hours: Optional[float] = None  # Must complete by this time
    
    # Fault tolerance
    checkpointing_enabled: bool = False
    checkpoint_interval_minutes: int = 30
    can_restart: bool = True
    
    # Resource needs
    min_gpu_vram_gb: int = 16
    gpu_type: str = "RTX"
    
    # Budget
    max_hourly_rate: float = 1.0
    total_budget: float = 10.0
    
    # Priority
    priority: str = "normal"  # low, normal, high, critical

class RentalDecisionEngine:
    """Decide optimal rental strategy."""
    
    def __init__(self, spot_analyzer):
        self.spot_analyzer = spot_analyzer
    
    def decide(self, requirements: JobRequirements, 
               server_id: int) -> tuple[RentalStrategy, dict]:
        """
        Decide the optimal rental strategy.
        
        Returns:
            (strategy, config dict with pricing/settings)
        """
        
        # Critical jobs always use on-demand
        if requirements.priority == "critical":
            return RentalStrategy.ON_DEMAND, {
                "reason": "Critical priority requires guaranteed availability"
            }
        
        # Get spot analysis
        try:
            analysis = self.spot_analyzer.analyze_server(server_id)
        except Exception:
            return RentalStrategy.ON_DEMAND, {
                "reason": "Could not analyze spot market"
            }
        
        # Check if spot price is within budget
        spot_viable = analysis.recommended_bid_usd <= requirements.max_hourly_rate
        
        # Check if savings are worthwhile
        significant_savings = analysis.savings_percent >= 25
        
        # Evaluate job characteristics
        fault_tolerant = (
            requirements.checkpointing_enabled or 
            (requirements.can_restart and requirements.max_duration_hours < 2)
        )
        
        has_hard_deadline = requirements.deadline_hours is not None
        time_pressure = has_hard_deadline and requirements.deadline_hours < requirements.max_duration_hours * 1.5
        
        # Decision logic
        if not spot_viable:
            return RentalStrategy.ON_DEMAND, {
                "reason": f"Spot price (${analysis.recommended_bid_usd:.2f}) exceeds budget"
            }
        
        if not significant_savings:
            return RentalStrategy.ON_DEMAND, {
                "reason": f"Spot savings ({analysis.savings_percent:.0f}%) not significant"
            }
        
        if time_pressure and not fault_tolerant:
            return RentalStrategy.ON_DEMAND, {
                "reason": "Hard deadline with non-fault-tolerant job"
            }
        
        if analysis.competition_level == "high" and not fault_tolerant:
            return RentalStrategy.SPOT_WITH_FALLBACK, {
                "reason": "High competition - use spot with on-demand fallback",
                "spotprice": analysis.recommended_bid_usd,
                "fallback_after_preemptions": 2
            }
        
        if fault_tolerant and significant_savings:
            return RentalStrategy.SPOT, {
                "reason": f"Fault-tolerant job with {analysis.savings_percent:.0f}% savings",
                "spotprice": analysis.recommended_bid_usd,
                "checkpoint_before_preemption": True
            }
        
        # Default to spot with fallback for moderate risk tolerance
        return RentalStrategy.SPOT_WITH_FALLBACK, {
            "reason": "Moderate risk - spot with fallback",
            "spotprice": analysis.recommended_bid_usd,
            "fallback_after_preemptions": 3
        }
    
    def explain_decision(self, requirements: JobRequirements, 
                        server_id: int) -> str:
        """Get human-readable explanation of decision."""
        
        strategy, config = self.decide(requirements, server_id)
        analysis = self.spot_analyzer.analyze_server(server_id)
        
        explanation = f"""
📋 Rental Decision for Server {server_id}
{'='*50}

Job Profile:
  - Duration: up to {requirements.max_duration_hours}h
  - Checkpointing: {'Yes' if requirements.checkpointing_enabled else 'No'}
  - Can restart: {'Yes' if requirements.can_restart else 'No'}
  - Priority: {requirements.priority}
  - Budget: ${requirements.max_hourly_rate}/hr (${requirements.total_budget} total)

Market Analysis:
  - On-Demand: ${analysis.on_demand_price_usd:.2f}/hr
  - Spot (recommended): ${analysis.recommended_bid_usd:.2f}/hr
  - Potential Savings: {analysis.savings_percent:.0f}%
  - Competition: {analysis.competition_level}

Decision: {strategy.value.upper()}
Reason: {config.get('reason', 'N/A')}
"""
        
        if strategy == RentalStrategy.SPOT:
            explanation += f"""
Spot Config:
  - Bid price: ${config['spot_price']:.2f}/hr
  - Enable checkpointing before preemption: {config.get('checkpoint_before_preemption', False)}
"""
        
        elif strategy == RentalStrategy.SPOT_WITH_FALLBACK:
            explanation += f"""
Spot with Fallback Config:
  - Initial spot bid: ${config['spot_price']:.2f}/hr
  - Switch to on-demand after: {config['fallback_after_preemptions']} preemptions
"""
        
        return explanation


# Example usage
if __name__ == "__main__":
    from spot_analyzer import SpotAnalyzer
    
    analyzer = SpotAnalyzer("YOUR_API_KEY")
    engine = RentalDecisionEngine(analyzer)
    
    # Define job requirements
    requirements = JobRequirements(
        max_duration_hours=4.0,
        checkpointing_enabled=True,
        checkpoint_interval_minutes=15,
        can_restart=True,
        gpu_type="RTX 4090",
        max_hourly_rate=0.50,
        total_budget=5.0,
        priority="normal"
    )
    
    # Get recommendation
    print(engine.explain_decision(requirements, server_id=12345))
```

## Step 3: Spot Manager with Preemption Handling

```python
# spot_manager.py
"""Manage spot instances with automatic preemption handling."""

import time
import threading
import logging
from typing import Callable, Optional, Dict, Any
from dataclasses import dataclass
from enum import Enum

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class PreemptionAction(Enum):
    CHECKPOINT_AND_WAIT = "checkpoint_and_wait"
    REBID_HIGHER = "rebid_higher"
    SWITCH_TO_ONDEMAND = "switch_to_ondemand"
    MIGRATE_TO_NEW_SERVER = "migrate_to_new_server"

@dataclass
class SpotConfig:
    """Configuration for spot instance management."""
    initial_bid: float
    max_bid: float
    bid_increment: float = 0.05
    max_rebids: int = 3
    preemption_action: PreemptionAction = PreemptionAction.REBID_HIGHER
    checkpoint_callback: Optional[Callable] = None
    fallback_to_ondemand: bool = True

class SpotManager:
    """Manage spot instances with preemption handling."""
    
    def __init__(self, client, spot_config: SpotConfig):
        self.client = client
        self.config = spot_config
        self.current_order_id: Optional[int] = None
        self.current_bid: float = spot_config.initial_bid
        self.rebid_count: int = 0
        self.preemption_count: int = 0
        self._monitor_thread: Optional[threading.Thread] = None
        self._stop_monitoring = threading.Event()
    
    def rent_spot(self, server_id: int, image: str, **kwargs) -> Dict:
        """Rent a server with spot pricing."""
        
        order = self.client.create_order(
            server_id=server_id,
            image=image,
            order_type="spot",
            spot_price=self.current_bid,
            **kwargs
        )
        
        self.current_order_id = order["order_id"]
        logger.info(f"Created spot order {self.current_order_id} with bid ${self.current_bid:.2f}")
        
        # Start monitoring
        self._start_monitoring()
        
        return order
    
    def _start_monitoring(self):
        """Start monitoring for preemption."""
        self._stop_monitoring.clear()
        self._monitor_thread = threading.Thread(target=self._monitor_loop, daemon=True)
        self._monitor_thread.start()
    
    def _monitor_loop(self):
        """Monitor order status for preemption."""
        while not self._stop_monitoring.is_set():
            try:
                if self.current_order_id:
                    order = self.client.get_order(self.current_order_id)
                    
                    if order and order.get("status") == "paused":
                        logger.warning(f"Order {self.current_order_id} preempted!")
                        self._handle_preemption(order)
                
            except Exception as e:
                logger.error(f"Monitor error: {e}")
            
            self._stop_monitoring.wait(timeout=10)  # Check every 10 seconds
    
    def _handle_preemption(self, order: Dict):
        """Handle preemption event."""
        self.preemption_count += 1
        logger.info(f"Handling preemption #{self.preemption_count}")
        
        # Run checkpoint callback if configured
        if self.config.checkpoint_callback:
            try:
                logger.info("Running checkpoint callback...")
                self.config.checkpoint_callback()
            except Exception as e:
                logger.error(f"Checkpoint failed: {e}")
        
        # Decide action
        action = self.config.preemption_action
        
        if action == PreemptionAction.REBID_HIGHER:
            self._rebid_higher()
        
        elif action == PreemptionAction.SWITCH_TO_ONDEMAND:
            self._switch_to_ondemand(order)
        
        elif action == PreemptionAction.MIGRATE_TO_NEW_SERVER:
            self._migrate_to_new_server(order)
        
        elif action == PreemptionAction.CHECKPOINT_AND_WAIT:
            logger.info("Waiting for spot price to drop...")
            time.sleep(300)  # Wait 5 minutes
            self._rebid_higher()
    
    def _rebid_higher(self):
        """Increase bid and retry."""
        if self.rebid_count >= self.config.max_rebids:
            logger.warning("Max rebids reached")
            if self.config.fallback_to_ondemand:
                self._switch_to_ondemand_new()
            return
        
        new_bid = min(
            self.current_bid + self.config.bid_increment,
            self.config.max_bid
        )
        
        if new_bid <= self.current_bid:
            logger.warning("Cannot increase bid further")
            if self.config.fallback_to_ondemand:
                self._switch_to_ondemand_new()
            return
        
        try:
            self.client.set_spot_price(self.current_order_id, new_bid)
            self.current_bid = new_bid
            self.rebid_count += 1
            logger.info(f"Rebid #{self.rebid_count}: ${new_bid:.2f}")
        except Exception as e:
            logger.error(f"Rebid failed: {e}")
    
    def _switch_to_ondemand(self, order: Dict):
        """Switch current server to on-demand."""
        server_id = order.get("renting_server")
        
        # Cancel spot order
        self.client.cancel_order(self.current_order_id)
        
        # Create on-demand order
        new_order = self.client.create_order(
            server_id=server_id,
            image=order.get("image", "nvidia/cuda:12.8.0-base-ubuntu22.04"),
            order_type="on-demand",
            ssh_password="SpotFallback123!"
        )
        
        self.current_order_id = new_order["order_id"]
        logger.info(f"Switched to on-demand: {self.current_order_id}")
    
    def _switch_to_ondemand_new(self):
        """Find a new server and rent on-demand."""
        logger.info("Finding new server for on-demand...")
        
        # Find any available server with similar specs
        servers = self.client.get_marketplace()
        available = [s for s in servers if not s.get("rented")]
        
        if available:
            server = available[0]
            new_order = self.client.create_order(
                server_id=server["id"],
                image="nvidia/cuda:12.8.0-base-ubuntu22.04",
                order_type="on-demand",
                ssh_password="SpotFallback123!"
            )
            self.current_order_id = new_order["order_id"]
            logger.info(f"Created on-demand order on server {server['id']}")
    
    def _migrate_to_new_server(self, order: Dict):
        """Migrate to a different server."""
        logger.info("Migrating to new server...")
        
        # Find servers with lower spot competition
        servers = self.client.get_marketplace()
        gpu_type = order.get("gpu_array", ["RTX"])[0] if order.get("gpu_array") else "RTX"
        
        candidates = []
        for server in servers:
            if server.get("rented"):
                continue
            if any(gpu_type in g for g in server.get("gpu_array", [])):
                candidates.append(server)
        
        if candidates:
            # Pick cheapest
            candidates.sort(key=lambda s: s["price"]["usd"].get("spot", 999))
            new_server = candidates[0]
            
            # Cancel old order
            self.client.cancel_order(self.current_order_id)
            
            # Create new spot order
            new_order = self.client.create_order(
                server_id=new_server["id"],
                image=order.get("image", "nvidia/cuda:12.8.0-base-ubuntu22.04"),
                order_type="spot",
                spot_price=self.config.initial_bid,
                ssh_password="SpotMigrate123!"
            )
            
            self.current_order_id = new_order["order_id"]
            self.current_bid = self.config.initial_bid
            self.rebid_count = 0
            logger.info(f"Migrated to server {new_server['id']}")
    
    def stop(self):
        """Stop monitoring and cleanup."""
        self._stop_monitoring.set()
        if self._monitor_thread:
            self._monitor_thread.join(timeout=5)
        
        if self.current_order_id:
            self.client.cancel_order(self.current_order_id)
            logger.info(f"Cancelled order {self.current_order_id}")
    
    def get_stats(self) -> Dict:
        """Get spot instance statistics."""
        return {
            "current_order_id": self.current_order_id,
            "current_bid": self.current_bid,
            "rebid_count": self.rebid_count,
            "preemption_count": self.preemption_count,
            "total_spent_extra": self.rebid_count * self.config.bid_increment
        }


# Example usage with checkpoint callback
def create_checkpoint():
    """Save training checkpoint."""
    print("💾 Saving checkpoint...")
    # In real code: torch.save(model.state_dict(), "checkpoint.pt")
    time.sleep(2)
    print("✅ Checkpoint saved")


if __name__ == "__main__":
    from client import CloreClient
    
    client = CloreClient("YOUR_API_KEY")
    
    config = SpotConfig(
        initial_bid=0.15,
        max_bid=0.35,
        bid_increment=0.05,
        max_rebids=4,
        preemption_action=PreemptionAction.REBID_HIGHER,
        checkpoint_callback=create_checkpoint,
        fallback_to_ondemand=True
    )
    
    manager = SpotManager(client, config)
    
    try:
        # Find a server
        servers = client.get_marketplace()
        server = next(s for s in servers if not s.get("rented"))
        
        # Rent with spot
        order = manager.rent_spot(
            server_id=server["id"],
            image="pytorch/pytorch:2.7.1-cuda12.8-cudnn9-runtime",
            ssh_password="SpotTest123!"
        )
        
        print(f"Order created: {order['order_id']}")
        print("Press Ctrl+C to stop")
        
        while True:
            time.sleep(60)
            stats = manager.get_stats()
            print(f"Stats: {stats}")
            
    except KeyboardInterrupt:
        print("\nStopping...")
    finally:
        manager.stop()
```

## Decision Flowchart

```
                    ┌─────────────────┐
                    │ New Job Request │
                    └────────┬────────┘
                             │
                    ┌────────▼────────┐
                    │ Priority =      │
                    │ Critical?       │
                    └────────┬────────┘
                        Yes  │  No
           ┌─────────────────┴─────────────────┐
           │                                   │
  ┌────────▼────────┐               ┌──────────▼──────────┐
  │   ON-DEMAND     │               │ Check Spot Savings  │
  │ (Guaranteed)    │               │ >= 25%?             │
  └─────────────────┘               └──────────┬──────────┘
                                          Yes  │  No
                               ┌───────────────┴───────────┐
                               │                           │
                    ┌──────────▼──────────┐      ┌─────────▼─────────┐
                    │ Is Job Fault-       │      │    ON-DEMAND      │
                    │ Tolerant?           │      │ (Savings too low) │
                    └──────────┬──────────┘      └───────────────────┘
                          Yes  │  No
               ┌───────────────┴───────────────┐
               │                               │
    ┌──────────▼──────────┐         ┌──────────▼──────────┐
    │ Has Hard Deadline?  │         │ SPOT WITH FALLBACK  │
    └──────────┬──────────┘         │ (Higher risk)       │
          Yes  │  No                └─────────────────────┘
   ┌───────────┴───────────┐
   │                       │
┌──▼───────────────┐  ┌────▼────────────┐
│ SPOT WITH        │  │      SPOT       │
│ FALLBACK         │  │ (Best savings)  │
│ (Deadline risk)  │  └─────────────────┘
└──────────────────┘
```

## Cost Comparison Example

```python
# cost_comparison.py
"""Compare spot vs on-demand costs for a job."""

def compare_costs(
    duration_hours: float,
    on_demand_rate: float,
    spot_rate: float,
    preemption_probability: float = 0.1,
    restart_overhead_minutes: float = 5
):
    """
    Compare expected costs for spot vs on-demand.
    
    Args:
        duration_hours: Expected job duration
        on_demand_rate: On-demand hourly rate (USD)
        spot_rate: Spot hourly rate (USD)
        preemption_probability: Probability of preemption per hour
        restart_overhead_minutes: Time lost per restart
    """
    
    # On-demand cost (simple)
    on_demand_cost = duration_hours * on_demand_rate
    on_demand_fee = on_demand_cost * 0.10  # 10% platform fee
    total_on_demand = on_demand_cost + on_demand_fee
    
    # Spot cost (with expected preemptions)
    expected_preemptions = duration_hours * preemption_probability
    restart_time = expected_preemptions * (restart_overhead_minutes / 60)
    effective_duration = duration_hours + restart_time
    
    spot_cost = effective_duration * spot_rate
    spot_fee = spot_cost * 0.025  # 2.5% platform fee
    total_spot = spot_cost + spot_fee
    
    # Savings
    savings = total_on_demand - total_spot
    savings_percent = (savings / total_on_demand) * 100
    
    return {
        "on_demand": {
            "compute": on_demand_cost,
            "fee": on_demand_fee,
            "total": total_on_demand
        },
        "spot": {
            "compute": spot_cost,
            "fee": spot_fee,
            "total": total_spot,
            "expected_preemptions": expected_preemptions,
            "effective_duration": effective_duration
        },
        "savings": savings,
        "savings_percent": savings_percent
    }


if __name__ == "__main__":
    result = compare_costs(
        duration_hours=4.0,
        on_demand_rate=0.40,
        spot_rate=0.15,
        preemption_probability=0.05
    )
    
    print("💰 Cost Comparison (4-hour training job)")
    print("="*50)
    print(f"\nOn-Demand:")
    print(f"  Compute: ${result['on_demand']['compute']:.2f}")
    print(f"  Fee (10%): ${result['on_demand']['fee']:.2f}")
    print(f"  Total: ${result['on_demand']['total']:.2f}")
    
    print(f"\nSpot:")
    print(f"  Compute: ${result['spot']['compute']:.2f}")
    print(f"  Fee (2.5%): ${result['spot']['fee']:.2f}")
    print(f"  Total: ${result['spot']['total']:.2f}")
    print(f"  Expected preemptions: {result['spot']['expected_preemptions']:.1f}")
    
    print(f"\n✅ Savings: ${result['savings']:.2f} ({result['savings_percent']:.0f}%)")
```

## Next Steps

* [Training a PyTorch Model on Clore](https://docs.clore.ai/dev/machine-learning-and-training/pytorch-basics)
* [Building a Spot Instance Manager](https://docs.clore.ai/dev/devops-and-automation/spot-manager)
* [Cost Optimization Strategies](https://docs.clore.ai/dev/devops-and-automation/cost-optimization)