Setting Up Your Development Environment

What We're Building

A complete development environment setup script that provisions a GPU, installs your preferred tools (VS Code Server, Jupyter, conda), syncs your code, and keeps everything reproducible.

Prerequisites

• Clore.ai API key

• Python 3.10+

• Your SSH public key (~/.ssh/id_rsa.pub)

Step 1: The Environment Configuration

# config.py
"""Environment configuration for Clore development setup."""

from dataclasses import dataclass
from typing import List, Dict, Optional
import os

@dataclass
class DevEnvironment:
    """Development environment configuration."""
    
    # GPU requirements
    gpu_type: str = "RTX 4090"
    min_vram_gb: int = 24
    max_price_usd: float = 0.50
    
    # Docker image
    base_image: str = "pytorch/pytorch:2.7.1-cuda12.8-cudnn9-devel"
    
    # Ports to expose
    ports: Dict[str, str] = None
    
    # Environment variables
    env_vars: Dict[str, str] = None
    
    # Packages to install
    pip_packages: List[str] = None
    apt_packages: List[str] = None
    
    # Code sync
    git_repos: List[str] = None
    
    def __post_init__(self):
        self.ports = self.ports or {
            "22": "tcp",      # SSH
            "8888": "http",   # Jupyter
            "8080": "http",   # VS Code Server
            "6006": "http",   # TensorBoard
        }
        
        self.env_vars = self.env_vars or {
            "NVIDIA_VISIBLE_DEVICES": "all",
            "PYTHONUNBUFFERED": "1",
            "JUPYTER_TOKEN": "cloredev",
        }
        
        self.pip_packages = self.pip_packages or [
            "jupyterlab",
            "tensorboard",
            "wandb",
            "transformers",
            "datasets",
            "accelerate",
            "bitsandbytes",
        ]
        
        self.apt_packages = self.apt_packages or [
            "git",
            "curl",
            "wget",
            "vim",
            "htop",
            "nvtop",
            "tmux",
        ]


# Preset configurations
PRESETS = {
    "ml-training": DevEnvironment(
        gpu_type="RTX 4090",
        min_vram_gb=24,
        base_image="pytorch/pytorch:2.7.1-cuda12.8-cudnn9-devel",
        pip_packages=[
            "transformers", "datasets", "accelerate", "bitsandbytes",
            "wandb", "tensorboard", "jupyterlab", "optuna"
        ]
    ),
    "inference": DevEnvironment(
        gpu_type="RTX 3090",
        min_vram_gb=24,
        base_image="nvidia/cuda:12.8.0-base-ubuntu22.04",
        pip_packages=[
            "vllm", "fastapi", "uvicorn", "transformers"
        ]
    ),
    "rendering": DevEnvironment(
        gpu_type="RTX 4090",
        min_vram_gb=24,
        base_image="nvidia/cuda:12.8.0-base-ubuntu22.04",
        apt_packages=["blender", "ffmpeg"]
    ),
}

Step 2: Server Setup Script Generator

# setup_generator.py
"""Generate setup scripts for Clore servers."""

from config import DevEnvironment
from typing import Optional

def generate_setup_script(config: DevEnvironment, 
                          ssh_key: Optional[str] = None) -> str:
    """Generate a bash setup script for the server."""
    
    script = """#!/bin/bash
set -e

echo "🚀 Setting up Clore development environment..."

# Update system
apt-get update && apt-get upgrade -y

# Install apt packages
apt-get install -y {apt_packages}

# Install Python packages
pip install --upgrade pip
pip install {pip_packages}

# Configure Jupyter
mkdir -p ~/.jupyter
cat > ~/.jupyter/jupyter_lab_config.py << 'EOF'
c.ServerApp.ip = '0.0.0.0'
c.ServerApp.port = 8888
c.ServerApp.open_browser = False
c.ServerApp.token = '{jupyter_token}'
c.ServerApp.allow_root = True
c.ServerApp.allow_origin = '*'
EOF

# Install VS Code Server (code-server)
curl -fsSL https://code-server.dev/install.sh | sh
cat > ~/.config/code-server/config.yaml << 'EOF'
bind-addr: 0.0.0.0:8080
auth: password
password: cloredev
cert: false
EOF

# Create workspace
mkdir -p /workspace
cd /workspace

# Clone git repos
{git_clone_commands}

# Start services
echo "Starting Jupyter Lab..."
nohup jupyter lab --allow-root > /var/log/jupyter.log 2>&1 &

echo "Starting VS Code Server..."
nohup code-server > /var/log/code-server.log 2>&1 &

# Print access info
echo ""
echo "✅ Environment ready!"
echo ""
echo "📝 Access:"
echo "   Jupyter: http://$(hostname):8888 (token: {jupyter_token})"
echo "   VS Code: http://$(hostname):8080 (password: cloredev)"
echo "   SSH: ssh root@$(hostname)"
echo ""
echo "🔧 GPU Info:"
nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv
""".format(
        apt_packages=" ".join(config.apt_packages),
        pip_packages=" ".join(config.pip_packages),
        jupyter_token=config.env_vars.get("JUPYTER_TOKEN", "cloredev"),
        git_clone_commands="\n".join([
            f"git clone {repo}" for repo in (config.git_repos or [])
        ]) or "# No repos configured"
    )
    
    # Add SSH key setup if provided
    if ssh_key:
        ssh_setup = f"""
# Setup SSH key
mkdir -p ~/.ssh
echo "{ssh_key}" >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys
"""
        script = script.replace("# Update system", ssh_setup + "\n# Update system")
    
    return script


def save_setup_script(config: DevEnvironment, filename: str = "setup.sh",
                      ssh_key: Optional[str] = None):
    """Save setup script to file."""
    script = generate_setup_script(config, ssh_key)
    with open(filename, "w") as f:
        f.write(script)
    print(f"✅ Saved setup script to {filename}")
    return filename


if __name__ == "__main__":
    from config import PRESETS
    
    # Generate script for ML training preset
    config = PRESETS["ml-training"]
    save_setup_script(config, "ml-training-setup.sh")

Step 3: Automated Environment Provisioner

# provisioner.py
"""Automated Clore environment provisioner."""

import requests
import time
import subprocess
import tempfile
import os
from typing import Dict, Optional
from config import DevEnvironment, PRESETS
from setup_generator import generate_setup_script

class CloreProvisioner:
    """Provision and setup Clore development environments."""
    
    BASE_URL = "https://api.clore.ai"
    
    def __init__(self, api_key: str, ssh_key_path: str = "~/.ssh/id_rsa.pub"):
        self.api_key = api_key
        self.headers = {"auth": api_key}
        
        # Load SSH key
        ssh_key_path = os.path.expanduser(ssh_key_path)
        if os.path.exists(ssh_key_path):
            with open(ssh_key_path) as f:
                self.ssh_key = f.read().strip()
        else:
            self.ssh_key = None
    
    def _request(self, method: str, endpoint: str, **kwargs) -> Dict:
        url = f"{self.BASE_URL}{endpoint}"
        response = requests.request(method, url, headers=self.headers, **kwargs)
        data = response.json()
        if data.get("code") != 0:
            raise Exception(f"API Error: {data}")
        return data
    
    def find_server(self, config: DevEnvironment) -> Dict:
        """Find a suitable server based on config."""
        servers = self._request("GET", "/v1/marketplace")["servers"]
        
        candidates = []
        for s in servers:
            if s.get("rented"):
                continue
            
            # Check GPU type
            gpus = s.get("gpu_array", [])
            if not any(config.gpu_type in g for g in gpus):
                continue
            
            # Check price
            price = s.get("price", {}).get("usd", {}).get("on_demand_clore")
            if not price or price > config.max_price_usd:
                continue
            
            candidates.append({
                "id": s["id"],
                "gpus": gpus,
                "price": price,
                "specs": s.get("specs", {}),
                "reliability": s.get("reliability", 0)
            })
        
        if not candidates:
            raise Exception(f"No suitable server found for {config.gpu_type}")
        
        # Sort by reliability, then price
        candidates.sort(key=lambda x: (-x["reliability"], x["price"]))
        return candidates[0]
    
    def provision(self, config: DevEnvironment, wait_timeout: int = 180) -> Dict:
        """Provision a new environment."""
        
        # Find server
        print(f"🔍 Finding {config.gpu_type} server...")
        server = self.find_server(config)
        print(f"   Found server {server['id']} at ${server['price']:.2f}/hr")
        
        # Create order
        print("📦 Creating order...")
        order_data = {
            "renting_server": server["id"],
            "type": "on-demand",
            "currency": "CLORE-Blockchain",
            "image": config.base_image,
            "ports": config.ports,
            "env": config.env_vars,
        }
        
        if self.ssh_key:
            order_data["ssh_key"] = self.ssh_key
        else:
            order_data["ssh_password"] = "CloreDevEnv123!"
        
        order = self._request("POST", "/v1/create_order", json=order_data)
        order_id = order["order_id"]
        print(f"   Order ID: {order_id}")
        
        # Wait for ready
        print("⏳ Waiting for server...")
        active_order = self._wait_for_ready(order_id, wait_timeout)
        
        # Get connection info
        ssh_info = active_order["connection"]["ssh"]
        print(f"✅ Server ready: {ssh_info}")
        
        return {
            "order_id": order_id,
            "server_id": server["id"],
            "ssh": ssh_info,
            "price_usd": server["price"],
            "gpus": server["gpus"],
            "ports": active_order["connection"].get("ports", {}),
        }
    
    def _wait_for_ready(self, order_id: int, timeout: int) -> Dict:
        """Wait for order to become active."""
        for _ in range(timeout // 2):
            orders = self._request("GET", "/v1/my_orders")["orders"]
            order = next((o for o in orders if o["order_id"] == order_id), None)
            if order and order.get("status") == "running":
                return order
            time.sleep(2)
        raise Exception("Timeout waiting for server")
    
    def setup_environment(self, ssh_info: str, config: DevEnvironment):
        """Run setup script on server via SSH."""
        
        # Parse SSH info
        # Format: "ssh root@host -p port"
        parts = ssh_info.split()
        user_host = parts[1]  # root@host
        port = parts[3] if len(parts) > 3 else "22"
        host = user_host.split("@")[1]
        
        # Generate setup script
        script = generate_setup_script(config, self.ssh_key)
        
        # Save to temp file
        with tempfile.NamedTemporaryFile(mode='w', suffix='.sh', delete=False) as f:
            f.write(script)
            script_path = f.name
        
        try:
            # Copy script to server
            print("📤 Uploading setup script...")
            subprocess.run([
                "scp", "-o", "StrictHostKeyChecking=no", "-P", port,
                script_path, f"root@{host}:/tmp/setup.sh"
            ], check=True)
            
            # Run script
            print("🔧 Running setup (this may take a few minutes)...")
            subprocess.run([
                "ssh", "-o", "StrictHostKeyChecking=no", "-p", port,
                f"root@{host}", "bash /tmp/setup.sh"
            ], check=True)
            
            print("✅ Environment setup complete!")
            
        finally:
            os.unlink(script_path)
    
    def cancel(self, order_id: int):
        """Cancel an order."""
        self._request("POST", "/v1/cancel_order", json={"id": order_id})
        print(f"✅ Order {order_id} cancelled")


def main():
    import sys
    
    if len(sys.argv) < 2:
        print("Usage: python provisioner.py API_KEY [preset]")
        print("Presets:", list(PRESETS.keys()))
        sys.exit(1)
    
    api_key = sys.argv[1]
    preset_name = sys.argv[2] if len(sys.argv) > 2 else "ml-training"
    
    config = PRESETS.get(preset_name, PRESETS["ml-training"])
    provisioner = CloreProvisioner(api_key)
    
    try:
        # Provision
        env = provisioner.provision(config)
        
        # Setup
        input("\n⏸️  Press Enter to run setup script (or Ctrl+C to skip)...")
        provisioner.setup_environment(env["ssh"], config)
        
        # Print access info
        print("\n" + "="*50)
        print("🎉 Development Environment Ready!")
        print("="*50)
        print(f"\nSSH: {env['ssh']}")
        print(f"Jupyter: Check port 8888")
        print(f"VS Code: Check port 8080")
        print(f"\nPrice: ${env['price_usd']:.2f}/hr")
        print(f"Order ID: {env['order_id']}")
        
        input("\n⏸️  Press Enter to cancel order and cleanup...")
        provisioner.cancel(env["order_id"])
        
    except KeyboardInterrupt:
        print("\nCancelled")


if __name__ == "__main__":
    main()

Step 4: Sync Your Local Code

# code_sync.py
"""Sync local code to Clore server."""

import subprocess
import os
from typing import List

class CodeSync:
    """Sync code between local machine and Clore server."""
    
    def __init__(self, host: str, port: int = 22, user: str = "root"):
        self.host = host
        self.port = port
        self.user = user
        self.remote = f"{user}@{host}"
    
    def push(self, local_path: str, remote_path: str = "/workspace",
             exclude: List[str] = None):
        """Push local directory to server."""
        
        exclude = exclude or [
            ".git", "__pycache__", "*.pyc", ".venv", "venv",
            "node_modules", ".env", "*.egg-info", "dist", "build"
        ]
        
        exclude_args = []
        for pattern in exclude:
            exclude_args.extend(["--exclude", pattern])
        
        cmd = [
            "rsync", "-avz", "--progress",
            "-e", f"ssh -p {self.port} -o StrictHostKeyChecking=no",
            *exclude_args,
            local_path.rstrip("/") + "/",
            f"{self.remote}:{remote_path}/"
        ]
        
        print(f"📤 Syncing {local_path} → {remote_path}")
        subprocess.run(cmd, check=True)
        print("✅ Sync complete")
    
    def pull(self, remote_path: str, local_path: str):
        """Pull directory from server to local."""
        
        cmd = [
            "rsync", "-avz", "--progress",
            "-e", f"ssh -p {self.port} -o StrictHostKeyChecking=no",
            f"{self.remote}:{remote_path}/",
            local_path.rstrip("/") + "/"
        ]
        
        print(f"📥 Syncing {remote_path} → {local_path}")
        subprocess.run(cmd, check=True)
        print("✅ Sync complete")
    
    def watch(self, local_path: str, remote_path: str = "/workspace"):
        """Watch for changes and auto-sync."""
        
        try:
            import watchdog
        except ImportError:
            print("Install watchdog: pip install watchdog")
            return
        
        from watchdog.observers import Observer
        from watchdog.events import FileSystemEventHandler
        
        class SyncHandler(FileSystemEventHandler):
            def __init__(self, sync):
                self.sync = sync
                self.local_path = local_path
                self.remote_path = remote_path
            
            def on_modified(self, event):
                if not event.is_directory:
                    self.sync.push(self.local_path, self.remote_path)
        
        observer = Observer()
        observer.schedule(SyncHandler(self), local_path, recursive=True)
        observer.start()
        
        print(f"👁️  Watching {local_path} for changes...")
        try:
            import time
            while True:
                time.sleep(1)
        except KeyboardInterrupt:
            observer.stop()
        observer.join()


if __name__ == "__main__":
    import sys
    
    if len(sys.argv) < 4:
        print("Usage: python code_sync.py HOST PORT LOCAL_PATH [REMOTE_PATH]")
        sys.exit(1)
    
    host = sys.argv[1]
    port = int(sys.argv[2])
    local_path = sys.argv[3]
    remote_path = sys.argv[4] if len(sys.argv) > 4 else "/workspace"
    
    sync = CodeSync(host, port)
    sync.push(local_path, remote_path)

Full Workflow Script

#!/usr/bin/env python3
"""
Complete dev environment setup: Provision → Setup → Sync → Work → Cleanup
"""

import sys
import os

# Add parent directory to path
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))

from provisioner import CloreProvisioner
from config import PRESETS, DevEnvironment
from code_sync import CodeSync


def main():
    if len(sys.argv) < 2:
        print("Usage: python dev_env.py API_KEY [preset] [local_code_path]")
        print("\nPresets:")
        for name, config in PRESETS.items():
            print(f"  {name}: {config.gpu_type}, {config.base_image}")
        sys.exit(1)
    
    api_key = sys.argv[1]
    preset = sys.argv[2] if len(sys.argv) > 2 else "ml-training"
    local_code = sys.argv[3] if len(sys.argv) > 3 else None
    
    config = PRESETS.get(preset, PRESETS["ml-training"])
    provisioner = CloreProvisioner(api_key)
    
    print("="*60)
    print(f"🚀 Clore Dev Environment: {preset}")
    print("="*60)
    
    # Provision
    env = provisioner.provision(config)
    
    # Parse SSH for sync
    ssh_parts = env["ssh"].split()
    host = ssh_parts[1].split("@")[1]
    port = int(ssh_parts[3]) if len(ssh_parts) > 3 else 22
    
    # Run setup
    print("\n" + "-"*60)
    response = input("Run setup script? [Y/n]: ").strip().lower()
    if response != "n":
        provisioner.setup_environment(env["ssh"], config)
    
    # Sync code
    if local_code and os.path.exists(local_code):
        print("\n" + "-"*60)
        response = input(f"Sync {local_code} to server? [Y/n]: ").strip().lower()
        if response != "n":
            sync = CodeSync(host, port)
            sync.push(local_code, "/workspace/code")
    
    # Print summary
    print("\n" + "="*60)
    print("🎉 Environment Ready!")
    print("="*60)
    print(f"\n📋 Connection:")
    print(f"   SSH: {env['ssh']}")
    print(f"   Jupyter: http://{host}:8888 (token: cloredev)")
    print(f"   VS Code: http://{host}:8080 (password: cloredev)")
    print(f"\n💰 Cost: ${env['price_usd']:.2f}/hr")
    print(f"🔢 Order ID: {env['order_id']}")
    
    # Keep running
    print("\n" + "-"*60)
    print("Press Ctrl+C to stop and cleanup")
    
    try:
        import time
        while True:
            time.sleep(60)
            print(f"⏱️  Still running... ${env['price_usd']/60:.4f} this minute")
    except KeyboardInterrupt:
        print("\n\n🧹 Cleaning up...")
        provisioner.cancel(env["order_id"])
        print("Done!")


if __name__ == "__main__":
    main()

Quick Commands

# Provision ML training environment
python dev_env.py YOUR_API_KEY ml-training ./my-project

# Provision inference environment
python dev_env.py YOUR_API_KEY inference

# Just generate setup script (no provisioning)
python setup_generator.py

# Sync code to existing server
python code_sync.py node123.clore.ai 40022 ./my-code /workspace

Docker Images Reference

Image	Use Case	Size
pytorch/pytorch:2.7.1-cuda12.8-cudnn9-devel	ML Training	~8GB
tensorflow/tensorflow:2.14.0-gpu	TensorFlow	~6GB
nvidia/cuda:12.8.0-base-ubuntu22.04	Minimal CUDA	~2GB
nvidia/cuda:12.8.0-devel-ubuntu22.04	CUDA Development	~4GB
huggingface/transformers-pytorch-gpu	Transformers	~10GB

Next Steps

• Automating GPU Rental with Python

• Training a PyTorch Model on Clore