DYNO: Docker-based test harness with mock tool calling

dyno/.gitignore

@@ -0,0 +1,12 @@
+# Logs
+logs/
+
+# Workspace
+workspace/
+
+# Docker
+*.dockerfile
+
+# OS
+.DS_Store
+Thumbs.db

dyno/Dockerfile

@@ -0,0 +1,67 @@
+# DYNO Dockerfile - Builds godex and runs it in an isolated container
+# that tracks all network, file, and command executions
+
+FROM alpine:3.19 AS builder
+
+# Install build dependencies
+RUN apk add --no-cache \
+    bash \
+    git \
+    curl \
+    go \
+    musl-dev \
+    gcc
+
+# Build godex from source (build context is project root)
+WORKDIR /build
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+RUN CGO_ENABLED=0 go build -ldflags="-s -w" -o godex ./cmd/godex
+
+# Final runtime image
+FROM alpine:3.19
+
+RUN apk add --no-cache \
+    bash \
+    python3 \
+    py3-pip \
+    curl \
+    wget \
+    strace \
+    tcpdump \
+    netcat-openbsd \
+    iptables \
+    tree \
+    jq
+
+# Install mock HTTP server dependencies
+RUN pip3 install --no-cache-dir flask requests
+
+# Create directories
+RUN mkdir -p /dyno/logs /dyno/workspace /dyno/mocks
+
+# Copy built binary
+COPY --from=builder /build/godex /usr/local/bin/godex
+
+# Copy dyno scripts
+COPY dyno-mock-server.py /dyno/dyno-mock-server.py
+COPY dyno-entrypoint.sh /dyno/dyno-entrypoint.sh
+COPY dyno.env /dyno/dyno.env
+
+RUN chmod +x /dyno/*.sh /dyno/dyno-mock-server.py
+
+WORKDIR /dyno
+
+# Environment for tracking
+ENV DYNO_MODE=true
+ENV DYNO_LOG_DIR=/dyno/logs
+ENV DYNO_MOCK_HTTP=true
+ENV DYNO_MOCK_FILEOPS=true
+ENV DYNO_MOCK_COMMANDS=true
+ENV DYNO_TRACKING_ENABLED=true
+
+# Volume for outputs
+VOLUME ["/dyno/logs", "/dyno/workspace"]
+ENTRYPOINT ["/dyno/dyno-entrypoint.sh"]

dyno/README.md

@@ -0,0 +1,97 @@
+# DYNO - Docker-based Dynamometer Test Harness for godex
+
+> Like a dynamometer ("dyno") used to test cars and engines under controlled
+> conditions, this harness tests godex in an isolated Docker container.
+
+## Overview
+
+DYNO runs godex in an isolated Docker container that tracks all:
+- **Network operations** (HTTP requests via mock server)
+- **File operations** (open, read, write, stat via strace)
+- **Command executions** (system, execve)
+- **Syscalls** (via strace)
+
+All external network interactions return **dummy responses** to simulate tool calling.
+
+## Quick Start
+
+```bash
+# Build the Docker image
+./dyno-run.sh build
+
+# Run godex with dyno
+./dyno-run.sh run "your prompt here"
+
+# View tracking logs
+./dyno-run.sh logs
+```
+
+## Directory Structure
+
+```
+dyno/
+├── Dockerfile              # Docker image definition
+├── dyno-run.sh             # Main test harness script
+├── dyno-entrypoint.sh      # Container entry point
+├── dyno-mock-server.py     # Mock HTTP server for tool simulation
+├── dyno.env                # Environment configuration
+├── README.md               # This file
+├── logs/                   # Tracking logs (created at runtime)
+│   ├── http_tracking.log
+│   ├── file_tracking.log
+│   ├── cmd_tracking.log
+│   ├── syscall_tracking.log
+│   └── godex_output.log
+└── workspace/              # Shared workspace directory
+```
+
+## How It Works
+
+### Docker Build
+- Builds godex from source code inside the Docker container
+- Installs all necessary tracking tools (strace, tcpdump, etc.)
+- Creates isolated environment for testing
+
+### Mock HTTP Server
+- Runs on port 8888 inside the container
+- Intercepts all HTTP requests from godex
+- Returns dummy responses that simulate AI tool calling
+- Logs all requests to `http_tracking.log`
+
+### Tracking
+- **Network**: All HTTP calls logged and mocked
+- **File**: strace captures file operations
+- **Commands**: All execve/syscall tracked
+- **Syscalls**: Full syscall trace to `syscall_tracking.log`
+
+## Mock Responses
+
+The mock server simulates tool calling by:
+1. Detecting when godex makes an API call with tools
+2. Returning a mock tool call response
+3. Then returning a mock tool result when godex follows up
+
+This allows testing godex's tool-calling behavior without real API calls.
+
+## Environment Variables
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `DYNO_MODE` | Enable DYNO mode | true |
+| `DYNO_TRACKING_ENABLED` | Enable tracking | true |
+| `DYNO_MOCK_HTTP` | Mock HTTP requests | true |
+| `DYNO_MOCK_PORT` | Mock server port | 8888 |
+| `DYNO_LOG_DIR` | Log directory | /dyno/logs |
+
+## Requirements
+
+- Docker
+- bash
+- Project source code (go.mod)
+
+## Notes
+
+- All network calls are intercepted and mocked
+- No real API calls are made
+- Perfect for testing and development
+- Logs are persisted to host directory

dyno/dyno-entrypoint.sh

@@ -0,0 +1,138 @@
+#!/bin/bash
+#
+# dyno-entrypoint.sh - Entry point for DYNO container
+# Sets up tracking environment and runs godex with mock responses
+#
+
+set -e
+
+LOG_DIR="${DYNO_LOG_DIR:-/dyno/logs}"
+MOCK_DIR="/dyno/mocks"
+WORKSPACE="/dyno/workspace"
+
+# Create directories
+mkdir -p "$LOG_DIR"
+mkdir -p "$MOCK_DIR"
+mkdir -p "$WORKSPACE"
+
+echo "[DYNO] Starting DYNO environment..."
+echo "[DYNO] Log directory: $LOG_DIR"
+echo "[DYNO] Mock directory: $MOCK_DIR"
+
+# =============================================================================
+# Start Mock HTTP Server (for dummy tool responses)
+# =============================================================================
+
+start_mock_server() {
+    if [ "${DYNO_MOCK_HTTP:-true}" = "true" ]; then
+        echo "[DYNO] Starting mock HTTP server for tool simulation..."
+        python3 /dyno/dyno-mock-server.py &
+        MOCK_PID=$!
+        echo "[DYNO] Mock server started (PID: $MOCK_PID)"
+        
+        # Wait for server to be ready
+        sleep 2
+    fi
+}
+
+# =============================================================================
+# Setup Network Interception
+# =============================================================================
+
+setup_network_tracking() {
+    echo "[DYNO] Setting up network tracking..."
+    
+    # Create iptables rules to redirect HTTP traffic to mock server
+    # This intercepts all outbound HTTP calls and returns mock responses
+    
+    # Log network attempts
+    echo "[DYNO] Network tracking enabled - all HTTP calls will be logged and mocked"
+    
+    # Set environment for godex to use mock server
+    export HTTP_PROXY="http://localhost:8888"
+    export HTTPS_PROXY="http://localhost:8888"
+    export NO_PROXY="localhost,127.0.0.1"
+}
+
+# =============================================================================
+# Setup File Operation Tracking
+# =============================================================================
+
+setup_file_tracking() {
+    echo "[DYNO] Setting up file operation tracking..."
+    
+    # Use strace to track file operations
+    export DYNO_FILE_TRACKING=true
+    
+    echo "[DYNO] File operation tracking enabled"
+}
+
+# =============================================================================
+# Setup Command Tracking
+# =============================================================================
+
+setup_cmd_tracking() {
+    echo "[DYNO] Setting up command execution tracking..."
+    
+    export DYNO_CMD_TRACKING=true
+    
+    echo "[DYNO] Command execution tracking enabled"
+}
+
+# =============================================================================
+# Run godex with Tracking
+# =============================================================================
+
+run_godex() {
+    local args="${GODEX_ARGS:-}"
+    
+    echo "[DYNO] Running godex with args: $args"
+    
+    # Run godex with strace for syscall tracking
+    if [ "${DYNO_TRACKING_ENABLED:-true}" = "true" ]; then
+        echo "[DYNO] Starting godex with full tracking..."
+        
+        # Run with strace to capture all syscalls
+        strace -f \
+            -e trace=network,file,process \
+            -o "$LOG_DIR/syscall_tracking.log" \
+            -s 256 \
+            godex $args 2>&1 | tee "$LOG_DIR/godex_output.log"
+    else
+        godex $args
+    fi
+}
+
+# =============================================================================
+# Main
+# =============================================================================
+
+main() {
+    # Setup tracking
+    setup_network_tracking
+    setup_file_tracking
+    setup_cmd_tracking
+    
+    # Start mock server
+    start_mock_server
+    
+    # Run godex
+    run_godex
+    
+    echo "[DYNO] Execution completed"
+    
+    # Show summary
+    echo ""
+    echo "[DYNO] === Tracking Summary ==="
+    echo "[DYNO] Logs available in: $LOG_DIR"
+    
+    for log in http file cmd syscall; do
+        local log_file="$LOG_DIR/${log}_tracking.log"
+        if [ -f "$log_file" ] && [ -s "$log_file" ]; then
+            local count=$(wc -l < "$log_file")
+            echo "[DYNO] $log tracking: $count entries"
+        fi
+    done
+}
+
+main "$@"

dyno/dyno-mock-server.py

@@ -0,0 +1,274 @@
+#!/usr/bin/env python3
+"""
+DYNO Mock Server - Intercepts all HTTP requests and returns mock responses
+to simulate tool calling for godex testing.
+
+This server:
+- Logs all HTTP requests to tracking files
+- Returns dummy responses that simulate AI tool calling
+- Supports OpenAI-compatible API responses
+"""
+
+import os
+import sys
+import json
+import logging
+import threading
+from http.server import HTTPServer, BaseHTTPRequestHandler
+from datetime import datetime
+from urllib.parse import urlparse
+import re
+
+# Configuration
+MOCK_PORT = int(os.environ.get('DYNO_MOCK_PORT', 8888))
+LOG_FILE = os.environ.get('DYNO_HTTP_LOG', '/dyno/logs/http_tracking.log')
+MOCK_MODE = os.environ.get('DYNO_MOCK_MODE', 'true').lower() == 'true'
+
+# Setup logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s [DYNO-MOCK] %(message)s'
+)
+logger = logging.getLogger('dyno-mock')
+
+# Request tracking
+request_log_lock = threading.Lock()
+
+def log_request(method, url, headers, body, response_status, response_body):
+    """Log request to tracking file"""
+    timestamp = datetime.now().isoformat()
+    log_entry = f"[{timestamp}] {method} {url}\n"
+    log_entry += f"  Headers: {dict(headers)}\n"
+    log_entry += f"  Body: {body[:500] if body else 'None'}\n"
+    log_entry += f"  Response: {response_status}\n"
+    log_entry += f"  Response Body: {response_body[:500]}\n"
+    log_entry += "-" * 80 + "\n"
+    
+    os.makedirs(os.path.dirname(LOG_FILE), exist_ok=True)
+    with request_log_lock:
+        with open(LOG_FILE, 'a') as f:
+            f.write(log_entry)
+    
+    logger.info(f"{method} {url} -> {response_status}")
+
+
+class MockHandler(BaseHTTPRequestHandler):
+    """Handler that returns mock responses for all HTTP requests"""
+    
+    protocol_version = 'HTTP/1.1'
+    
+    def log_message(self, format, *args):
+        logger.debug(format % args)
+    
+    def get_mock_response(self, method, path, headers=None, body=None):
+        """Determine mock response based on request"""
+        
+        # Check for tool result in request
+        if body:
+            try:
+                req_data = json.loads(body)
+                if 'messages' in req_data:
+                    last_msg = req_data['messages'][-1] if req_data['messages'] else {}
+                    if last_msg.get('role') == 'tool':
+                        return self.mock_tool_result_response(last_msg)
+            except:
+                pass
+        
+        # OpenAI/ChatGPT style endpoints
+        if '/v1/chat/completions' in path or 'chat/completions' in path:
+            return self.mock_chat_completion(body)
+        
+        # OpenAI embeddings
+        if '/v1/embeddings' in path or 'embeddings' in path:
+            return self.mock_embeddings_response()
+        
+        # Anthropic Claude
+        if 'anthropic.com' in path or '/v1/messages' in path:
+            return self.mock_anthropic_response(body)
+        
+        # Default: generic success response
+        return self.mock_default_response()
+    
+    def mock_tool_result_response(self, tool_msg):
+        """Return mock response after tool execution"""
+        tool_name = tool_msg.get('name', 'unknown')
+        content = tool_msg.get('content', '')
+        
+        logger.info(f"Mocking tool result for: {tool_name}")
+        
+        response = {
+            "choices": [{
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": f"[Tool {tool_name} executed successfully. Result: {content[:200]}]"
+                },
+                "finish_reason": "stop"
+            }],
+            "usage": {
+                "prompt_tokens": 100,
+                "completion_tokens": 50,
+                "total_tokens": 150
+            }
+        }
+        return 200, response
+    
+    def mock_chat_completion(self, body):
+        """Mock OpenAI chat completion - simulates tool calling"""
+        try:
+            req_data = json.loads(body) if body else {}
+        except:
+            req_data = {}
+        
+        # Check if this is a tool call request
+        tools = req_data.get('tools', [])
+        messages = req_data.get('messages', [])
+        
+        # If tools provided and no tool call yet, simulate a tool call
+        if tools and not any(m.get('tool_calls', []) for m in messages if isinstance(m, dict)):
+            # Simulate tool calling behavior
+            tool_name = tools[0].get('function', {}).get('name', 'example_tool')
+            
+            response = {
+                "id": "chatcmpl-mock-dyno-001",
+                "object": "chat.completion",
+                "created": int(datetime.now().timestamp()),
+                "model": req_data.get('model', 'gpt-4'),
+                "choices": [{
+                    "index": 0,
+                    "message": {
+                        "role": "assistant",
+                        "tool_calls": [{
+                            "id": "call_mock_001",
+                            "type": "function",
+                            "function": {
+                                "name": tool_name,
+                                "arguments": json.dumps({"query": "mocked query for testing"})
+                            }
+                        }]
+                    },
+                    "finish_reason": "tool_calls"
+                }],
+                "usage": {
+                    "prompt_tokens": 50,
+                    "completion_tokens": 30,
+                    "total_tokens": 80
+                }
+            }
+            return 200, response
+        
+        # Regular chat response
+        response = {
+            "id": "chatcmpl-mock-dyno-002",
+            "object": "chat.completion",
+            "created": int(datetime.now().timestamp()),
+            "model": req_data.get('model', 'gpt-4'),
+            "choices": [{
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": "[DYNO MOCK RESPONSE] This is a simulated AI response. All network calls are being tracked and mocked for testing purposes."
+                },
+                "finish_reason": "stop"
+            }],
+            "usage": {
+                "prompt_tokens": 50,
+                "completion_tokens": 50,
+                "total_tokens": 100
+            }
+        }
+        return 200, response
+    
+    def mock_embeddings_response(self):
+        """Mock OpenAI embeddings response"""
+        response = {
+            "object": "list",
+            "data": [{
+                "object": "embedding",
+                "embedding": [0.1] * 1536,
+                "index": 0
+            }],
+            "model": "text-embedding-ada-002",
+            "usage": {
+                "prompt_tokens": 8,
+                "total_tokens": 8
+            }
+        }
+        return 200, response
+    
+    def mock_anthropic_response(self, body):
+        """Mock Anthropic Claude response"""
+        response = {
+            "id": "msg_mock_001",
+            "type": "message",
+            "role": "assistant",
+            "content": [{
+                "type": "text",
+                "text": "[DYNO MOCK RESPONSE] This is a simulated Claude response. All network calls are being tracked and mocked."
+            }],
+            "model": "claude-3-sonnet-20240229",
+            "stop_reason": "end_turn",
+            "usage": {
+                "input_tokens": 50,
+                "output_tokens": 50
+            }
+        }
+        return 200, response
+    
+    def mock_default_response(self):
+        """Default mock response"""
+        response = {
+            "status": "mocked",
+            "message": "DYNO mock server - all network calls are simulated",
+            "timestamp": datetime.now().isoformat()
+        }
+        return 200, response
+    
+    def do_GET(self):
+        """Handle GET requests"""
+        path = self.path
+        logger.info(f"GET {path}")
+        
+        status, response = self.get_mock_response('GET', path)
+        
+        self.send_response(status)
+        self.send_header('Content-Type', 'application/json')
+        self.end_headers()
+        self.wfile.write(json.dumps(response).encode())
+    
+    def do_POST(self):
+        """Handle POST requests"""
+        path = self.path
+        content_length = int(self.headers.get('Content-Length', 0))
+        body = self.rfile.read(content_length).decode('utf-8') if content_length > 0 else ''
+        
+        logger.info(f"POST {path}")
+        
+        status, response = self.get_mock_response('POST', path, dict(self.headers), body)
+        
+        # Log the request
+        log_request('POST', path, dict(self.headers), body, status, json.dumps(response))
+        
+        self.send_response(status)
+        self.send_header('Content-Type', 'application/json')
+        self.end_headers()
+        self.wfile.write(json.dumps(response).encode())
+
+
+def main():
+    """Start the mock server"""
+    logger.info(f"Starting DYNO mock server on port {MOCK_PORT}")
+    logger.info(f"Logging to: {LOG_FILE}")
+    
+    server = HTTPServer(('0.0.0.0', MOCK_PORT), MockHandler)
+    logger.info(f"DYNO mock server ready at http://0.0.0.0:{MOCK_PORT}")
+    
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        logger.info("Shutting down DYNO mock server")
+        server.shutdown()
+
+
+if __name__ == '__main__':
+    main()

dyno/dyno-run.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
+LOG_DIR="$SCRIPT_DIR/logs"
+DOCKER_IMAGE="godex-dyno:latest"
+CONTAINER_NAME="godex-dyno-$$"
+
+log_info() { echo "[DYNO] $1"; }
+log_success() { echo "[DYNO] $1"; }
+log_error() { echo "[DYNO] ERROR: $1"; }
+
+build_dyno_image() {
+    log_info "Building DYNO Docker image..."
+    cd "$PROJECT_ROOT"
+build_dyno_image() {
+    log_info "Building DYNO Docker image..."
+    cd "$SCRIPT_DIR"
+    docker build -t "$DOCKER_IMAGE" .
+    log_success "Built Docker image: $DOCKER_IMAGE"
+}
+    log_info "Running godex in isolated DYNO container..."
+    log_info "Arguments: $args"
+
+    if ! docker image inspect "$DOCKER_IMAGE" &>/dev/null; then
+        log_info "Docker image not found. Building..."
+        build_dyno_image
+    fi
+
+    mkdir -p "$LOG_DIR"
+
+    docker run --rm \
+        --name "$CONTAINER_NAME" \
+        --cap-add=SYS_PTRACE \
+        -v "$LOG_DIR:/dyno/logs" \
+        -v "$SCRIPT_DIR/workspace:/dyno/workspace" \
+        -e DYNO_MODE=true \
+        -e DYNO_TRACKING_ENABLED=true \
+        -e DYNO_MOCK_HTTP=true \
+        -e DYNO_MOCK_FILEOPS=true \
+        -e DYNO_MOCK_COMMANDS=true \
+        -e DYNO_LOG_DIR=/dyno/logs \
+        -e GODEX_ARGS="$args" \
+        "$DOCKER_IMAGE" \
+        godex $args
+
+    log_success "DYNO execution completed"
+}
+
+show_logs() {
+    log_info "=== DYNO Tracking Logs ==="
+    for log in http file cmd syscall; do
+        local log_file="$LOG_DIR/${log}_tracking.log"
+        if [ -f "$log_file" ]; then
+            echo "=== $log tracking ==="
+            cat "$log_file"
+        fi
+    done
+}
+
+main() {
+    local cmd="${1:-run}"
+    shift || true
+
+    case "$cmd" in
+        build)
+            build_dyno_image
+            ;;
+        run)
+            run_dyno "$@"
+            ;;
+        logs)
+            show_logs
+            ;;
+        *)
+            run_dyno "$cmd $@"
+            ;;
+    esac
+}
+
+main "$@"

dyno/dyno.env

@@ -0,0 +1,48 @@
+# =============================================================================
+# DYNO Docker Environment Configuration
+# =============================================================================
+
+# Enable DYNO mode
+DYNO_MODE=true
+DYNO_TRACKING_ENABLED=true
+
+# Mock HTTP responses (simulate tool calling)
+DYNO_MOCK_HTTP=true
+DYNO_MOCK_HTTP_STATUS=200
+DYNO_MOCK_HTTP_BODY='{"status": "mocked", "data": {"result": "success"}}'
+DYNO_MOCK_HTTP_HEADERS='Content-Type: application/json'
+DYNO_MOCK_PORT=8888
+
+# Mock file operations
+DYNO_MOCK_FILEOPS=true
+DYNO_MOCK_FILE_CONTENT="[MOCKED FILE CONTENT]"
+DYNO_MOCK_FILE_EXISTS=true
+DYNO_MOCK_DIR_CONTENT="mock_file1.txt,mock_file2.txt,mock_dir/"
+
+# Mock command executions
+DYNO_MOCK_COMMANDS=true
+DYNO_MOCK_CMD_EXITCODE=0
+DYNO_MOCK_CMD_STDOUT="[MOCKED COMMAND OUTPUT]"
+DYNO_MOCK_CMD_STDERR=""
+
+# Mock syscalls
+DYNO_MOCK_SYSCALLS=true
+DYNO_MOCK_SYSCALL_RESULT=0
+
+# Tracking log paths
+DYNO_LOG_DIR=/dyno/logs
+DYNO_HTTP_LOG=$DYNO_LOG_DIR/http_tracking.log
+DYNO_FILE_LOG=$DYNO_LOG_DIR/file_tracking.log
+DYNO_CMD_LOG=$DYNO_LOG_DIR/cmd_tracking.log
+DYNO_SYSCALL_LOG=$DYNO_LOG_DIR/syscall_tracking.log
+
+# Network isolation
+DYNO_NETWORK_ISOLATED=true
+DYNO_ALLOWED_HOSTS="localhost,127.0.0.1"
+
+# Filesystem isolation
+DYNO_FS_ISOLATED=true
+DYNO_ALLOWED_PATHS="/tmp,/dyno"
+
+# Workspace
+DYNO_WORKSPACE=/dyno/workspace