From 7c5e85e116030054cbda0ab030b41d54fda3ef71 Mon Sep 17 00:00:00 2001 From: Cheikh Seck Date: Wed, 22 Apr 2026 09:34:44 +0000 Subject: [PATCH] DYNO: Docker-based test harness with mock tool calling --- dyno/.gitignore | 12 ++ dyno/Dockerfile | 67 ++++++++++ dyno/README.md | 97 ++++++++++++++ dyno/dyno-entrypoint.sh | 138 ++++++++++++++++++++ dyno/dyno-mock-server.py | 274 +++++++++++++++++++++++++++++++++++++++ dyno/dyno-run.sh | 82 ++++++++++++ dyno/dyno.env | 48 +++++++ 7 files changed, 718 insertions(+) create mode 100644 dyno/.gitignore create mode 100644 dyno/Dockerfile create mode 100644 dyno/README.md create mode 100644 dyno/dyno-entrypoint.sh create mode 100644 dyno/dyno-mock-server.py create mode 100755 dyno/dyno-run.sh create mode 100644 dyno/dyno.env diff --git a/dyno/.gitignore b/dyno/.gitignore new file mode 100644 index 0000000..454fca2 --- /dev/null +++ b/dyno/.gitignore @@ -0,0 +1,12 @@ +# Logs +logs/ + +# Workspace +workspace/ + +# Docker +*.dockerfile + +# OS +.DS_Store +Thumbs.db diff --git a/dyno/Dockerfile b/dyno/Dockerfile new file mode 100644 index 0000000..d4e296a --- /dev/null +++ b/dyno/Dockerfile @@ -0,0 +1,67 @@ +# DYNO Dockerfile - Builds godex and runs it in an isolated container +# that tracks all network, file, and command executions + +FROM alpine:3.19 AS builder + +# Install build dependencies +RUN apk add --no-cache \ + bash \ + git \ + curl \ + go \ + musl-dev \ + gcc + +# Build godex from source (build context is project root) +WORKDIR /build +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . +RUN CGO_ENABLED=0 go build -ldflags="-s -w" -o godex ./cmd/godex + +# Final runtime image +FROM alpine:3.19 + +RUN apk add --no-cache \ + bash \ + python3 \ + py3-pip \ + curl \ + wget \ + strace \ + tcpdump \ + netcat-openbsd \ + iptables \ + tree \ + jq + +# Install mock HTTP server dependencies +RUN pip3 install --no-cache-dir flask requests + +# Create directories +RUN mkdir -p /dyno/logs /dyno/workspace /dyno/mocks + +# Copy built binary +COPY --from=builder /build/godex /usr/local/bin/godex + +# Copy dyno scripts +COPY dyno-mock-server.py /dyno/dyno-mock-server.py +COPY dyno-entrypoint.sh /dyno/dyno-entrypoint.sh +COPY dyno.env /dyno/dyno.env + +RUN chmod +x /dyno/*.sh /dyno/dyno-mock-server.py + +WORKDIR /dyno + +# Environment for tracking +ENV DYNO_MODE=true +ENV DYNO_LOG_DIR=/dyno/logs +ENV DYNO_MOCK_HTTP=true +ENV DYNO_MOCK_FILEOPS=true +ENV DYNO_MOCK_COMMANDS=true +ENV DYNO_TRACKING_ENABLED=true + +# Volume for outputs +VOLUME ["/dyno/logs", "/dyno/workspace"] +ENTRYPOINT ["/dyno/dyno-entrypoint.sh"] diff --git a/dyno/README.md b/dyno/README.md new file mode 100644 index 0000000..9e871c7 --- /dev/null +++ b/dyno/README.md @@ -0,0 +1,97 @@ +# DYNO - Docker-based Dynamometer Test Harness for godex + +> Like a dynamometer ("dyno") used to test cars and engines under controlled +> conditions, this harness tests godex in an isolated Docker container. + +## Overview + +DYNO runs godex in an isolated Docker container that tracks all: +- **Network operations** (HTTP requests via mock server) +- **File operations** (open, read, write, stat via strace) +- **Command executions** (system, execve) +- **Syscalls** (via strace) + +All external network interactions return **dummy responses** to simulate tool calling. + +## Quick Start + +```bash +# Build the Docker image +./dyno-run.sh build + +# Run godex with dyno +./dyno-run.sh run "your prompt here" + +# View tracking logs +./dyno-run.sh logs +``` + +## Directory Structure + +``` +dyno/ +├── Dockerfile # Docker image definition +├── dyno-run.sh # Main test harness script +├── dyno-entrypoint.sh # Container entry point +├── dyno-mock-server.py # Mock HTTP server for tool simulation +├── dyno.env # Environment configuration +├── README.md # This file +├── logs/ # Tracking logs (created at runtime) +│ ├── http_tracking.log +│ ├── file_tracking.log +│ ├── cmd_tracking.log +│ ├── syscall_tracking.log +│ └── godex_output.log +└── workspace/ # Shared workspace directory +``` + +## How It Works + +### Docker Build +- Builds godex from source code inside the Docker container +- Installs all necessary tracking tools (strace, tcpdump, etc.) +- Creates isolated environment for testing + +### Mock HTTP Server +- Runs on port 8888 inside the container +- Intercepts all HTTP requests from godex +- Returns dummy responses that simulate AI tool calling +- Logs all requests to `http_tracking.log` + +### Tracking +- **Network**: All HTTP calls logged and mocked +- **File**: strace captures file operations +- **Commands**: All execve/syscall tracked +- **Syscalls**: Full syscall trace to `syscall_tracking.log` + +## Mock Responses + +The mock server simulates tool calling by: +1. Detecting when godex makes an API call with tools +2. Returning a mock tool call response +3. Then returning a mock tool result when godex follows up + +This allows testing godex's tool-calling behavior without real API calls. + +## Environment Variables + +| Variable | Description | Default | +|----------|-------------|---------| +| `DYNO_MODE` | Enable DYNO mode | true | +| `DYNO_TRACKING_ENABLED` | Enable tracking | true | +| `DYNO_MOCK_HTTP` | Mock HTTP requests | true | +| `DYNO_MOCK_PORT` | Mock server port | 8888 | +| `DYNO_LOG_DIR` | Log directory | /dyno/logs | + +## Requirements + +- Docker +- bash +- Project source code (go.mod) + +## Notes + +- All network calls are intercepted and mocked +- No real API calls are made +- Perfect for testing and development +- Logs are persisted to host directory diff --git a/dyno/dyno-entrypoint.sh b/dyno/dyno-entrypoint.sh new file mode 100644 index 0000000..f055d42 --- /dev/null +++ b/dyno/dyno-entrypoint.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# +# dyno-entrypoint.sh - Entry point for DYNO container +# Sets up tracking environment and runs godex with mock responses +# + +set -e + +LOG_DIR="${DYNO_LOG_DIR:-/dyno/logs}" +MOCK_DIR="/dyno/mocks" +WORKSPACE="/dyno/workspace" + +# Create directories +mkdir -p "$LOG_DIR" +mkdir -p "$MOCK_DIR" +mkdir -p "$WORKSPACE" + +echo "[DYNO] Starting DYNO environment..." +echo "[DYNO] Log directory: $LOG_DIR" +echo "[DYNO] Mock directory: $MOCK_DIR" + +# ============================================================================= +# Start Mock HTTP Server (for dummy tool responses) +# ============================================================================= + +start_mock_server() { + if [ "${DYNO_MOCK_HTTP:-true}" = "true" ]; then + echo "[DYNO] Starting mock HTTP server for tool simulation..." + python3 /dyno/dyno-mock-server.py & + MOCK_PID=$! + echo "[DYNO] Mock server started (PID: $MOCK_PID)" + + # Wait for server to be ready + sleep 2 + fi +} + +# ============================================================================= +# Setup Network Interception +# ============================================================================= + +setup_network_tracking() { + echo "[DYNO] Setting up network tracking..." + + # Create iptables rules to redirect HTTP traffic to mock server + # This intercepts all outbound HTTP calls and returns mock responses + + # Log network attempts + echo "[DYNO] Network tracking enabled - all HTTP calls will be logged and mocked" + + # Set environment for godex to use mock server + export HTTP_PROXY="http://localhost:8888" + export HTTPS_PROXY="http://localhost:8888" + export NO_PROXY="localhost,127.0.0.1" +} + +# ============================================================================= +# Setup File Operation Tracking +# ============================================================================= + +setup_file_tracking() { + echo "[DYNO] Setting up file operation tracking..." + + # Use strace to track file operations + export DYNO_FILE_TRACKING=true + + echo "[DYNO] File operation tracking enabled" +} + +# ============================================================================= +# Setup Command Tracking +# ============================================================================= + +setup_cmd_tracking() { + echo "[DYNO] Setting up command execution tracking..." + + export DYNO_CMD_TRACKING=true + + echo "[DYNO] Command execution tracking enabled" +} + +# ============================================================================= +# Run godex with Tracking +# ============================================================================= + +run_godex() { + local args="${GODEX_ARGS:-}" + + echo "[DYNO] Running godex with args: $args" + + # Run godex with strace for syscall tracking + if [ "${DYNO_TRACKING_ENABLED:-true}" = "true" ]; then + echo "[DYNO] Starting godex with full tracking..." + + # Run with strace to capture all syscalls + strace -f \ + -e trace=network,file,process \ + -o "$LOG_DIR/syscall_tracking.log" \ + -s 256 \ + godex $args 2>&1 | tee "$LOG_DIR/godex_output.log" + else + godex $args + fi +} + +# ============================================================================= +# Main +# ============================================================================= + +main() { + # Setup tracking + setup_network_tracking + setup_file_tracking + setup_cmd_tracking + + # Start mock server + start_mock_server + + # Run godex + run_godex + + echo "[DYNO] Execution completed" + + # Show summary + echo "" + echo "[DYNO] === Tracking Summary ===" + echo "[DYNO] Logs available in: $LOG_DIR" + + for log in http file cmd syscall; do + local log_file="$LOG_DIR/${log}_tracking.log" + if [ -f "$log_file" ] && [ -s "$log_file" ]; then + local count=$(wc -l < "$log_file") + echo "[DYNO] $log tracking: $count entries" + fi + done +} + +main "$@" diff --git a/dyno/dyno-mock-server.py b/dyno/dyno-mock-server.py new file mode 100644 index 0000000..c11f90c --- /dev/null +++ b/dyno/dyno-mock-server.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +""" +DYNO Mock Server - Intercepts all HTTP requests and returns mock responses +to simulate tool calling for godex testing. + +This server: +- Logs all HTTP requests to tracking files +- Returns dummy responses that simulate AI tool calling +- Supports OpenAI-compatible API responses +""" + +import os +import sys +import json +import logging +import threading +from http.server import HTTPServer, BaseHTTPRequestHandler +from datetime import datetime +from urllib.parse import urlparse +import re + +# Configuration +MOCK_PORT = int(os.environ.get('DYNO_MOCK_PORT', 8888)) +LOG_FILE = os.environ.get('DYNO_HTTP_LOG', '/dyno/logs/http_tracking.log') +MOCK_MODE = os.environ.get('DYNO_MOCK_MODE', 'true').lower() == 'true' + +# Setup logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s [DYNO-MOCK] %(message)s' +) +logger = logging.getLogger('dyno-mock') + +# Request tracking +request_log_lock = threading.Lock() + +def log_request(method, url, headers, body, response_status, response_body): + """Log request to tracking file""" + timestamp = datetime.now().isoformat() + log_entry = f"[{timestamp}] {method} {url}\n" + log_entry += f" Headers: {dict(headers)}\n" + log_entry += f" Body: {body[:500] if body else 'None'}\n" + log_entry += f" Response: {response_status}\n" + log_entry += f" Response Body: {response_body[:500]}\n" + log_entry += "-" * 80 + "\n" + + os.makedirs(os.path.dirname(LOG_FILE), exist_ok=True) + with request_log_lock: + with open(LOG_FILE, 'a') as f: + f.write(log_entry) + + logger.info(f"{method} {url} -> {response_status}") + + +class MockHandler(BaseHTTPRequestHandler): + """Handler that returns mock responses for all HTTP requests""" + + protocol_version = 'HTTP/1.1' + + def log_message(self, format, *args): + logger.debug(format % args) + + def get_mock_response(self, method, path, headers=None, body=None): + """Determine mock response based on request""" + + # Check for tool result in request + if body: + try: + req_data = json.loads(body) + if 'messages' in req_data: + last_msg = req_data['messages'][-1] if req_data['messages'] else {} + if last_msg.get('role') == 'tool': + return self.mock_tool_result_response(last_msg) + except: + pass + + # OpenAI/ChatGPT style endpoints + if '/v1/chat/completions' in path or 'chat/completions' in path: + return self.mock_chat_completion(body) + + # OpenAI embeddings + if '/v1/embeddings' in path or 'embeddings' in path: + return self.mock_embeddings_response() + + # Anthropic Claude + if 'anthropic.com' in path or '/v1/messages' in path: + return self.mock_anthropic_response(body) + + # Default: generic success response + return self.mock_default_response() + + def mock_tool_result_response(self, tool_msg): + """Return mock response after tool execution""" + tool_name = tool_msg.get('name', 'unknown') + content = tool_msg.get('content', '') + + logger.info(f"Mocking tool result for: {tool_name}") + + response = { + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": f"[Tool {tool_name} executed successfully. Result: {content[:200]}]" + }, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150 + } + } + return 200, response + + def mock_chat_completion(self, body): + """Mock OpenAI chat completion - simulates tool calling""" + try: + req_data = json.loads(body) if body else {} + except: + req_data = {} + + # Check if this is a tool call request + tools = req_data.get('tools', []) + messages = req_data.get('messages', []) + + # If tools provided and no tool call yet, simulate a tool call + if tools and not any(m.get('tool_calls', []) for m in messages if isinstance(m, dict)): + # Simulate tool calling behavior + tool_name = tools[0].get('function', {}).get('name', 'example_tool') + + response = { + "id": "chatcmpl-mock-dyno-001", + "object": "chat.completion", + "created": int(datetime.now().timestamp()), + "model": req_data.get('model', 'gpt-4'), + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "tool_calls": [{ + "id": "call_mock_001", + "type": "function", + "function": { + "name": tool_name, + "arguments": json.dumps({"query": "mocked query for testing"}) + } + }] + }, + "finish_reason": "tool_calls" + }], + "usage": { + "prompt_tokens": 50, + "completion_tokens": 30, + "total_tokens": 80 + } + } + return 200, response + + # Regular chat response + response = { + "id": "chatcmpl-mock-dyno-002", + "object": "chat.completion", + "created": int(datetime.now().timestamp()), + "model": req_data.get('model', 'gpt-4'), + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "[DYNO MOCK RESPONSE] This is a simulated AI response. All network calls are being tracked and mocked for testing purposes." + }, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 50, + "completion_tokens": 50, + "total_tokens": 100 + } + } + return 200, response + + def mock_embeddings_response(self): + """Mock OpenAI embeddings response""" + response = { + "object": "list", + "data": [{ + "object": "embedding", + "embedding": [0.1] * 1536, + "index": 0 + }], + "model": "text-embedding-ada-002", + "usage": { + "prompt_tokens": 8, + "total_tokens": 8 + } + } + return 200, response + + def mock_anthropic_response(self, body): + """Mock Anthropic Claude response""" + response = { + "id": "msg_mock_001", + "type": "message", + "role": "assistant", + "content": [{ + "type": "text", + "text": "[DYNO MOCK RESPONSE] This is a simulated Claude response. All network calls are being tracked and mocked." + }], + "model": "claude-3-sonnet-20240229", + "stop_reason": "end_turn", + "usage": { + "input_tokens": 50, + "output_tokens": 50 + } + } + return 200, response + + def mock_default_response(self): + """Default mock response""" + response = { + "status": "mocked", + "message": "DYNO mock server - all network calls are simulated", + "timestamp": datetime.now().isoformat() + } + return 200, response + + def do_GET(self): + """Handle GET requests""" + path = self.path + logger.info(f"GET {path}") + + status, response = self.get_mock_response('GET', path) + + self.send_response(status) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps(response).encode()) + + def do_POST(self): + """Handle POST requests""" + path = self.path + content_length = int(self.headers.get('Content-Length', 0)) + body = self.rfile.read(content_length).decode('utf-8') if content_length > 0 else '' + + logger.info(f"POST {path}") + + status, response = self.get_mock_response('POST', path, dict(self.headers), body) + + # Log the request + log_request('POST', path, dict(self.headers), body, status, json.dumps(response)) + + self.send_response(status) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps(response).encode()) + + +def main(): + """Start the mock server""" + logger.info(f"Starting DYNO mock server on port {MOCK_PORT}") + logger.info(f"Logging to: {LOG_FILE}") + + server = HTTPServer(('0.0.0.0', MOCK_PORT), MockHandler) + logger.info(f"DYNO mock server ready at http://0.0.0.0:{MOCK_PORT}") + + try: + server.serve_forever() + except KeyboardInterrupt: + logger.info("Shutting down DYNO mock server") + server.shutdown() + + +if __name__ == '__main__': + main() diff --git a/dyno/dyno-run.sh b/dyno/dyno-run.sh new file mode 100755 index 0000000..3d8cb4a --- /dev/null +++ b/dyno/dyno-run.sh @@ -0,0 +1,82 @@ +#!/bin/bash +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" +LOG_DIR="$SCRIPT_DIR/logs" +DOCKER_IMAGE="godex-dyno:latest" +CONTAINER_NAME="godex-dyno-$$" + +log_info() { echo "[DYNO] $1"; } +log_success() { echo "[DYNO] $1"; } +log_error() { echo "[DYNO] ERROR: $1"; } + +build_dyno_image() { + log_info "Building DYNO Docker image..." + cd "$PROJECT_ROOT" +build_dyno_image() { + log_info "Building DYNO Docker image..." + cd "$SCRIPT_DIR" + docker build -t "$DOCKER_IMAGE" . + log_success "Built Docker image: $DOCKER_IMAGE" +} + log_info "Running godex in isolated DYNO container..." + log_info "Arguments: $args" + + if ! docker image inspect "$DOCKER_IMAGE" &>/dev/null; then + log_info "Docker image not found. Building..." + build_dyno_image + fi + + mkdir -p "$LOG_DIR" + + docker run --rm \ + --name "$CONTAINER_NAME" \ + --cap-add=SYS_PTRACE \ + -v "$LOG_DIR:/dyno/logs" \ + -v "$SCRIPT_DIR/workspace:/dyno/workspace" \ + -e DYNO_MODE=true \ + -e DYNO_TRACKING_ENABLED=true \ + -e DYNO_MOCK_HTTP=true \ + -e DYNO_MOCK_FILEOPS=true \ + -e DYNO_MOCK_COMMANDS=true \ + -e DYNO_LOG_DIR=/dyno/logs \ + -e GODEX_ARGS="$args" \ + "$DOCKER_IMAGE" \ + godex $args + + log_success "DYNO execution completed" +} + +show_logs() { + log_info "=== DYNO Tracking Logs ===" + for log in http file cmd syscall; do + local log_file="$LOG_DIR/${log}_tracking.log" + if [ -f "$log_file" ]; then + echo "=== $log tracking ===" + cat "$log_file" + fi + done +} + +main() { + local cmd="${1:-run}" + shift || true + + case "$cmd" in + build) + build_dyno_image + ;; + run) + run_dyno "$@" + ;; + logs) + show_logs + ;; + *) + run_dyno "$cmd $@" + ;; + esac +} + +main "$@" diff --git a/dyno/dyno.env b/dyno/dyno.env new file mode 100644 index 0000000..d0bc105 --- /dev/null +++ b/dyno/dyno.env @@ -0,0 +1,48 @@ +# ============================================================================= +# DYNO Docker Environment Configuration +# ============================================================================= + +# Enable DYNO mode +DYNO_MODE=true +DYNO_TRACKING_ENABLED=true + +# Mock HTTP responses (simulate tool calling) +DYNO_MOCK_HTTP=true +DYNO_MOCK_HTTP_STATUS=200 +DYNO_MOCK_HTTP_BODY='{"status": "mocked", "data": {"result": "success"}}' +DYNO_MOCK_HTTP_HEADERS='Content-Type: application/json' +DYNO_MOCK_PORT=8888 + +# Mock file operations +DYNO_MOCK_FILEOPS=true +DYNO_MOCK_FILE_CONTENT="[MOCKED FILE CONTENT]" +DYNO_MOCK_FILE_EXISTS=true +DYNO_MOCK_DIR_CONTENT="mock_file1.txt,mock_file2.txt,mock_dir/" + +# Mock command executions +DYNO_MOCK_COMMANDS=true +DYNO_MOCK_CMD_EXITCODE=0 +DYNO_MOCK_CMD_STDOUT="[MOCKED COMMAND OUTPUT]" +DYNO_MOCK_CMD_STDERR="" + +# Mock syscalls +DYNO_MOCK_SYSCALLS=true +DYNO_MOCK_SYSCALL_RESULT=0 + +# Tracking log paths +DYNO_LOG_DIR=/dyno/logs +DYNO_HTTP_LOG=$DYNO_LOG_DIR/http_tracking.log +DYNO_FILE_LOG=$DYNO_LOG_DIR/file_tracking.log +DYNO_CMD_LOG=$DYNO_LOG_DIR/cmd_tracking.log +DYNO_SYSCALL_LOG=$DYNO_LOG_DIR/syscall_tracking.log + +# Network isolation +DYNO_NETWORK_ISOLATED=true +DYNO_ALLOWED_HOSTS="localhost,127.0.0.1" + +# Filesystem isolation +DYNO_FS_ISOLATED=true +DYNO_ALLOWED_PATHS="/tmp,/dyno" + +# Workspace +DYNO_WORKSPACE=/dyno/workspace