Upload folder using huggingface_hub

Dockerfile

@@ -0,0 +1,81 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# Multi-stage build using openenv-base
+# This Dockerfile is flexible and works for both:
+# - In-repo environments (with local OpenEnv sources)
+# - Standalone environments (with openenv from PyPI/Git)
+# The build script (openenv build) handles context detection and sets appropriate build args.
+
+ARG BASE_IMAGE=ghcr.io/meta-pytorch/openenv-base:latest
+FROM ${BASE_IMAGE} AS builder
+
+WORKDIR /app
+
+# Ensure git is available (required for installing dependencies from VCS)
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends git && \
+    rm -rf /var/lib/apt/lists/*
+
+# Build argument to control whether we're building standalone or in-repo
+ARG BUILD_MODE=in-repo
+ARG ENV_NAME=benchmark
+
+# Copy environment code (always at root of build context)
+COPY . /app/env
+
+# For in-repo builds, openenv is already vendored in the build context
+# For standalone builds, openenv will be installed via pyproject.toml
+WORKDIR /app/env
+
+# Ensure uv is available (for local builds where base image lacks it)
+RUN if ! command -v uv >/dev/null 2>&1; then \
+        curl -LsSf https://astral.sh/uv/install.sh | sh && \
+        mv /root/.local/bin/uv /usr/local/bin/uv && \
+        mv /root/.local/bin/uvx /usr/local/bin/uvx; \
+    fi
+    
+# Install dependencies using uv sync
+# If uv.lock exists, use it; otherwise resolve on the fly
+RUN --mount=type=cache,target=/root/.cache/uv \
+    if [ -f uv.lock ]; then \
+        uv sync --frozen --no-install-project --no-editable; \
+    else \
+        uv sync --no-install-project --no-editable; \
+    fi
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    if [ -f uv.lock ]; then \
+        uv sync --frozen --no-editable; \
+    else \
+        uv sync --no-editable; \
+    fi
+
+# Final runtime stage
+FROM ${BASE_IMAGE}
+
+WORKDIR /app
+
+# Copy the virtual environment from builder
+COPY --from=builder /app/env/.venv /app/.venv
+
+# Copy the environment code
+COPY --from=builder /app/env /app/env
+
+# Set PATH to use the virtual environment
+ENV PATH="/app/.venv/bin:$PATH"
+
+# Set PYTHONPATH so imports work correctly
+ENV PYTHONPATH="/app/env:$PYTHONPATH"
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
+    CMD curl -f http://localhost:8000/health || exit 1
+
+# Run the FastAPI server
+# The module path is constructed to work with the /app/env structure
+ENV ENABLE_WEB_INTERFACE=true
+CMD ["sh", "-c", "cd /app/env && uvicorn server.app:app --host 0.0.0.0 --port 8000"]

README.md

@@ -1,10 +1,255 @@
 ---
-title: Openenv Benchmark Ws
-emoji: 🐢
-colorFrom: purple
-colorTo: indigo
+title: Benchmark Environment Server
+emoji: 🎸
+colorFrom: red
+colorTo: red
 sdk: docker
 pinned: false
+app_port: 8000
+base_path: /web
+tags:
+  - openenv
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Benchmark Environment
+
+A simple test environment that echoes back messages. Perfect for testing the env APIs as well as demonstrating environment usage patterns.
+
+## Quick Start
+
+The simplest way to use the Benchmark environment is through the `BenchmarkEnv` class:
+
+```python
+from benchmark import BenchmarkAction, BenchmarkEnv
+
+try:
+    # Create environment from Docker image
+    benchmarkenv = BenchmarkEnv.from_docker_image("benchmark-env:latest")
+
+    # Reset
+    result = benchmarkenv.reset()
+    print(f"Reset: {result.observation.echoed_message}")
+
+    # Send multiple messages
+    messages = ["Hello, World!", "Testing echo", "Final message"]
+
+    for msg in messages:
+        result = benchmarkenv.step(BenchmarkAction(message=msg))
+        print(f"Sent: '{msg}'")
+        print(f"  → Echoed: '{result.observation.echoed_message}'")
+        print(f"  → Length: {result.observation.message_length}")
+        print(f"  → Reward: {result.reward}")
+
+finally:
+    # Always clean up
+    benchmarkenv.close()
+```
+
+That's it! The `BenchmarkEnv.from_docker_image()` method handles:
+- Starting the Docker container
+- Waiting for the server to be ready
+- Connecting to the environment
+- Container cleanup when you call `close()`
+
+## Building the Docker Image
+
+Before using the environment, you need to build the Docker image:
+
+```bash
+# From project root
+docker build -t benchmark-env:latest -f server/Dockerfile .
+```
+
+## Deploying to Hugging Face Spaces
+
+You can easily deploy your OpenEnv environment to Hugging Face Spaces using the `openenv push` command:
+
+```bash
+# From the environment directory (where openenv.yaml is located)
+openenv push
+
+# Or specify options
+openenv push --namespace my-org --private
+```
+
+The `openenv push` command will:
+1. Validate that the directory is an OpenEnv environment (checks for `openenv.yaml`)
+2. Prepare a custom build for Hugging Face Docker space (enables web interface)
+3. Upload to Hugging Face (ensuring you're logged in)
+
+### Prerequisites
+
+- Authenticate with Hugging Face: The command will prompt for login if not already authenticated
+
+### Options
+
+- `--directory`, `-d`: Directory containing the OpenEnv environment (defaults to current directory)
+- `--repo-id`, `-r`: Repository ID in format 'username/repo-name' (defaults to 'username/env-name' from openenv.yaml)
+- `--base-image`, `-b`: Base Docker image to use (overrides Dockerfile FROM)
+- `--private`: Deploy the space as private (default: public)
+
+### Examples
+
+```bash
+# Push to your personal namespace (defaults to username/env-name from openenv.yaml)
+openenv push
+
+# Push to a specific repository
+openenv push --repo-id my-org/my-env
+
+# Push with a custom base image
+openenv push --base-image ghcr.io/meta-pytorch/openenv-base:latest
+
+# Push as a private space
+openenv push --private
+
+# Combine options
+openenv push --repo-id my-org/my-env --base-image custom-base:latest --private
+```
+
+After deployment, your space will be available at:
+`https://huggingface.co/spaces/<repo-id>`
+
+The deployed space includes:
+- **Web Interface** at `/web` - Interactive UI for exploring the environment
+- **API Documentation** at `/docs` - Full OpenAPI/Swagger interface
+- **Health Check** at `/health` - Container health monitoring
+- **WebSocket** at `/ws` - Persistent session endpoint for low-latency interactions
+
+## Environment Details
+
+### Action
+**BenchmarkAction**: Contains a single field
+- `message` (str) - The message to echo back
+
+### Observation
+**BenchmarkObservation**: Contains the echo response and metadata
+- `echoed_message` (str) - The message echoed back
+- `message_length` (int) - Length of the message
+- `reward` (float) - Reward based on message length (length × 0.1)
+- `done` (bool) - Always False for echo environment
+- `metadata` (dict) - Additional info like step count
+
+### Reward
+The reward is calculated as: `message_length × 0.1`
+- "Hi" → reward: 0.2
+- "Hello, World!" → reward: 1.3
+- Empty message → reward: 0.0
+
+## Advanced Usage
+
+### Connecting to an Existing Server
+
+If you already have a Benchmark environment server running, you can connect directly:
+
+```python
+from benchmark import BenchmarkEnv
+
+# Connect to existing server
+benchmarkenv = BenchmarkEnv(base_url="<ENV_HTTP_URL_HERE>")
+
+# Use as normal
+result = benchmarkenv.reset()
+result = benchmarkenv.step(BenchmarkAction(message="Hello!"))
+```
+
+Note: When connecting to an existing server, `benchmarkenv.close()` will NOT stop the server.
+
+### WebSocket Client for Persistent Sessions
+
+For long-running episodes or when you need lower latency, use the WebSocket client:
+
+```python
+from benchmark import BenchmarkAction, BenchmarkEnvWS
+
+# Connect via WebSocket (maintains persistent connection)
+with BenchmarkEnvWS(base_url="http://localhost:8000") as env:
+    result = env.reset()
+    print(f"Reset: {result.observation.echoed_message}")
+    # Multiple steps with low latency
+    for msg in ["Hello", "World", "!"]:
+        result = env.step(BenchmarkAction(message=msg))
+        print(f"Echoed: {result.observation.echoed_message}")
+```
+
+WebSocket advantages:
+- **Lower latency**: No HTTP connection overhead per request
+- **Persistent session**: Server maintains your environment state
+- **Efficient for episodes**: Better for many sequential steps
+
+### Concurrent WebSocket Sessions
+
+The server supports multiple concurrent WebSocket connections. To enable this,
+modify `server/app.py` to use factory mode:
+
+```python
+# In server/app.py - use factory mode for concurrent sessions
+app = create_app(
+    BenchmarkEnvironment,  # Pass class, not instance
+    BenchmarkAction,
+    BenchmarkObservation,
+    max_concurrent_envs=4,  # Allow 4 concurrent sessions
+)
+```
+
+Then multiple clients can connect simultaneously:
+
+```python
+from benchmark import BenchmarkAction, BenchmarkEnvWS
+from concurrent.futures import ThreadPoolExecutor
+
+def run_episode(client_id: int):
+    with BenchmarkEnvWS(base_url="http://localhost:8000") as env:
+        result = env.reset()
+        for i in range(10):
+            result = env.step(BenchmarkAction(message=f"Client {client_id}, step {i}"))
+        return client_id, result.observation.message_length
+
+# Run 4 episodes concurrently
+with ThreadPoolExecutor(max_workers=4) as executor:
+    results = list(executor.map(run_episode, range(4)))
+```
+
+## Development & Testing
+
+### Direct Environment Testing
+
+Test the environment logic directly without starting the HTTP server:
+
+```bash
+# From the server directory
+python3 server/benchmark_environment.py
+```
+
+This verifies that:
+- Environment resets correctly
+- Step executes actions properly
+- State tracking works
+- Rewards are calculated correctly
+
+### Running Locally
+
+Run the server locally for development:
+
+```bash
+uvicorn server.app:app --reload
+```
+
+## Project Structure
+
+```
+benchmark/
+├── .dockerignore         # Docker build exclusions
+├── __init__.py            # Module exports
+├── README.md              # This file
+├── openenv.yaml           # OpenEnv manifest
+├── pyproject.toml         # Project metadata and dependencies
+├── uv.lock                # Locked dependencies (generated)
+├── client.py              # BenchmarkEnv (HTTP) and BenchmarkEnvWS (WebSocket) clients
+├── models.py              # Action and Observation models
+└── server/
+    ├── __init__.py        # Server module exports
+    ├── benchmark_environment.py  # Core environment logic
+    ├── app.py             # FastAPI application (HTTP + WebSocket endpoints)
+    └── Dockerfile         # Container image definition
+```

__init__.py

@@ -0,0 +1,17 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Benchmark Environment - Test environment for concurrency and infrastructure."""
+
+from .client import BenchmarkEnv, BenchmarkEnvWS
+from .models import BenchmarkAction, BenchmarkObservation
+
+__all__ = [
+    "BenchmarkAction",
+    "BenchmarkObservation",
+    "BenchmarkEnv",
+    "BenchmarkEnvWS",
+]

build/lib/benchmark/__init__.py

@@ -0,0 +1,17 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Benchmark Environment - A simple test environment for HTTP server."""
+
+from .client import BenchmarkEnv, BenchmarkEnvWS
+from .models import BenchmarkAction, BenchmarkObservation
+
+__all__ = [
+    "BenchmarkAction",
+    "BenchmarkObservation",
+    "BenchmarkEnv",
+    "BenchmarkEnvWS",
+]

build/lib/benchmark/client.py

@@ -0,0 +1,189 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Benchmark Environment Clients.
+
+This module provides clients for connecting to a Benchmark Environment server:
+- BenchmarkEnv: HTTP client for request/response interactions
+- BenchmarkEnvWS: WebSocket client for persistent sessions
+"""
+
+from typing import Any, Dict
+
+from openenv.core.client_types import StepResult
+from openenv.core.env_server.types import State
+from openenv.core.http_env_client import HTTPEnvClient
+from openenv.core.ws_env_client import WebSocketEnvClient
+
+from .models import BenchmarkAction, BenchmarkObservation
+
+
+class BenchmarkEnv(HTTPEnvClient[BenchmarkAction, BenchmarkObservation]):
+    """
+    HTTP client for the Benchmark Environment.
+
+    This client connects to a BenchmarkEnvironment HTTP server and provides
+    methods to interact with it: reset(), step(), and state access.
+
+    Example:
+        >>> # Connect to a running server
+        >>> client = BenchmarkEnv(base_url="http://localhost:8000")
+        >>> result = client.reset()
+        >>> print(result.observation.echoed_message)
+        >>>
+        >>> # Send a message
+        >>> result = client.step(BenchmarkAction(message="Hello!"))
+        >>> print(result.observation.echoed_message)
+        >>> print(result.reward)
+
+    Example with Docker:
+        >>> # Automatically start container and connect
+        >>> client = BenchmarkEnv.from_docker_image("benchmark-env:latest")
+        >>> result = client.reset()
+        >>> result = client.step(BenchmarkAction(message="Test"))
+    """
+
+    def _step_payload(self, action: BenchmarkAction) -> Dict:
+        """
+        Convert BenchmarkAction to JSON payload for step request.
+
+        Args:
+            action: BenchmarkAction instance
+
+        Returns:
+            Dictionary representation suitable for JSON encoding
+        """
+        return {
+            "message": action.message,
+        }
+
+    def _parse_result(self, payload: Dict) -> StepResult[BenchmarkObservation]:
+        """
+        Parse server response into StepResult[BenchmarkObservation].
+
+        Args:
+            payload: JSON response from server
+
+        Returns:
+            StepResult with BenchmarkObservation
+        """
+        obs_data = payload.get("observation", {})
+        observation = BenchmarkObservation(
+            echoed_message=obs_data.get("echoed_message", ""),
+            message_length=obs_data.get("message_length", 0),
+            done=payload.get("done", False),
+            reward=payload.get("reward"),
+            metadata=obs_data.get("metadata", {}),
+        )
+
+        return StepResult(
+            observation=observation,
+            reward=payload.get("reward"),
+            done=payload.get("done", False),
+        )
+
+    def _parse_state(self, payload: Dict) -> State:
+        """
+        Parse server response into State object.
+
+        Args:
+            payload: JSON response from /state endpoint
+
+        Returns:
+            State object with episode_id and step_count
+        """
+        return State(
+            episode_id=payload.get("episode_id"),
+            step_count=payload.get("step_count", 0),
+        )
+
+
+class BenchmarkEnvWS(WebSocketEnvClient[BenchmarkAction, BenchmarkObservation]):
+    """
+    WebSocket client for the Benchmark Environment.
+
+    This client maintains a persistent WebSocket connection to the environment server,
+    enabling efficient multi-step interactions with lower latency than HTTP.
+    Each client instance has its own dedicated environment session on the server.
+
+    Advantages over HTTP client:
+    - Lower latency for sequential interactions (no connection overhead per request)
+    - Session state is maintained server-side
+    - Better suited for long-running episodes
+
+    Example:
+        >>> # Connect to a running server via WebSocket
+        >>> with BenchmarkEnvWS(base_url="http://localhost:8000") as client:
+        ...     result = client.reset()
+        ...     print(result.observation.echoed_message)
+        ...
+        ...     result = client.step(BenchmarkAction(message="Hello!"))
+        ...     print(result.observation.echoed_message)
+
+    Example with Docker:
+        >>> # Automatically start container and connect via WebSocket
+        >>> client = BenchmarkEnvWS.from_docker_image("benchmark-env:latest")
+        >>> try:
+        ...     result = client.reset()
+        ...     result = client.step(BenchmarkAction(message="Test"))
+        ... finally:
+        ...     client.close()
+    """
+
+    def _step_payload(self, action: BenchmarkAction) -> Dict:
+        """
+        Convert BenchmarkAction to JSON payload for step message.
+
+        Args:
+            action: BenchmarkAction instance
+
+        Returns:
+            Dictionary representation suitable for JSON encoding
+        """
+        return {
+            "message": action.message,
+        }
+
+    def _parse_result(self, payload: Dict) -> StepResult[BenchmarkObservation]:
+        """
+        Parse WebSocket response into StepResult[BenchmarkObservation].
+
+        Args:
+            payload: JSON response data from server
+
+        Returns:
+            StepResult with BenchmarkObservation
+        """
+        obs_data = payload.get("observation", {})
+        observation = BenchmarkObservation(
+            echoed_message=obs_data.get("echoed_message", ""),
+            message_length=obs_data.get("message_length", 0),
+            done=payload.get("done", False),
+            reward=payload.get("reward"),
+            metadata=obs_data.get("metadata", {}),
+        )
+
+        return StepResult(
+            observation=observation,
+            reward=payload.get("reward"),
+            done=payload.get("done", False),
+        )
+
+    def _parse_state(self, payload: Dict) -> State:
+        """
+        Parse WebSocket state response into State object.
+
+        Args:
+            payload: JSON response from state request
+
+        Returns:
+            State object with episode_id and step_count
+        """
+        return State(
+            episode_id=payload.get("episode_id"),
+            step_count=payload.get("step_count", 0),
+        )

build/lib/benchmark/models.py

@@ -0,0 +1,31 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Data models for the Benchmark Environment.
+
+The benchmark environment is a simple test environment that echoes back messages.
+"""
+
+from dataclasses import dataclass
+
+from openenv.core.env_server.types import Action, Observation
+
+
+@dataclass(kw_only=True)
+class BenchmarkAction(Action):
+    """Action for the Benchmark environment - just a message to echo."""
+
+    message: str
+
+
+@dataclass(kw_only=True)
+class BenchmarkObservation(Observation):
+    """Observation from the Benchmark environment - the echoed message."""
+
+    echoed_message: str
+    message_length: int = 0
+

build/lib/benchmark/server/__init__.py

@@ -0,0 +1,12 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Benchmark environment server components."""
+
+from .benchmark_environment import BenchmarkEnvironment
+
+__all__ = ["BenchmarkEnvironment"]
+

build/lib/benchmark/server/app.py

@@ -0,0 +1,80 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+FastAPI application for the Benchmark Environment.
+
+This module creates an HTTP server that exposes the BenchmarkEnvironment
+over HTTP and WebSocket endpoints, compatible with HTTPEnvClient and WebSocketEnvClient.
+
+Endpoints:
+    - POST /reset: Reset the environment
+    - POST /step: Execute an action
+    - GET /state: Get current environment state
+    - GET /schema: Get action/observation schemas
+    - WS /ws: WebSocket endpoint for persistent sessions
+
+Usage:
+    # Development (with auto-reload):
+    uvicorn server.app:app --reload --host 0.0.0.0 --port 8000
+
+    # Production:
+    uvicorn server.app:app --host 0.0.0.0 --port 8000 --workers 4
+
+    # Or run directly:
+    python -m server.app
+"""
+
+try:
+    from openenv.core.env_server.http_server import create_app
+except Exception as e:  # pragma: no cover
+    raise ImportError(
+        "openenv is required for the web interface. Install dependencies with '\n    uv sync\n'"
+    ) from e
+
+from benchmark.models import BenchmarkAction, BenchmarkObservation
+from .benchmark_environment import BenchmarkEnvironment
+
+
+# Create the app with web interface and README integration
+app = create_app(
+    BenchmarkEnvironment,
+    BenchmarkAction,
+    BenchmarkObservation,
+    env_name="benchmark",
+    max_concurrent_envs=1,  # increase this number to allow more concurrent WebSocket sessions
+)
+
+
+def main(host: str = "0.0.0.0", port: int = 8000):
+    """
+    Entry point for direct execution via uv run or python -m.
+
+    This function enables running the server without Docker:
+        uv run --project . server
+        uv run --project . server --port 8001
+        python -m benchmark.server.app
+
+    Args:
+        host: Host address to bind to (default: "0.0.0.0")
+        port: Port number to listen on (default: 8000)
+
+    For production deployments, consider using uvicorn directly with
+    multiple workers:
+        uvicorn benchmark.server.app:app --workers 4
+    """
+    import uvicorn
+
+    uvicorn.run(app, host=host, port=port)
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--port", type=int, default=8000)
+    args = parser.parse_args()
+    main(port=args.port)

build/lib/benchmark/server/benchmark_environment.py

@@ -0,0 +1,101 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Benchmark Environment Implementation.
+
+A simple test environment that echoes back messages sent to it.
+Perfect for testing HTTP server infrastructure.
+"""
+
+from uuid import uuid4
+
+from openenv.core.env_server.interfaces import Environment
+from openenv.core.env_server.types import State
+
+from models import BenchmarkAction, BenchmarkObservation
+
+
+class BenchmarkEnvironment(Environment):
+    """
+    A simple echo environment that echoes back messages.
+
+    This environment is designed for testing the HTTP server infrastructure.
+    It maintains minimal state and simply echoes back whatever message it receives.
+
+    Example:
+        >>> env = BenchmarkEnvironment()
+        >>> obs = env.reset()
+        >>> print(obs.echoed_message)  # "Benchmark environment ready!"
+        >>>
+        >>> obs = env.step(BenchmarkAction(message="Hello"))
+        >>> print(obs.echoed_message)  # "Hello"
+        >>> print(obs.message_length)  # 5
+    """
+
+    # Enable concurrent WebSocket sessions.
+    # Set to True if your environment isolates state between instances.
+    # When True, multiple WebSocket clients can connect simultaneously, each
+    # getting their own environment instance (when using factory mode in app.py).
+    CONCURRENCY_SAFE: bool = True
+
+    def __init__(self):
+        """Initialize the benchmark environment."""
+        self._state = State(episode_id=str(uuid4()), step_count=0)
+        self._reset_count = 0
+
+    def reset(self) -> BenchmarkObservation:
+        """
+        Reset the environment.
+
+        Returns:
+            BenchmarkObservation with a ready message
+        """
+        self._state = State(episode_id=str(uuid4()), step_count=0)
+        self._reset_count += 1
+
+        return BenchmarkObservation(
+            echoed_message="Benchmark environment ready!",
+            message_length=0,
+            done=False,
+            reward=0.0,
+        )
+
+    def step(self, action: BenchmarkAction) -> BenchmarkObservation:  # type: ignore[override]
+        """
+        Execute a step in the environment by echoing the message.
+
+        Args:
+            action: BenchmarkAction containing the message to echo
+
+        Returns:
+            BenchmarkObservation with the echoed message and its length
+        """
+        self._state.step_count += 1
+
+        message = action.message
+        length = len(message)
+
+        # Simple reward: longer messages get higher rewards
+        reward = length * 0.1
+
+        return BenchmarkObservation(
+            echoed_message=message,
+            message_length=length,
+            done=False,
+            reward=reward,
+            metadata={"original_message": message, "step": self._state.step_count},
+        )
+
+    @property
+    def state(self) -> State:
+        """
+        Get the current environment state.
+
+        Returns:
+            Current State with episode_id and step_count
+        """
+        return self._state

client.py

@@ -0,0 +1,82 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Benchmark Environment Clients.
+
+Provides HTTP and WebSocket clients for the Benchmark Environment.
+"""
+
+from typing import Dict
+
+from openenv.core.client_types import StepResult
+from openenv.core.env_server.types import State
+from openenv.core.http_env_client import HTTPEnvClient
+from openenv.core.ws_env_client import WebSocketEnvClient
+
+from .models import BenchmarkAction, BenchmarkObservation
+
+
+class BenchmarkEnv(HTTPEnvClient[BenchmarkAction, BenchmarkObservation]):
+    """HTTP client for the Benchmark Environment."""
+
+    def _step_payload(self, action: BenchmarkAction) -> Dict:
+        return {"wait_seconds": action.wait_seconds}
+
+    def _parse_result(self, payload: Dict) -> StepResult[BenchmarkObservation]:
+        obs_data = payload.get("observation", {})
+        observation = BenchmarkObservation(
+            host_url=obs_data.get("host_url", ""),
+            pid=obs_data.get("pid", 0),
+            session_hash=obs_data.get("session_hash", ""),
+            waited_seconds=obs_data.get("waited_seconds", 0.0),
+            timestamp=obs_data.get("timestamp", 0.0),
+            done=payload.get("done", False),
+            reward=payload.get("reward"),
+            metadata=obs_data.get("metadata", {}),
+        )
+        return StepResult(
+            observation=observation,
+            reward=payload.get("reward"),
+            done=payload.get("done", False),
+        )
+
+    def _parse_state(self, payload: Dict) -> State:
+        return State(
+            episode_id=payload.get("episode_id"),
+            step_count=payload.get("step_count", 0),
+        )
+
+
+class BenchmarkEnvWS(WebSocketEnvClient[BenchmarkAction, BenchmarkObservation]):
+    """WebSocket client for the Benchmark Environment."""
+
+    def _step_payload(self, action: BenchmarkAction) -> Dict:
+        return {"wait_seconds": action.wait_seconds}
+
+    def _parse_result(self, payload: Dict) -> StepResult[BenchmarkObservation]:
+        obs_data = payload.get("observation", {})
+        observation = BenchmarkObservation(
+            host_url=obs_data.get("host_url", ""),
+            pid=obs_data.get("pid", 0),
+            session_hash=obs_data.get("session_hash", ""),
+            waited_seconds=obs_data.get("waited_seconds", 0.0),
+            timestamp=obs_data.get("timestamp", 0.0),
+            done=payload.get("done", False),
+            reward=payload.get("reward"),
+            metadata=obs_data.get("metadata", {}),
+        )
+        return StepResult(
+            observation=observation,
+            reward=payload.get("reward"),
+            done=payload.get("done", False),
+        )
+
+    def _parse_state(self, payload: Dict) -> State:
+        return State(
+            episode_id=payload.get("episode_id"),
+            step_count=payload.get("step_count", 0),
+        )

models.py

@@ -0,0 +1,35 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Data models for the Benchmark Environment.
+
+The benchmark environment is designed for testing concurrency and infrastructure.
+Actions specify a wait time in seconds, allowing testing of parallel execution.
+"""
+
+from pydantic import Field
+
+from openenv.core.env_server.types import Action, Observation
+
+
+class BenchmarkAction(Action):
+    """Action for the Benchmark environment - specifies seconds to wait."""
+
+    wait_seconds: float = Field(default=0.0, ge=0.0, description="Seconds to wait/sleep")
+
+
+class BenchmarkObservation(Observation):
+    """Observation from the Benchmark environment with server identity info."""
+
+    # Server identity
+    host_url: str = Field(default="", description="URL of the server that handled the request")
+    pid: int = Field(default=0, description="Process ID of the server")
+    session_hash: str = Field(default="", description="Unique hash identifying this server session")
+
+    # Timing info
+    waited_seconds: float = Field(default=0.0, description="Actual seconds waited")
+    timestamp: float = Field(default=0.0, description="Unix timestamp when observation was created")

openenv.yaml

@@ -0,0 +1,7 @@
+spec_version: 1
+name: benchmark
+type: space
+runtime: fastapi
+app: server.app:app
+port: 8000
+

openenv_benchmark.egg-info/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: openenv-benchmark
+Version: 0.1.0
+Summary: Benchmark environment for OpenEnv
+Requires-Python: >=3.10
+Requires-Dist: openenv[core]>=0.2.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"

openenv_benchmark.egg-info/SOURCES.txt

@@ -0,0 +1,18 @@
+README.md
+__init__.py
+client.py
+models.py
+pyproject.toml
+./__init__.py
+./client.py
+./models.py
+./test_concurrency.py
+openenv_benchmark.egg-info/PKG-INFO
+openenv_benchmark.egg-info/SOURCES.txt
+openenv_benchmark.egg-info/dependency_links.txt
+openenv_benchmark.egg-info/entry_points.txt
+openenv_benchmark.egg-info/requires.txt
+openenv_benchmark.egg-info/top_level.txt
+server/__init__.py
+server/app.py
+server/benchmark_environment.py

openenv_benchmark.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

openenv_benchmark.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+server = benchmark.server.app:main

openenv_benchmark.egg-info/requires.txt

@@ -0,0 +1,5 @@
+openenv[core]>=0.2.0
+
+[dev]
+pytest>=8.0.0
+pytest-cov>=4.0.0

openenv_benchmark.egg-info/top_level.txt

@@ -0,0 +1 @@
+benchmark

pyproject.toml

@@ -0,0 +1,43 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+[build-system]
+requires = ["setuptools>=45", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "openenv-benchmark"
+version = "0.1.0"
+description = "Benchmark environment for OpenEnv"
+requires-python = ">=3.10"
+dependencies = [
+    # Core OpenEnv runtime (provides FastAPI server + HTTP client types)
+    "openenv[core] @ git+https://github.com/meta-pytorch/OpenEnv.git@ws-in-cli",
+    # Environment-specific dependencies
+    # Add all dependencies needed for your environment here
+    # Examples:
+    # "numpy>=1.19.0",
+    # "torch>=2.0.0",
+    # "gymnasium>=0.29.0",
+    # "openspiel>=1.0.0",
+    # "smolagents>=1.22.0,<2",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0.0",
+    "pytest-cov>=4.0.0",
+]
+
+[project.scripts]
+# Server entry point - enables running via: uv run --project . server
+# or: python -m benchmark.server.app
+server = "benchmark.server.app:main"
+
+[tool.setuptools]
+include-package-data = true
+packages = ["benchmark", "benchmark.server"]
+package-dir = { "benchmark" = ".", "benchmark.server" = "server" }

server/__init__.py

@@ -0,0 +1,12 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Benchmark environment server components."""
+
+from .benchmark_environment import BenchmarkEnvironment
+
+__all__ = ["BenchmarkEnvironment"]
+

server/app.py

@@ -0,0 +1,82 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+FastAPI application for the Benchmark Environment.
+
+This module creates an HTTP server that exposes the BenchmarkEnvironment
+over HTTP and WebSocket endpoints, compatible with HTTPEnvClient and WebSocketEnvClient.
+
+Endpoints:
+    - POST /reset: Reset the environment
+    - POST /step: Execute an action
+    - GET /state: Get current environment state
+    - GET /schema: Get action/observation schemas
+    - WS /ws: WebSocket endpoint for persistent sessions
+
+Usage:
+    # Development (with auto-reload):
+    uvicorn server.app:app --reload --host 0.0.0.0 --port 8000
+
+    # Production:
+    uvicorn server.app:app --host 0.0.0.0 --port 8000 --workers 4
+
+    # Or run directly:
+    python -m server.app
+"""
+
+try:
+    from openenv.core.env_server.http_server import create_app
+except Exception as e:  # pragma: no cover
+    raise ImportError(
+        "openenv is required for the web interface. Install dependencies with '\n    uv sync\n'"
+    ) from e
+
+from benchmark.models import BenchmarkAction, BenchmarkObservation
+from .benchmark_environment import BenchmarkEnvironment
+
+
+# Create the app with web interface and README integration
+# max_concurrent_envs controls how many simultaneous WebSocket sessions can exist
+# Each session gets its own BenchmarkEnvironment instance
+app = create_app(
+    BenchmarkEnvironment,
+    BenchmarkAction,
+    BenchmarkObservation,
+    env_name="benchmark",
+    max_concurrent_envs=1000,  # Allow many concurrent WebSocket sessions
+)
+
+
+def main(host: str = "0.0.0.0", port: int = 8000):
+    """
+    Entry point for direct execution via uv run or python -m.
+
+    This function enables running the server without Docker:
+        uv run --project . server
+        uv run --project . server --port 8001
+        python -m benchmark.server.app
+
+    Args:
+        host: Host address to bind to (default: "0.0.0.0")
+        port: Port number to listen on (default: 8000)
+
+    For production deployments, consider using uvicorn directly with
+    multiple workers:
+        uvicorn benchmark.server.app:app --workers 4
+    """
+    import uvicorn
+
+    uvicorn.run(app, host=host, port=port)
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--port", type=int, default=8000)
+    args = parser.parse_args()
+    main(port=args.port)

server/benchmark_environment.py

@@ -0,0 +1,118 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Benchmark Environment Implementation.
+
+A test environment for benchmarking infrastructure and concurrency.
+Actions specify how many seconds to wait, allowing testing of parallel execution.
+"""
+
+import asyncio
+import hashlib
+import os
+import socket
+import time
+from uuid import uuid4
+
+from openenv.core.env_server.interfaces import Environment
+from openenv.core.env_server.types import State
+
+from models import BenchmarkAction, BenchmarkObservation
+
+
+def _get_host_url() -> str:
+    """Get the host URL for this server."""
+    hostname = socket.gethostname()
+    port = os.environ.get("PORT", "8000")
+    try:
+        ip = socket.gethostbyname(hostname)
+    except socket.gaierror:
+        ip = "127.0.0.1"
+    return f"http://{ip}:{port}"
+
+
+class BenchmarkEnvironment(Environment):
+    """
+    A benchmark environment for testing concurrency and infrastructure.
+
+    Actions specify a number of seconds to wait (sleep), which is useful for
+    testing parallel execution and concurrency limits. The environment returns
+    identity information (host_url, pid, session_hash) to help verify which
+    server instance handled the request.
+
+    Example:
+        >>> env = BenchmarkEnvironment()
+        >>> obs = env.reset()
+        >>> print(obs.host_url)  # "http://192.168.1.1:8000"
+        >>> print(obs.pid)  # 12345
+        >>> print(obs.session_hash)  # "a1b2c3d4..."
+        >>>
+        >>> obs = env.step(BenchmarkAction(wait_seconds=2.0))
+        >>> print(obs.waited_seconds)  # 2.0
+    """
+
+    # Enable concurrent WebSocket sessions
+    CONCURRENCY_SAFE: bool = True
+
+    def __init__(self):
+        """Initialize the benchmark environment."""
+        self._state = State(episode_id=str(uuid4()), step_count=0)
+        self._session_hash = hashlib.sha256(
+            f"{uuid4()}-{time.time()}-{os.getpid()}".encode()
+        ).hexdigest()[:16]
+        self._pid = os.getpid()
+        self._host_url = _get_host_url()
+
+    def _make_observation(
+        self, waited_seconds: float = 0.0, done: bool = False, reward: float = 0.0
+    ) -> BenchmarkObservation:
+        """Create an observation with current server identity."""
+        return BenchmarkObservation(
+            host_url=self._host_url,
+            pid=self._pid,
+            session_hash=self._session_hash,
+            waited_seconds=waited_seconds,
+            timestamp=time.time(),
+            done=done,
+            reward=reward,
+        )
+
+    def reset(self) -> BenchmarkObservation:
+        """Reset the environment."""
+        self._state = State(episode_id=str(uuid4()), step_count=0)
+        return self._make_observation(waited_seconds=0.0, done=False, reward=0.0)
+
+    def step(self, action: BenchmarkAction) -> BenchmarkObservation:
+        """Execute a step by waiting for the specified seconds (sync)."""
+        self._state.step_count += 1
+        wait_time = max(0.0, action.wait_seconds)
+
+        if wait_time > 0:
+            time.sleep(wait_time)
+
+        reward = 1.0 / (1.0 + wait_time)
+        return self._make_observation(
+            waited_seconds=wait_time, done=False, reward=reward
+        )
+
+    async def step_async(self, action: BenchmarkAction) -> BenchmarkObservation:
+        """Execute a step by waiting for the specified seconds (async)."""
+        self._state.step_count += 1
+        wait_time = max(0.0, action.wait_seconds)
+
+        if wait_time > 0:
+            await asyncio.sleep(wait_time)
+
+        reward = 1.0 / (1.0 + wait_time)
+        return self._make_observation(
+            waited_seconds=wait_time, done=False, reward=reward
+        )
+
+    @property
+    def state(self) -> State:
+        """Get the current environment state."""
+        return self._state

server/requirements.txt

@@ -0,0 +1,6 @@
+openenv[core]>=0.2.0
+fastapi>=0.115.0
+uvicorn>=0.24.0
+
+
+

test_concurrency.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+"""
+Concurrency test for benchmark environment using WebSockets.
+
+Each WebSocket connection gets its own dedicated environment session,
+enabling true concurrent execution across multiple sessions.
+
+Run the server first:
+    cd benchmark && uvicorn server.app:app --port 8000
+
+Then run this script:
+    python test_concurrency.py --requests 100 --wait 1.0
+    python test_concurrency.py -n 100 -w 1 --url wss://your-server.hf.space
+"""
+
+import argparse
+import asyncio
+import json
+import time
+from dataclasses import dataclass
+
+try:
+    import websockets
+except ImportError:
+    print("Install websockets: pip install websockets")
+    raise
+
+
+@dataclass
+class RequestResult:
+    """Result from a single WebSocket request."""
+
+    request_id: int
+    wait_requested: float
+    waited_seconds: float
+    elapsed: float
+    pid: int
+    session_hash: str
+    host_url: str
+
+
+def convert_to_ws_url(url: str) -> str:
+    """Convert HTTP URL to WebSocket URL."""
+    url = url.rstrip("/")
+    if url.startswith("http://"):
+        return "ws://" + url[7:] + "/ws"
+    elif url.startswith("https://"):
+        return "wss://" + url[8:] + "/ws"
+    elif url.startswith("ws://") or url.startswith("wss://"):
+        return url if url.endswith("/ws") else url + "/ws"
+    return "ws://" + url + "/ws"
+
+
+async def ws_session(
+    ws_url: str,
+    request_id: int,
+    wait_seconds: float,
+    timeout: float = 60.0,
+) -> RequestResult:
+    """
+    Run a complete WebSocket session: connect, reset, step, close.
+
+    Each session gets its own environment instance on the server.
+    """
+    start = time.perf_counter()
+
+    async with websockets.connect(ws_url, open_timeout=timeout) as ws:
+        # Reset to initialize environment
+        await ws.send(json.dumps({"type": "reset", "data": {}}))
+        reset_response = json.loads(await asyncio.wait_for(ws.recv(), timeout))
+
+        if reset_response.get("type") == "error":
+            raise RuntimeError(f"Reset error: {reset_response}")
+
+        # Step with wait time
+        await ws.send(
+            json.dumps({
+                "type": "step",
+                "data": {"wait_seconds": wait_seconds},
+            })
+        )
+        step_response = json.loads(await asyncio.wait_for(ws.recv(), timeout))
+
+        if step_response.get("type") == "error":
+            raise RuntimeError(f"Step error: {step_response}")
+
+        # Close cleanly
+        await ws.send(json.dumps({"type": "close"}))
+
+    elapsed = time.perf_counter() - start
+    obs = step_response.get("data", {}).get("observation", {})
+
+    return RequestResult(
+        request_id=request_id,
+        wait_requested=wait_seconds,
+        waited_seconds=obs.get("waited_seconds", 0.0),
+        elapsed=elapsed,
+        pid=obs.get("pid", 0),
+        session_hash=obs.get("session_hash", ""),
+        host_url=obs.get("host_url", ""),
+    )
+
+
+async def run_concurrent_test(
+    url: str,
+    num_requests: int,
+    wait_seconds: float,
+    timeout: float = 120.0,
+) -> dict:
+    """Run concurrent WebSocket sessions and collect results."""
+    ws_url = convert_to_ws_url(url)
+
+    print(f"WebSocket URL: {ws_url}")
+    print(f"Running {num_requests} concurrent sessions, each waiting {wait_seconds}s...")
+    print()
+
+    start = time.perf_counter()
+
+    # Launch all sessions concurrently
+    tasks = [
+        ws_session(ws_url, i, wait_seconds, timeout) for i in range(num_requests)
+    ]
+
+    results = await asyncio.gather(*tasks, return_exceptions=True)
+
+    total_time = time.perf_counter() - start
+
+    # Process results
+    successful = [r for r in results if isinstance(r, RequestResult)]
+    failed = [r for r in results if isinstance(r, Exception)]
+
+    if not successful:
+        print("All requests failed!")
+        for i, err in enumerate(failed[:5]):
+            print(f"  Error {i}: {err}")
+        return {"error": "All requests failed"}
+
+    avg_time = sum(r.elapsed for r in successful) / len(successful)
+    unique_pids = set(r.pid for r in successful)
+    unique_sessions = set(r.session_hash for r in successful)
+    unique_hosts = set(r.host_url for r in successful)
+
+    return {
+        "num_requests": num_requests,
+        "successful": len(successful),
+        "failed": len(failed),
+        "wait_seconds": wait_seconds,
+        "total_time": total_time,
+        "avg_time": avg_time,
+        "unique_pids": len(unique_pids),
+        "unique_sessions": len(unique_sessions),
+        "unique_hosts": len(unique_hosts),
+        "pids": list(unique_pids)[:10],  # First 10 for display
+    }
+
+
+async def main():
+    parser = argparse.ArgumentParser(
+        description="Test benchmark environment concurrency via WebSocket"
+    )
+    parser.add_argument(
+        "--requests", "-n", type=int, default=10,
+        help="Number of concurrent WebSocket sessions"
+    )
+    parser.add_argument(
+        "--wait", "-w", type=float, default=1.0,
+        help="Wait time per request (seconds)"
+    )
+    parser.add_argument(
+        "--url", "-u", type=str, default="http://localhost:8000",
+        help="Server URL (http/https/ws/wss)"
+    )
+    parser.add_argument(
+        "--timeout", "-t", type=float, default=120.0,
+        help="Timeout per request (seconds)"
+    )
+    args = parser.parse_args()
+
+    result = await run_concurrent_test(
+        args.url, args.requests, args.wait, args.timeout
+    )
+
+    if "error" in result:
+        return
+
+    print(f"Successful:      {result['successful']}/{result['num_requests']}")
+    if result["failed"]:
+        print(f"Failed:          {result['failed']}")
+    print(f"Total time:      {result['total_time']:.3f}s")
+    print(f"Avg time:        {result['avg_time']:.3f}s")
+    print(f"Unique PIDs:     {result['unique_pids']}")
+    print(f"Unique sessions: {result['unique_sessions']}")
+    print(f"Unique hosts:    {result['unique_hosts']}")
+
+    # Calculate concurrency metrics
+    ideal_time = args.wait
+    actual_concurrency = (args.requests * args.wait) / result["total_time"]
+    print()
+    print(f"Ideal time (full concurrency): {ideal_time:.3f}s")
+    print(f"Effective concurrency: {actual_concurrency:.1f}x")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+