{ "reviewed_at": "2026-10-16", "entries": [ { "id": "gpt5-asteroid-game", "name": "Asteroid Game", "description_en": "A browser space-shooter demo from OpenAI's single-prompt GPT-5 coding collection, with the generating prompt and published app files.", "description_zh": "OpenAI 单次提示生成作品集中的浏览器太空射击游戏，公开生成提示和应用文件。", "content_type": "game", "platforms": [ "web" ], "uses": [ "showcase" ], "implementation": "ai-built", "primary_url": "https://github.com/openai/gpt-5-coding-examples/tree/main/apps/asteroid-game", "repo_url": "https://github.com/openai/gpt-5-coding-examples", "demo_url": "https://gpt5-coding-examples.vercel.app/asteroid-game/", "paper_url": null, "limits_en": "An illustrative coding demo, with no published agent task evaluator. The public demo URL responded; gameplay was not tested.", "limits_zh": "属于编程演示，未提供 Agent 任务评分器。已检查公开演示链接，未进行游戏操作测试。", "enables_en": "Inspect and remix an AI-generated arcade game, using its original prompt as a starting point.", "enables_zh": "查看和改造 AI 生成的街机游戏，并参考原始提示进行再创作。", "resource_kind": "ai-generated-output", "provenance_source": "https://github.com/openai/gpt-5-coding-examples#readme", "evidence": [ { "url": "https://github.com/openai/gpt-5-coding-examples#readme", "supports": "OpenAI states that the collection was generated from single GPT-5 or GPT-5.2 prompts without handwritten code." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/examples/asteroid-game.yaml", "supports": "Identifies this example and its space-shooter prompt." }, { "url": "https://gpt5-coding-examples.vercel.app/asteroid-game/", "supports": "Public app endpoint returned HTTP 200; no interaction test was performed." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "gpt5-tiny-kanban", "name": "Tiny Kanban", "description_en": "A generated browser app for moving task cards between Todo, Doing and Done, with local persistence, search and work-in-progress limits.", "description_zh": "AI 生成的浏览器看板应用，支持卡片流转、本地保存、搜索和进行中任务数量限制。", "content_type": "app", "platforms": [ "web" ], "uses": [ "showcase" ], "implementation": "ai-built", "primary_url": "https://github.com/openai/gpt-5-coding-examples/tree/main/apps/tiny-kanban", "repo_url": "https://github.com/openai/gpt-5-coding-examples", "demo_url": "https://gpt5-coding-examples.vercel.app/tiny-kanban/", "paper_url": null, "limits_en": "The published implementation stores data in browser localStorage. One synthetic card creation, title search and same-browser reload were observed separately; drag/drop and WIP enforcement were not tested. It is a demo, not a documented multi-user service or benchmark.", "limits_zh": "公开实现使用浏览器 localStorage 保存数据。已单独记录一次合成卡片创建、标题搜索和同一浏览器重载；未测试拖放或 WIP 限制。属于演示项目，未声明多人协作服务或评测能力。", "enables_en": "Inspect how an AI-generated single-file app implements a useful task-management workflow.", "enables_zh": "观察 AI 生成的单文件应用如何实现实际的任务管理流程。", "resource_kind": "ai-generated-output", "provenance_source": "https://github.com/openai/gpt-5-coding-examples#readme", "evidence": [ { "url": "https://github.com/openai/gpt-5-coding-examples#readme", "supports": "The publisher explicitly attributes the collection to single-prompt AI code generation." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/apps/tiny-kanban/index.html", "supports": "Source contains card dragging, search, WIP checks and localStorage." }, { "url": "https://gpt5-coding-examples.vercel.app/tiny-kanban/", "supports": "Public demo with separately recorded synthetic card creation, title search and one same-browser reload. Other behavior was not tested." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "claudes-c-compiler", "name": "Claude's C Compiler", "description_en": "An experimental C compiler written in Rust by Claude Opus 4.6, released by Anthropic as an artifact of its agent-team research.", "description_zh": "Anthropic 发布的 Agent 团队研究产物：由 Claude Opus 4.6 编写、以 Rust 实现的实验性 C 编译器。", "content_type": "app", "platforms": [ "linux" ], "uses": [ "showcase" ], "implementation": "ai-built", "primary_url": "https://www.anthropic.com/engineering/building-c-compiler", "repo_url": "https://github.com/anthropics/claudes-c-compiler", "demo_url": null, "paper_url": null, "limits_en": "The human-authored repository notice warns that correctness is unvalidated and discourages use. Linux is the documented host; this is a research artifact.", "limits_zh": "仓库中的人工说明明确表示正确性未经验证，并不建议实际使用。文档要求 Linux 主机，应视作研究产物。", "enables_en": "Study the source and development history of a substantial AI-authored systems program.", "enables_zh": "研究较大规模 AI 编写系统软件的源码和开发历史。", "resource_kind": "ai-generated-output", "provenance_source": "https://github.com/anthropics/claudes-c-compiler#readme", "evidence": [ { "url": "https://github.com/anthropics/claudes-c-compiler#readme", "supports": "The human-authored notice attributes code and documentation to Claude, with human-written tests and Linux-only development." }, { "url": "https://www.anthropic.com/engineering/building-c-compiler", "supports": "The author describes the agent-team process and its limitations." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "fastrender", "name": "FastRender", "description_en": "An experimental Rust browser engine and desktop shell produced during Cursor's research on long-running, collaborating coding agents.", "description_zh": "Cursor 长时间多 Agent 编程研究产生的实验性 Rust 浏览器引擎及桌面外壳。", "content_type": "app", "platforms": [ "macos", "linux" ], "uses": [ "showcase" ], "implementation": "ai-built", "primary_url": "https://github.com/wilsonzlin/fastrender", "repo_url": "https://github.com/wilsonzlin/fastrender", "demo_url": null, "paper_url": null, "limits_en": "The authors describe unstable APIs and incomplete web APIs, and advise against production use. Platform tags follow the documented macOS/Linux build instructions.", "limits_zh": "作者说明 API 不稳定、部分 Web API 尚不完整，不建议用于生产。平台标签依据 macOS/Linux 构建文档。", "enables_en": "Inspect an AI-authored browser implementation and its rendering pipeline as a research example.", "enables_zh": "以研究案例形式查看 AI 编写的浏览器实现及其渲染流程。", "resource_kind": "ai-generated-output", "provenance_source": "https://cursor.com/blog/scaling-agents", "evidence": [ { "url": "https://cursor.com/blog/scaling-agents", "supports": "Cursor identifies the linked repository as a browser built by its long-running coding agents." }, { "url": "https://github.com/wilsonzlin/fastrender#readme", "supports": "The repository links the agent research, documents macOS/Linux builds, and states incomplete support and production limitations." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "crafter", "name": "Crafter", "description_en": "A procedurally generated survival game for studying exploration, resource collection and crafting, with achievement-based agent evaluation.", "description_zh": "程序化生成的生存游戏，通过资源收集、制作和生存成就研究探索与长期决策能力。", "content_type": "game", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/danijar/crafter", "repo_url": "https://github.com/danijar/crafter", "demo_url": null, "paper_url": "https://arxiv.org/abs/2109.06780", "limits_en": "A simplified 2D research world. Python and a Pygame human interface are documented; the reviewed sources do not enumerate tested operating systems. Craftax-Classic is a reimplementation and full Craftax changes the mechanics; results are not interchangeable.", "limits_zh": "属于简化的二维研究环境。文档提供 Python 和 Pygame 人类界面，但未列出已测试的操作系统。 Craftax-Classic 是重实现，完整 Craftax 则改变了机制，结果不能直接互换。", "enables_en": "Train agents and measure which survival and crafting achievements they can unlock.", "enables_zh": "训练 Agent，并观察其能解锁哪些生存和制作成就。", "resource_kind": "environment-and-benchmark", "runtime": "Python; optional Pygame GUI", "platform_note": "No OS tag was inferred from the Python implementation.", "evidence": [ { "url": "https://github.com/danijar/crafter#readme", "supports": "Documents the playable environment, Python API, achievements and GUI installation." }, { "url": "https://arxiv.org/abs/2109.06780", "supports": "Original paper defines the training and achievement-based evaluation setting." }, { "url": "https://github.com/MichaelTMatthews/Craftax#readme", "supports": "Distinguishes Craftax-Classic, which reimplements Crafter, from the expanded full Craftax environment." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "textworld", "name": "TextWorld", "description_en": "A toolkit for generating and playing text adventures with controlled task difficulty, state tracking and rewards for learning agents.", "description_zh": "可生成和运行文字冒险的工具包，支持控制任务难度、跟踪状态并为学习型 Agent 提供奖励。", "content_type": "game", "platforms": [ "linux", "macos" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/microsoft/TextWorld", "repo_url": "https://github.com/microsoft/TextWorld", "demo_url": null, "paper_url": "https://arxiv.org/abs/1806.11532", "limits_en": "Text interaction rather than a visual desktop. Native support is documented for Linux/macOS; Windows requires a Docker workaround.", "limits_zh": "交互以文字为主。文档声明原生支持 Linux/macOS；Windows 需采用 Docker 方案。", "enables_en": "Create controlled text-game curricula and test generalization across related adventures.", "enables_zh": "构建难度可控的文字游戏课程，评估 Agent 在相关游戏间的泛化能力。", "resource_kind": "environment-framework", "evidence": [ { "url": "https://github.com/microsoft/TextWorld#readme", "supports": "Documents game generation, RL use, Python support and operating-system constraints." }, { "url": "https://arxiv.org/abs/1806.11532", "supports": "Original paper explains controllable generated games and learning/evaluation use." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "minihack", "name": "MiniHack", "description_en": "A NetHack-based sandbox for building compact game tasks, from navigation and object use to procedurally generated challenges.", "description_zh": "基于 NetHack 的任务沙盒，可构建导航、物品使用以及程序化生成的游戏挑战。", "content_type": "game", "platforms": [ "linux", "macos" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/NetHack-LE/minihack", "repo_url": "https://github.com/NetHack-LE/minihack", "demo_url": null, "paper_url": "https://arxiv.org/abs/2109.13202", "limits_en": "Requires NetHack Learning Environment dependencies. Windows uses Docker. The public level editor is an authoring tool, not a hosted benchmark run.", "limits_zh": "依赖 NetHack Learning Environment；Windows 使用 Docker。公开关卡编辑器是任务制作工具，并非在线评测服务。", "enables_en": "Design controlled game tasks to test exploration, planning and skill composition.", "enables_zh": "设计可控游戏任务，研究探索、规划和技能组合。", "resource_kind": "environment-framework", "authoring_demo_url": "https://minihack-editor.github.io/", "evidence": [ { "url": "https://github.com/NetHack-LE/minihack#readme", "supports": "Current author-maintained repository documents NetHack-based tasks and the Gymnasium API." }, { "url": "https://github.com/NetHack-LE/minihack/blob/main/docs/getting-started/installation.md", "supports": "Documents macOS/Ubuntu dependency guidance and the Windows Docker path." }, { "url": "https://arxiv.org/abs/2109.13202", "supports": "Original paper defines MiniHack as a configurable RL sandbox." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "webarena", "name": "WebArena", "description_en": "Self-hostable websites and task evaluators for agents working across shopping, forums, content management and software collaboration.", "description_zh": "可自行部署的网站及任务评分器，覆盖购物、论坛、内容管理和软件协作等网页操作。", "content_type": "website", "platforms": [ "web" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://webarena.dev/", "repo_url": "https://github.com/web-arena-x/webarena", "demo_url": null, "paper_url": "https://arxiv.org/abs/2307.13854", "limits_en": "Reproducible evaluation requires self-hosted services and controlled state resets; shared demos do not replace that setup. WebArena-Verified separately revises the same task family and evaluator. Original/Verified and full/Hard task counts are not additive, and evaluator versions must accompany reported scores.", "limits_zh": "可复现评测需要自行部署服务并控制状态重置，共享演示不能替代。WebArena-Verified 单独修订同一任务族及评分器。原始／Verified 及完整／Hard 数量不能相加，报告成绩应注明评分器版本。", "enables_en": "Measure completion of multi-step web tasks using executable checks of the resulting state.", "enables_zh": "通过可执行的结果检查，评估 Agent 是否完成多步骤网页任务。", "resource_kind": "environment-and-benchmark", "evidence": [ { "url": "https://github.com/web-arena-x/webarena#readme", "supports": "Canonical implementation documents self-hosting, task execution and the distinction between demos and reproducible experiments." }, { "url": "https://arxiv.org/abs/2307.13854", "supports": "Original paper describes functional websites and task-success evaluation." }, { "url": "https://github.com/ServiceNow/webarena-verified/blob/6473f72db5dcefc97b5725b59e734504edc28a21/README.md", "supports": "WebArena-Verified revises the same task family; evaluator and subset counts are not additive." }, { "url": "https://servicenow.github.io/webarena-verified/", "supports": "WebArena-Verified revises the same task family; evaluator and subset counts are not additive." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "visualwebarena", "name": "VisualWebArena", "description_en": "A web-agent benchmark whose shopping, classifieds and forum tasks require visual information alongside text.", "description_zh": "结合图像和文字信息的网页 Agent 基准，涵盖购物、分类信息和论坛任务。", "content_type": "website", "platforms": [ "web" ], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://jykoh.com/vwa", "repo_url": "https://github.com/web-arena-x/visualwebarena", "demo_url": null, "paper_url": "https://arxiv.org/abs/2401.13649", "limits_en": "Requires hosting the provided sites and matching the task configuration. The repository documents Python-version constraints; no hosted evaluation service is claimed.", "limits_zh": "需要部署配套站点并匹配任务配置；仓库还列有 Python 版本限制。此处不提供托管评测服务。", "enables_en": "Test whether an agent can use images and page content to finish realistic web tasks.", "enables_zh": "评估 Agent 能否结合图像与页面内容完成实际网页任务。", "resource_kind": "environment-and-benchmark", "evidence": [ { "url": "https://github.com/web-arena-x/visualwebarena#readme", "supports": "Documents the website setup, visual tasks, task configurations and execution-based evaluation." }, { "url": "https://arxiv.org/abs/2401.13649", "supports": "Original paper explains why the tasks require multimodal web interaction." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "workarena", "name": "WorkArena", "description_en": "A ServiceNow-based task suite for evaluating agents on workplace workflows such as searching knowledge bases, filling forms and filtering records.", "description_zh": "基于 ServiceNow 的办公任务套件，用于评估知识库搜索、表单填写和记录筛选等能力。", "content_type": "website", "platforms": [ "web" ], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/ServiceNow/WorkArena", "repo_url": "https://github.com/ServiceNow/WorkArena", "demo_url": null, "paper_url": "https://arxiv.org/abs/2403.07718", "limits_en": "ServiceNow instances currently require approved access through the authors' gated dataset. WorkArena++ extends the original suite; task versions must be reported.", "limits_zh": "当前需通过作者的受限数据集申请 ServiceNow 实例访问。WorkArena++ 扩展了原始套件，报告结果时应注明版本。", "enables_en": "Evaluate structured enterprise workflows with task-specific validation.", "enables_zh": "使用任务专用验证器评估企业软件中的结构化操作流程。", "resource_kind": "benchmark-suite", "related_paper_urls": [ "https://arxiv.org/abs/2407.05291" ], "evidence": [ { "url": "https://github.com/ServiceNow/WorkArena#readme", "supports": "Documents task families, gated instance access and the WorkArena/WorkArena++ relationship." }, { "url": "https://arxiv.org/abs/2403.07718", "supports": "Original paper introduces the ServiceNow knowledge-work benchmark." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "browsergym", "name": "BrowserGym", "description_en": "A shared browser environment interface that connects agents to multiple web benchmarks and exposes consistent observations and actions.", "description_zh": "连接多种网页基准的统一浏览器环境接口，为 Agent 提供一致的观察和动作表示。", "content_type": "website", "platforms": [ "web" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/ServiceNow/BrowserGym", "repo_url": "https://github.com/ServiceNow/BrowserGym", "demo_url": null, "paper_url": "https://arxiv.org/abs/2412.05467", "limits_en": "A research framework rather than a new website task collection. Each integrated benchmark keeps its own setup, access and evaluation requirements.", "limits_zh": "这是研究框架，并非另一套独立网页任务；接入的各个基准仍有各自的部署、访问和评测要求。", "enables_en": "Reuse agent integrations and compare behavior across supported web environments.", "enables_zh": "复用 Agent 接口，并比较它在不同网页环境中的行为。", "resource_kind": "environment-framework", "evidence": [ { "url": "https://github.com/ServiceNow/BrowserGym#readme", "supports": "Documents the common interface, benchmark integrations and per-benchmark setup." }, { "url": "https://arxiv.org/abs/2412.05467", "supports": "The ecosystem paper explains standardized browser observations, actions and evaluation." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "osworld", "name": "OSWorld", "description_en": "A desktop-agent environment with application workflows, file operations and executable task checks; the original paper introduced 369 tasks.", "description_zh": "包含应用操作、文件处理和可执行任务检查的桌面 Agent 环境，原始论文发布了 369 项任务。", "content_type": "app", "platforms": [ "linux", "windows" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/xlang-ai/OSWorld", "repo_url": "https://github.com/xlang-ai/OSWorld", "demo_url": null, "paper_url": "https://arxiv.org/abs/2404.07972", "limits_en": "Requires VM images and supported providers. This entry covers original OSWorld and OSWorld-Verified; OSWorld 2.0/2.1 is a separate long-horizon benchmark with different tasks and scores. Identify the version when comparing results. Running on a macOS host does not imply macOS task coverage.", "limits_zh": "需要虚拟机镜像及受支持的运行方式。本条目覆盖原始 OSWorld 和 OSWorld-Verified；OSWorld 2.0／2.1 是任务及评分口径不同的独立长流程基准。比较结果时需注明版本。Mac 主机支持不等于 macOS 任务覆盖。", "enables_en": "Evaluate multi-application computer use against final application and file states.", "enables_zh": "通过应用和文件的最终状态，评估跨应用电脑操作能力。", "resource_kind": "environment-and-benchmark", "platform_note": "Tags follow the Ubuntu/Windows guest choices in the reviewed repository, rather than the broader OS list in the original abstract.", "evidence": [ { "url": "https://github.com/xlang-ai/OSWorld#readme", "supports": "Documents VM setup, Ubuntu/Windows guest choices and OSWorld-Verified version changes." }, { "url": "https://arxiv.org/abs/2404.07972", "supports": "Original paper describes application tasks and execution-based checks." }, { "url": "https://osworld-v1.xlang.ai/", "supports": "The original official site links the OSWorld 2.0 announcement; this does not change the original task set." }, { "url": "https://osworld-v2.xlang.ai/", "supports": "Official OSWorld 2.0 release page distinguishes long-horizon workflows and links the new paper and code." }, { "url": "https://github.com/xlang-ai/OSWorld-V2", "supports": "Separate official repository for the second release and its additional web services." }, { "url": "https://arxiv.org/abs/2606.29537", "supports": "Primary paper identifies 108 long-horizon tasks; first arXiv submission was 2026-06-28, distinct from the website announcement date." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "androidworld", "name": "AndroidWorld", "description_en": "An Android emulator environment with parameterized tasks and state-based rewards across everyday mobile applications.", "description_zh": "运行于 Android 模拟器的环境，为日常移动应用提供参数化任务和基于系统状态的奖励。", "content_type": "app", "platforms": [ "android" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://google-research.github.io/android_world/", "repo_url": "https://github.com/google-research/android_world", "demo_url": null, "paper_url": "https://arxiv.org/abs/2405.14573", "limits_en": "Requires a configured Android emulator and the prescribed apps. Host compatibility differs from the Android target; Apple Silicon Docker setup has documented caveats.", "limits_zh": "需要配置 Android 模拟器及指定应用。主机兼容性与 Android 目标平台应分开理解，Apple Silicon 的 Docker 方案有已知限制。", "enables_en": "Train or evaluate agents on repeatable mobile workflows with varied task parameters.", "enables_zh": "通过不同任务参数，训练或评估可重复的移动端操作流程。", "resource_kind": "environment-and-benchmark", "evidence": [ { "url": "https://github.com/google-research/android_world#readme", "supports": "Documents the live emulator, dynamic tasks, reward signals and setup constraints." }, { "url": "https://arxiv.org/abs/2405.14573", "supports": "Original paper describes parameterized workflows and system-state evaluation." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "windows-agent-arena", "name": "Windows Agent Arena", "description_en": "A Windows desktop-agent benchmark and environment for application, browser and operating-system tasks, with local and Azure execution paths.", "description_zh": "面向 Windows 应用、浏览器和操作系统任务的桌面 Agent 基准与环境，提供本地及 Azure 运行方式。", "content_type": "app", "platforms": [ "windows" ], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://microsoft.github.io/WindowsAgentArena/", "repo_url": "https://github.com/microsoft/WindowsAgentArena", "demo_url": null, "paper_url": "https://arxiv.org/abs/2409.08264", "limits_en": "Setup requires a Windows VM image, Docker and dependencies. The documented evaluation ISO is a time-limited trial; Azure execution can incur costs.", "limits_zh": "需要 Windows 虚拟机镜像、Docker 和配套依赖。文档使用有期限的评估版 ISO；Azure 运行可能产生费用。", "enables_en": "Assess Windows GUI task completion with repeatable initial states and evaluators.", "enables_zh": "使用可重复的初始状态和评分器，评估 Windows 图形界面任务完成情况。", "resource_kind": "environment-and-benchmark", "evidence": [ { "url": "https://github.com/microsoft/WindowsAgentArena#readme", "supports": "Documents the Windows environment, Docker/VM setup and local/cloud execution." }, { "url": "https://arxiv.org/abs/2409.08264", "supports": "Original paper defines the Windows task suite and scalable evaluation approach." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "iosworld", "name": "iOSWorld", "description_en": "An iPhone Simulator benchmark with 26 purpose-built apps, a shared fictional user profile and tasks that span apps and personal context.", "description_zh": "基于 iPhone 模拟器的基准，包含 26 个专门构建的应用、统一虚构用户数据，以及跨应用和个人背景相关任务。", "content_type": "app", "platforms": [ "ios" ], "uses": [ "evaluation" ], "implementation": "ai-assisted", "primary_url": "https://github.com/ljang0/iOSWorld", "repo_url": "https://github.com/ljang0/iOSWorld", "demo_url": null, "paper_url": "https://arxiv.org/abs/2606.09764", "limits_en": "Requires macOS, Xcode and an iOS simulator. These are seeded research apps, not App Store products; the default scoring path uses a model judge.", "limits_zh": "需要 macOS、Xcode 和 iOS 模拟器。应用使用预置研究数据，并非 App Store 产品；默认评分流程采用模型评审。", "enables_en": "Evaluate cross-app phone tasks and reasoning over a consistent synthetic personal history.", "enables_zh": "评估跨应用手机任务，以及对一致的合成个人历史进行推理的能力。", "resource_kind": "environment-and-benchmark", "provenance_source": "https://arxiv.org/html/2606.09764", "evidence": [ { "url": "https://arxiv.org/html/2606.09764", "supports": "The authors state that apps were developed or adapted with Claude Code assistance and manually verified by human developers." }, { "url": "https://github.com/ljang0/iOSWorld#readme", "supports": "Documents runnable SwiftUI apps, benchmark runners, simulator requirements and default judge behavior." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "macarena", "name": "MacArena", "description_en": "A macOS computer-use benchmark combining ported desktop tasks with macOS-specific workflows in an Apple Silicon virtual-machine environment.", "description_zh": "面向 macOS 的电脑操作基准，在 Apple Silicon 虚拟机环境中组合移植桌面任务与 macOS 特有流程。", "content_type": "app", "platforms": [ "macos" ], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/MacPaw/MacArena", "repo_url": "https://github.com/MacPaw/MacArena", "demo_url": null, "paper_url": "https://arxiv.org/abs/2606.06560", "limits_en": "Requires Apple hardware and macOS VM setup. Tasks inherited from macOSWorld carry CC BY-NC 4.0 restrictions; application and OS licenses also apply.", "limits_zh": "需要 Apple 硬件和 macOS 虚拟机配置。继承自 macOSWorld 的任务采用 CC BY-NC 4.0，另需遵守系统和应用许可。", "enables_en": "Test whether desktop-agent skills transfer to native macOS interfaces and workflows.", "enables_zh": "检验桌面 Agent 能力能否迁移到原生 macOS 界面及操作流程。", "resource_kind": "environment-and-benchmark", "evidence": [ { "url": "https://github.com/MacPaw/MacArena#readme", "supports": "Documents UTM/VM setup, task sources and source-specific license restrictions." }, { "url": "https://arxiv.org/abs/2606.06560", "supports": "Original paper describes the Apple Silicon environment and macOS-specific evaluation." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "gpt5-brutalist-dev-platform", "name": "Brutalist Dev Platform", "description_en": "An AI-generated developer-platform landing page with a monochrome grid, animated terminal, code tabs, pricing table and signup dialog.", "description_zh": "AI 生成的开发者平台落地页，包含黑白网格、终端动画、代码标签页、价格表和注册弹窗。", "content_type": "website", "platforms": [ "web" ], "uses": [ "showcase" ], "implementation": "ai-built", "primary_url": "https://github.com/openai/gpt-5-coding-examples/tree/main/apps/brutalist-dev-landing-page", "repo_url": "https://github.com/openai/gpt-5-coding-examples", "demo_url": "https://gpt5-coding-examples.vercel.app/brutalist-dev-landing-page/", "paper_url": null, "limits_en": "A fictional marketing-site demo, not an operating developer service or agent benchmark. The public page responded; its interactions were not independently tested.", "limits_zh": "这是虚构产品的营销网站演示，并非实际开发者服务或 Agent 基准。公开页面可访问，未独立测试交互。", "enables_en": "Inspect a prompt and its generated HTML to study a complete landing-page design.", "enables_zh": "对照原始提示与生成的 HTML，研究完整落地页的设计和实现。", "resource_kind": "ai-generated-output", "provenance_source": "https://github.com/openai/gpt-5-coding-examples#readme", "evidence": [ { "url": "https://github.com/openai/gpt-5-coding-examples#readme", "supports": "OpenAI attributes this collection to single GPT-5 or GPT-5.2 prompts without handwritten code." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/examples/brutalist-dev-landing-page.yaml", "supports": "The example's original prompt identifies its landing-page design and requested interactions." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/apps/brutalist-dev-landing-page/index.html", "supports": "Published source contains the terminal animation, code copying and signup dialog." }, { "url": "https://gpt5-coding-examples.vercel.app/brutalist-dev-landing-page/", "supports": "Public demo returned HTTP 200; no interactive or service-functionality test was performed." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "gpt5-smartwatch-product-page", "name": "Smartwatch Product Page", "description_en": "An AI-generated product-launch website for a fictional smartwatch, with a specification section, product options and a simulated preorder form.", "description_zh": "AI 生成的虚构智能手表发布网站，包含规格介绍、产品选项和模拟预订表单。", "content_type": "website", "platforms": [ "web" ], "uses": [ "showcase" ], "implementation": "ai-built", "primary_url": "https://github.com/openai/gpt-5-coding-examples/tree/main/apps/product-launch-page", "repo_url": "https://github.com/openai/gpt-5-coding-examples", "demo_url": "https://gpt5-coding-examples.vercel.app/product-launch-page/", "paper_url": null, "limits_en": "The source stores simulated preorders in browser localStorage; no real sale or payment service is documented. The page responded, but the form was not exercised.", "limits_zh": "源码将模拟预订保存在浏览器 localStorage，未提供真实销售或支付服务。页面可访问，但未提交或操作表单。", "enables_en": "Study how a single prompt becomes a product-marketing page with a client-side form.", "enables_zh": "研究单次提示如何生成包含前端表单的产品营销页面。", "resource_kind": "ai-generated-output", "provenance_source": "https://github.com/openai/gpt-5-coding-examples#readme", "evidence": [ { "url": "https://github.com/openai/gpt-5-coding-examples#readme", "supports": "The publisher explicitly attributes the collection to AI generation." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/examples/product-launch-page.yaml", "supports": "Original example specifies the product-launch page and preorder form." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/apps/product-launch-page/index.html", "supports": "Source labels the page a concept demo and implements preorder storage with localStorage." }, { "url": "https://gpt5-coding-examples.vercel.app/product-launch-page/", "supports": "Public demo returned HTTP 200; no form submission or interaction test was performed." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/LICENSE", "supports": "Collection publishes an MIT license." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "webshop", "name": "WebShop", "description_en": "A simulated shopping website where agents search, inspect and select products to satisfy natural-language requests, with HTML and simplified text interfaces.", "description_zh": "模拟购物网站，Agent 根据自然语言要求搜索、查看并选择商品，提供 HTML 和简化文本两种接口。", "content_type": "website", "platforms": [ "web" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/princeton-nlp/WebShop", "repo_url": "https://github.com/princeton-nlp/WebShop", "demo_url": null, "paper_url": "https://arxiv.org/abs/2207.01206", "limits_en": "Requires product data, Python/Java dependencies and a search index. The default preview loads a product subset; report the chosen dataset configuration when comparing results.", "limits_zh": "需要商品数据、Python/Java 依赖和搜索索引。默认预览仅加载部分商品，比较结果时应说明数据配置。", "enables_en": "Train shopping agents and evaluate whether their final product selection matches the requested attributes.", "enables_zh": "训练购物 Agent，并评估最终商品选择是否符合用户要求的属性。", "resource_kind": "environment-and-benchmark", "evidence": [ { "url": "https://github.com/princeton-nlp/WebShop#readme", "supports": "Documents shopping tasks, HTML/text interfaces, setup and the default 1,000-product configuration." }, { "url": "https://arxiv.org/abs/2207.01206", "supports": "Original paper introduces grounded shopping interaction and product-selection rewards." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "miniwob-plusplus", "name": "MiniWoB++", "description_en": "A collection of compact browser tasks for clicking, typing, selecting and manipulating interface elements, exposed through Gymnasium and JavaScript interfaces.", "description_zh": "由点击、输入、选择和操作界面元素等小型浏览器任务组成，提供 Gymnasium 与 JavaScript 接口。", "content_type": "website", "platforms": [ "web" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://miniwob.farama.org/", "repo_url": "https://github.com/Farama-Foundation/miniwob-plusplus", "demo_url": null, "paper_url": "https://arxiv.org/abs/1802.08802", "limits_en": "Synthetic tasks isolate interaction skills rather than full production workflows. The project is in maintenance mode; its Python interface documents Linux/macOS and Chrome or Chromium setup.", "limits_zh": "合成任务主要隔离测试交互技能，不能等同于完整生产流程。项目处于维护模式；Python 接口文档要求 Linux/macOS 及 Chrome 或 Chromium 配置。", "enables_en": "Train and compare agents on repeatable, relatively small web-control problems.", "enables_zh": "在可重复、规模较小的网页操作问题上训练和比较 Agent。", "resource_kind": "environment-and-benchmark", "platform_note": "Web is the task target; Linux and macOS are documented hosts for the Python runner.", "evidence": [ { "url": "https://github.com/Farama-Foundation/miniwob-plusplus#readme", "supports": "Documents over 100 browser environments, Gymnasium/Selenium interfaces, host support and maintenance status." }, { "url": "https://miniwob.farama.org/content/action_space/", "supports": "Official documentation defines the supported browser actions." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "mind2web", "name": "Mind2Web", "description_en": "An offline dataset and benchmark of human web-task demonstrations, pairing instructions and action sequences with captured webpage information.", "description_zh": "离线网页任务演示数据集与基准，将任务指令、人工操作序列和抓取的页面信息配对。", "content_type": "website", "platforms": [ "web" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://osu-nlp-group.github.io/Mind2Web/", "repo_url": "https://github.com/OSU-NLP-Group/Mind2Web", "demo_url": null, "paper_url": "https://arxiv.org/abs/2306.06070", "limits_en": "The original benchmark evaluates recorded actions rather than executing live websites. Dataset and code licenses differ; use the stated splits and macro-average metrics for paper comparisons.", "limits_zh": "原始基准评估记录中的操作，并不实际执行在线网站。数据集和代码采用不同许可；对比论文时应使用指定划分与宏平均指标。", "enables_en": "Train action-prediction models and test generalization to unseen tasks, websites and domains.", "enables_zh": "训练动作预测模型，并评估对未见任务、网站和领域的泛化能力。", "resource_kind": "dataset-and-benchmark", "evidence": [ { "url": "https://github.com/OSU-NLP-Group/Mind2Web#readme", "supports": "Documents demonstrations, snapshots, training/evaluation splits, corrected averaging guidance and separate data/code licenses." }, { "url": "https://arxiv.org/abs/2306.06070", "supports": "Original paper defines generalization settings using real-website demonstrations." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "online-mind2web", "name": "Online-Mind2Web", "description_en": "A live-web benchmark with tasks across public websites, trajectory submissions and WebJudge evaluation, maintained separately from the offline Mind2Web dataset.", "description_zh": "面向公开在线网站的任务基准，提供轨迹提交规范和 WebJudge 评估，与离线 Mind2Web 数据集分别维护。", "content_type": "website", "platforms": [ "web" ], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/OSU-NLP-Group/Online-Mind2Web", "repo_url": "https://github.com/OSU-NLP-Group/Online-Mind2Web", "demo_url": null, "paper_url": "https://arxiv.org/abs/2504.01382", "limits_en": "Live sites and task definitions change; retain the task revision and run date. Automatic judging is model-based, and leaderboard human review has its own submission and pricing policies.", "limits_zh": "在线网站和任务定义会变化，应记录任务版本与运行日期。自动评分采用模型评审；排行榜人工审核另有提交和收费规则。", "enables_en": "Evaluate end-to-end web agents on current websites and inspect the supporting action and screenshot evidence.", "enables_zh": "在在线网站上评估端到端网页 Agent，并检查其操作记录和截图证据。", "resource_kind": "benchmark-suite", "evidence": [ { "url": "https://github.com/OSU-NLP-Group/Online-Mind2Web#readme", "supports": "Documents 300 live-site tasks, ongoing replacements, WebJudge, v2 submission schema and human-review policy." }, { "url": "https://arxiv.org/abs/2504.01382", "supports": "Original paper examines online web-agent evaluation and automatic judging." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "webvoyager", "name": "WebVoyager", "description_en": "A browser-agent implementation and benchmark that combine screenshots and textual information to carry out tasks on real websites using Selenium.", "description_zh": "结合截图与文字信息、通过 Selenium 在真实网站上执行任务的浏览器 Agent 实现及基准。", "content_type": "website", "platforms": [ "web" ], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/MinorJerry/WebVoyager", "repo_url": "https://github.com/MinorJerry/WebVoyager", "demo_url": null, "paper_url": "https://arxiv.org/abs/2401.13919", "limits_en": "Depends on changing external websites, Chrome/Selenium and model access. Its automated evaluation uses a vision-language judge; results are not deterministic state-check scores.", "limits_zh": "依赖持续变化的外部网站、Chrome/Selenium 和模型访问。自动评估使用视觉语言模型评审，结果并非确定性的状态检查分数。", "enables_en": "Study a runnable multimodal web agent and evaluate complete browsing trajectories.", "enables_zh": "研究可运行的多模态网页 Agent，并评估完整浏览轨迹。", "resource_kind": "environment-and-benchmark", "evidence": [ { "url": "https://github.com/MinorJerry/WebVoyager#readme", "supports": "Provides the Selenium browser environment, live task data, agent implementation and automated evaluation protocol." }, { "url": "https://arxiv.org/abs/2401.13919", "supports": "Original paper describes end-to-end web interaction using multimodal observations." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "visualwebbench", "name": "VisualWebBench", "description_en": "A static multimodal web benchmark covering page understanding, OCR, element localization and action grounding from website screenshots.", "description_zh": "基于网站截图的静态多模态网页基准，覆盖页面理解、OCR、元素定位和动作定位。", "content_type": "website", "platforms": [ "web" ], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://visualwebbench.github.io/", "repo_url": "https://github.com/VisualWebBench/VisualWebBench", "demo_url": null, "paper_url": "https://arxiv.org/abs/2404.05955", "limits_en": "Question-answer and grounding evaluation does not execute multi-step browsing tasks. The seven task subsets should be reported separately when diagnosing a model.", "limits_zh": "问答和定位评估不会执行多步骤浏览任务。分析模型时应分别报告七个任务子集的结果。", "enables_en": "Identify whether a web agent's visual model struggles with reading, page semantics or locating controls.", "enables_zh": "区分网页 Agent 的视觉模型在阅读、页面语义或控件定位方面的不足。", "resource_kind": "dataset-and-benchmark", "evidence": [ { "url": "https://github.com/VisualWebBench/VisualWebBench#readme", "supports": "Defines seven QA-style visual web tasks and supplies evaluation adapters." }, { "url": "https://huggingface.co/datasets/visualwebbench/VisualWebBench", "supports": "Official dataset exposes test subsets with screenshots, instructions and reference answers." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "agentbench", "name": "AgentBench", "description_en": "A multi-domain agent benchmark whose current function-calling edition provides containerized tasks for operating-system commands, databases, knowledge graphs, shopping and text-based household tasks.", "description_zh": "跨领域 Agent 基准，当前函数调用版本为系统命令、数据库、知识图谱、购物和文本家庭任务等任务提供容器化运行。", "content_type": "app", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/THUDM/AgentBench", "repo_url": "https://github.com/THUDM/AgentBench", "demo_url": null, "paper_url": "https://arxiv.org/abs/2308.03688", "limits_en": "The current function-calling edition differs from the original eight-environment benchmark. Docker services can be resource intensive; the README documents ALFWorld memory/disk leakage and WebShop memory needs.", "limits_zh": "当前函数调用版本与原始八环境基准不同。Docker 服务可能消耗较多资源；README 提醒 ALFWorld 存在内存／磁盘增长问题，并列出 WebShop 内存需求。", "enables_en": "Compare multi-turn reasoning and tool use across several controlled task domains.", "enables_zh": "在多种受控任务领域比较多轮推理与工具使用能力。", "resource_kind": "benchmark-suite", "runtime": "Docker Compose; function-calling task workers", "platform_note": "A mixed-domain tool benchmark, not a claim of native desktop application support.", "evidence": [ { "url": "https://github.com/THUDM/AgentBench#readme", "supports": "Documents the 2025 function-calling update, AgentRL integration, five containerized task families, legacy versions and runtime caveats." }, { "url": "https://arxiv.org/abs/2308.03688", "supports": "Original paper describes the earlier eight-environment AgentBench setting." }, { "url": "https://github.com/THUDM/AgentBench/blob/main/src/server/tasks/alfworld/environment.py", "supports": "The ALFWorld environment instantiates AlfredTWEnv for text-based household tasks." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "swe-bench", "name": "SWE-bench", "description_en": "A source-code repair benchmark that asks an agent to resolve a real repository issue, then evaluates its patch with the project's tests.", "description_zh": "源码修复基准，要求 Agent 解决真实仓库问题，再通过项目测试评估补丁。", "content_type": "app", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://www.swebench.com/", "repo_url": "https://github.com/SWE-bench/SWE-bench", "demo_url": null, "paper_url": "https://arxiv.org/abs/2310.06770", "limits_en": "Measures repository-level code repair, not GUI operation or complete app generation. Docker evaluation needs substantial resources; identify the dataset variant and harness version in comparisons.", "limits_zh": "衡量仓库级代码修复，并非图形界面操作或完整应用生成。Docker 评测需要较多资源，比较时应注明数据集变体和评测框架版本。", "enables_en": "Evaluate coding agents against executable issue-resolution tests and reuse the supported training splits.", "enables_zh": "通过可执行的问题修复测试评估编程 Agent，并使用上游支持的训练划分。", "resource_kind": "benchmark-suite", "runtime": "Python; Docker evaluation harness", "platform_note": "The benchmark target is repository source code; host compatibility is not used as an application-platform tag.", "evidence": [ { "url": "https://github.com/SWE-bench/SWE-bench#readme", "supports": "Describes issue-to-patch evaluation, available variants, training data, Docker requirements and harness changes." }, { "url": "https://github.com/SWE-bench/SWE-bench/blob/main/docs/guides/docker_setup.md", "supports": "Official setup guide documents container-based evaluation and resource requirements." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "design2code", "name": "Design2Code", "description_en": "A screenshot-to-HTML benchmark with reference webpages, generated predictions and automated visual metrics for evaluating front-end code generation.", "description_zh": "截图到 HTML 的生成基准，提供参考网页、模型生成结果和自动视觉指标，用于评估前端代码生成。", "content_type": "website", "platforms": [ "web" ], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/NoviScl/Design2Code", "repo_url": "https://github.com/NoviScl/Design2Code", "demo_url": null, "paper_url": "https://arxiv.org/abs/2403.03163", "limits_en": "Evaluates visual reconstruction rather than working backend features or interactive task completion. The authors specify research-only use; Design2Code and Design2Code-Hard are distinct test sets.", "limits_zh": "评估视觉还原，不验证后端功能或交互任务完成情况。作者限定研究用途；Design2Code 与 Design2Code-Hard 为不同测试集。", "enables_en": "Compare how well generated webpage code reproduces a supplied visual design.", "enables_zh": "比较生成的网页代码对给定视觉设计的还原程度。", "resource_kind": "dataset-and-benchmark", "evidence": [ { "url": "https://github.com/NoviScl/Design2Code#readme", "supports": "Documents screenshot/HTML test pairs, the Hard set, Playwright rendering, evaluation and research-use restrictions." }, { "url": "https://arxiv.org/abs/2403.03163", "supports": "Original paper evaluates multimodal code generation for front-end reconstruction." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "webcode2m", "name": "WebCode2M", "description_en": "A web-code dataset pairing rendered page screenshots with HTML/CSS and layout information, accompanied by a screenshot-to-code evaluation pipeline.", "description_zh": "将网页渲染截图、HTML/CSS 和布局信息配对的数据集，附带截图到代码的评估流程。", "content_type": "website", "platforms": [ "web" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://webcode2m.github.io/", "repo_url": "https://github.com/CGCL-codes/naturalcc/tree/main/examples/webcode2m", "demo_url": null, "paper_url": "https://arxiv.org/abs/2404.06369", "limits_en": "Static page-generation data, not an interactive agent environment. The dataset card lists CC BY 4.0, while the project page states research-only use; confirm applicable terms with the authors before reuse.", "limits_zh": "这是静态页面生成数据，并非可交互的 Agent 环境。数据卡标注 CC BY 4.0，项目页另写研究用途限定；复用前应向作者确认适用条款。", "enables_en": "Train and evaluate models that translate page images into code and structural layouts.", "enables_zh": "训练和评估从页面图像生成代码与结构布局的模型。", "resource_kind": "dataset-and-benchmark", "evidence": [ { "url": "https://huggingface.co/datasets/xcodemind/webcode2m", "supports": "Official data card defines screenshot, code and layout fields and lists CC BY 4.0." }, { "url": "https://github.com/CGCL-codes/naturalcc/blob/main/examples/webcode2m/README.MD", "supports": "Official implementation documents test data, model inference and evaluation scripts." }, { "url": "https://webcode2m.github.io/", "supports": "Author project page links the code/data and includes the separate research-use statement." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "screenspot", "name": "ScreenSpot", "description_en": "A screenshot-based GUI grounding benchmark that asks a model to locate the control named by an instruction across mobile, desktop and web interfaces.", "description_zh": "基于截图的 GUI 定位基准，要求模型在移动端、桌面和网页界面中找到指令所指的控件。", "content_type": "app", "platforms": [ "web", "macos", "windows", "ios", "android" ], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/njucckevin/SeeClick", "repo_url": "https://github.com/njucckevin/SeeClick", "demo_url": null, "paper_url": "https://arxiv.org/abs/2401.10935", "limits_en": "Tests locating a target in a static screenshot, not executing the click or completing a workflow. Platform labels describe the screenshot sources, not installable app support.", "limits_zh": "只测试在静态截图中定位目标，不执行点击，也不验证流程完成。平台标签表示截图来源，并非可安装应用的支持范围。", "enables_en": "Measure the grounding component of a GUI agent independently of planning and execution.", "enables_zh": "将 GUI Agent 的界面定位能力与规划、执行能力分开评估。", "resource_kind": "dataset-and-benchmark", "platform_note": "Screenshot-source platforms; not executable native environments.", "evidence": [ { "url": "https://github.com/njucckevin/SeeClick#readme", "supports": "Defines ScreenSpot samples as screenshots, instructions and target boxes across five platform families." }, { "url": "https://arxiv.org/abs/2401.10935", "supports": "SeeClick paper introduces the GUI grounding benchmark and its role in visual agents." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "screenspot-pro", "name": "ScreenSpot-Pro", "description_en": "A GUI grounding benchmark using high-resolution screenshots of professional software, including development tools, creative applications, CAD and office suites.", "description_zh": "采用专业软件高分辨率截图的 GUI 定位基准，覆盖开发工具、创意应用、CAD 和办公套件。", "content_type": "app", "platforms": [ "macos", "windows", "linux" ], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/likaixin2000/ScreenSpot-Pro-GUI-Grounding", "repo_url": "https://github.com/likaixin2000/ScreenSpot-Pro-GUI-Grounding", "demo_url": null, "paper_url": "https://arxiv.org/abs/2504.07981", "limits_en": "A static target-localization test rather than a desktop VM or task environment. Keep screenshot resolution and language settings comparable; the listed software is not bundled for use.", "limits_zh": "属于静态目标定位测试，并非桌面虚拟机或任务环境。比较时应保持截图分辨率与语言设置一致；不提供所列专业软件的使用副本。", "enables_en": "Test whether a visual agent can identify small or unfamiliar controls in dense professional interfaces.", "enables_zh": "评估视觉 Agent 能否在密集的专业界面中识别细小或陌生控件。", "resource_kind": "dataset-and-benchmark", "platform_note": "Platforms reflect the source screenshots, including Windows, macOS and Ubuntu interfaces.", "evidence": [ { "url": "https://github.com/likaixin2000/ScreenSpot-Pro-GUI-Grounding#readme", "supports": "Official repository provides the grounding evaluation scripts and language-specific entry points." }, { "url": "https://huggingface.co/datasets/likaixin/ScreenSpot-Pro", "supports": "Official dataset lists the software, versions and screenshot operating systems." }, { "url": "https://arxiv.org/abs/2504.07981", "supports": "Original paper defines high-resolution professional GUI grounding." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "androidenv", "name": "AndroidEnv", "description_en": "A reinforcement-learning framework that exposes Android applications through touchscreen actions, pixel observations and configurable task rewards.", "description_zh": "将 Android 应用封装为强化学习环境的框架，提供触屏动作、像素观察和可配置的任务奖励。", "content_type": "app", "platforms": [ "android" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/google-deepmind/android_env", "repo_url": "https://github.com/google-deepmind/android_env", "demo_url": null, "paper_url": "https://arxiv.org/abs/2105.13231", "limits_en": "Requires an Android emulator and task-specific reward definitions. It is a framework, not one fixed benchmark; Windows host support is documented as less maintained.", "limits_zh": "需要 Android 模拟器及任务专用奖励定义。它是框架，而非单一固定基准；文档注明 Windows 主机支持的维护程度较低。", "enables_en": "Build repeatable Android learning tasks with a common touch interface.", "enables_zh": "使用统一触屏接口构建可重复的 Android 学习任务。", "resource_kind": "environment-framework", "evidence": [ { "url": "https://github.com/google-deepmind/android_env#readme", "supports": "Documents the touch-action interface, pixel observations, custom task definitions, emulator requirement and Windows caveat." }, { "url": "https://arxiv.org/abs/2105.13231", "supports": "Original technical report describes Android as an RL research platform." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "android-in-the-wild", "name": "Android in the Wild", "description_en": "A large offline collection of human Android demonstrations containing natural-language goals, UI screenshots and recorded touch, typing and navigation actions.", "description_zh": "大规模离线人工 Android 操作演示数据，包含自然语言目标、界面截图以及触屏、输入和导航动作记录。", "content_type": "app", "platforms": [ "android" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/google-research/google-research/tree/master/android_in_the_wild", "repo_url": "https://github.com/google-research/google-research/tree/master/android_in_the_wild", "demo_url": null, "paper_url": "https://arxiv.org/abs/2307.10088", "limits_en": "Recorded demonstrations and offline action-matching metrics do not provide a live Android environment. Use the published episode splits; data spans Android versions and collection-time app states.", "limits_zh": "记录式演示及离线动作匹配指标不等同于可运行的 Android 环境。应遵循公开的 episode 划分；数据包含不同 Android 版本和采集时的应用状态。", "enables_en": "Train mobile action-prediction models and test robustness across task, app and Android-version splits.", "enables_zh": "训练移动端动作预测模型，并通过任务、应用和 Android 版本划分评估稳健性。", "resource_kind": "dataset-and-benchmark", "evidence": [ { "url": "https://github.com/google-research/google-research/blob/master/android_in_the_wild/README.md", "supports": "Defines human demonstrations, Android versions, action formats, episode splits and offline action matching." }, { "url": "https://arxiv.org/abs/2307.10088", "supports": "Original paper introduces the mobile-device-control dataset." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "androidlab", "name": "AndroidLab", "description_en": "An Android-agent framework with prepared virtual devices, reproducible app tasks, evaluation tooling and the Android Instruct training dataset.", "description_zh": "Android Agent 框架，提供预设虚拟设备、可复现应用任务、评估工具和 Android Instruct 训练数据。", "content_type": "app", "platforms": [ "android" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/THUDM/Android-Lab", "repo_url": "https://github.com/THUDM/Android-Lab", "demo_url": null, "paper_url": "https://arxiv.org/abs/2410.24024", "limits_en": "Requires the prescribed Android images. Documented execution uses macOS arm64 AVDs or Linux x86_64 Docker; result generation uses a model judge and needs model access.", "limits_zh": "需要指定 Android 镜像。文档提供 macOS arm64 AVD 与 Linux x86_64 Docker 两种运行方式；结果生成使用模型评审，需要模型访问。", "enables_en": "Compare Android agents on controlled mobile workflows and train with the accompanying instruction data.", "enables_zh": "在受控移动端流程上比较 Android Agent，并使用配套指令数据进行训练。", "resource_kind": "environment-and-benchmark", "evidence": [ { "url": "https://github.com/THUDM/Android-Lab#readme", "supports": "Documents prepared virtual devices, 138 tasks across nine apps, supported hosts, instruction data and judge-based result generation." }, { "url": "https://arxiv.org/abs/2410.24024", "supports": "Original paper describes training and systematic benchmarking of Android agents." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "appworld", "name": "AppWorld", "description_en": "A simulated world of connected everyday apps with APIs, persistent task state and executable checks for function-calling and interactive coding agents.", "description_zh": "由相互关联的日常应用组成的模拟世界，提供 API、持久任务状态和可执行检查，用于函数调用及交互式编程 Agent。", "content_type": "app", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://appworld.dev/", "repo_url": "https://github.com/StonyBrookNLP/appworld", "demo_url": null, "paper_url": "https://arxiv.org/abs/2407.18901", "limits_en": "Apps are API simulations rather than native GUIs. Follow train/dev/test restrictions; protected app and task bundles have an additional encrypted-redistribution requirement.", "limits_zh": "应用以 API 模拟运行，并非原生图形界面。应遵循训练／开发／测试划分；受保护的应用与任务 bundle 另有加密再分发要求。", "enables_en": "Train and evaluate agents that coordinate several apps through code, API calls or the provided MCP integration.", "enables_zh": "训练和评估通过代码、API 调用或配套 MCP 接口协同操作多个应用的 Agent。", "resource_kind": "environment-and-benchmark", "runtime": "Python; simulated HTTP APIs; optional MCP interface", "platform_note": "API simulation; no native desktop or mobile target platform is asserted.", "evidence": [ { "url": "https://github.com/StonyBrookNLP/appworld#readme", "supports": "Documents the app/API engine, task-state evaluation, MCP integration, split restrictions and public/protected licensing." }, { "url": "https://arxiv.org/abs/2407.18901", "supports": "Original paper defines controllable app worlds and interactive coding tasks." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "weblinx", "name": "WebLINX", "description_en": "A dataset and benchmark for conversational web navigation, with multi-turn human demonstrations, page observations and action-prediction evaluation tools.", "description_zh": "面向对话式网页导航的数据集与基准，包含多轮人工演示、页面观察和动作预测评估工具。", "content_type": "website", "platforms": [ "web" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://mcgill-nlp.github.io/weblinx/", "repo_url": "https://github.com/McGill-NLP/weblinx", "demo_url": null, "paper_url": "https://arxiv.org/abs/2402.05930", "limits_en": "Evaluation follows recorded demonstrations rather than freely executing live websites. Data uses CC BY-NC-SA 4.0; BrowserGym/WebLINX 1.1 changes evaluated steps and must be distinguished from 1.0.", "limits_zh": "评估依据记录式演示，不是自由执行在线网站。数据采用 CC BY-NC-SA 4.0；BrowserGym/WebLINX 1.1 改变了评估步骤，应与 1.0 区分。", "enables_en": "Study agents that follow changing user instructions over a multi-turn browsing conversation.", "enables_zh": "研究在多轮浏览对话中跟随用户指令变化的 Agent。", "resource_kind": "dataset-and-benchmark", "evidence": [ { "url": "https://github.com/McGill-NLP/weblinx#readme", "supports": "Official library documents demonstration/replay abstractions, training code and action-evaluation tools." }, { "url": "https://huggingface.co/datasets/McGill-NLP/WebLINX", "supports": "Official dataset card specifies noncommercial/share-alike terms and explains the 1.1 BrowserGym evaluation changes." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "workbench", "name": "WorkBench", "description_en": "A workplace-agent benchmark with sandbox databases and read/write tools for tasks such as email, scheduling and project management, evaluated by the resulting state.", "description_zh": "提供沙盒数据库和读写工具的办公 Agent 基准，包含邮件、日程与项目管理等任务，并依据最终状态评分。", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/olly-styles/WorkBench", "repo_url": "https://github.com/olly-styles/WorkBench", "demo_url": null, "paper_url": "https://arxiv.org/abs/2405.00823", "limits_en": "Uses simulated workplace tools rather than real email accounts or desktop apps. The 2026 task corrections and evaluator differ from the original release; report the ground-truth and scoring versions.", "limits_zh": "使用模拟办公工具，并非真实邮箱账户或桌面应用。2026 年的任务修订和评分器与原始版本不同，应报告标准答案及评分版本。", "enables_en": "Measure task completion and unintended state changes in controlled office workflows.", "enables_zh": "在受控办公流程中衡量任务完成情况和非预期状态变更。", "resource_kind": "environment-and-benchmark", "runtime": "Python; local sandbox databases and tools", "platform_note": "Tool/API simulation rather than an operating-system-specific GUI environment.", "related_paper_urls": [ "https://arxiv.org/abs/2606.13715" ], "evidence": [ { "url": "https://github.com/olly-styles/WorkBench#readme", "supports": "Documents five sandbox databases, 26 tools, outcome evaluation, 2026 revisions and versioned ground truth." }, { "url": "https://arxiv.org/abs/2405.00823", "supports": "Original paper defines workplace tasks and outcome-centric evaluation." }, { "url": "https://github.com/olly-styles/WorkBench/tree/main/src/tools", "supports": "Tool domains cover email, calendar, analytics, CRM and project management." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "browserart", "name": "BrowserART", "description_en": "A browser-agent safety evaluation toolkit with synthetic websites, behavior datasets and trajectory classifiers for studying whether agents follow harmful requests.", "description_zh": "浏览器 Agent 安全评估工具包，提供合成网站、行为数据和轨迹分类器，用于研究 Agent 是否执行有害请求。", "content_type": "website", "platforms": [ "web" ], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/scaleapi/browser-art", "repo_url": "https://github.com/scaleapi/browser-art", "demo_url": null, "paper_url": "https://arxiv.org/abs/2410.13886", "limits_en": "A specialized safety benchmark rather than a general task-success ranking. It mixes synthetic and live-site tasks; the code is MIT-licensed while the behavior dataset lists CC BY-NC 4.0.", "limits_zh": "属于专门的安全基准，不是通用任务成功率排行榜。任务同时涉及合成和在线网站；代码采用 MIT，行为数据集标注 CC BY-NC 4.0。", "enables_en": "Evaluate browser-agent refusal behavior in a controlled research setup and inspect recorded trajectories.", "enables_zh": "在受控研究配置中评估浏览器 Agent 的拒绝行为，并检查操作轨迹。", "resource_kind": "environment-and-benchmark", "evidence": [ { "url": "https://github.com/scaleapi/browser-art#readme", "supports": "Defines the safety evaluation toolkit, behavior classifiers, synthetic website source and MIT code license." }, { "url": "https://github.com/scaleapi/browser-art/blob/main/src/README.md", "supports": "Documents local website hosting and BrowserGym/OpenDevin evaluation setup." }, { "url": "https://huggingface.co/datasets/ScaleAI/BrowserART", "supports": "Official behavior dataset card specifies CC BY-NC 4.0." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "spider2-v", "name": "Spider2-V", "description_en": "An executable desktop benchmark for data-science and engineering workflows that combine code, GUI actions and enterprise tools such as dbt, Airbyte and BigQuery.", "description_zh": "面向数据科学与工程流程的可执行桌面基准，结合代码、GUI 操作以及 dbt、Airbyte、BigQuery 等企业工具。", "content_type": "app", "platforms": [ "linux" ], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://spider2-v.github.io/", "repo_url": "https://github.com/xlang-ai/Spider2-V", "demo_url": null, "paper_url": "https://arxiv.org/abs/2407.10956", "limits_en": "Requires the prepared Ubuntu VM and setup dependencies. Some tasks need real cloud accounts and credentials; account/non-account and abstract/verbose instruction subsets are different evaluation settings.", "limits_zh": "需要预设 Ubuntu 虚拟机和配套依赖。部分任务需要真实云账户及凭证；有账户／无账户、简要／详细指令属于不同评测设置。", "enables_en": "Evaluate whether agents can carry a data workflow across graphical interfaces, code and task-specific state checks.", "enables_zh": "通过图形界面、代码和任务专用状态检查，评估 Agent 完成数据工作流程的能力。", "resource_kind": "environment-and-benchmark", "platform_note": "The executable guest environment is Ubuntu; host support does not imply macOS or Windows task coverage.", "evidence": [ { "url": "https://spider2-v.github.io/", "supports": "Project page describes 494 tasks, professional data tools and the executable Ubuntu desktop." }, { "url": "https://github.com/xlang-ai/Spider2-V#readme", "supports": "Documents VM installation, account requirements and the separate instruction/account subsets." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "nethack-learning-environment", "name": "NetHack Learning Environment", "description_en": "A Gymnasium interface to NetHack with procedurally generated dungeons, inventory management and long sequences of consequential actions.", "description_zh": "为 NetHack 提供 Gymnasium 接口，包含程序化生成的地牢、物品管理和需要连续决策的任务。", "content_type": "game", "platforms": [ "linux", "macos" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/NetHack-LE/nle", "repo_url": "https://github.com/NetHack-LE/nle", "demo_url": null, "paper_url": "https://arxiv.org/abs/2006.13760", "limits_en": "Requires Python 3.10+, CMake 3.28+ and native build dependencies. The current repository uses NetHack 3.6.7 and the NetHack General Public License; older NLE releases differ.", "limits_zh": "需要 Python 3.10+、CMake 3.28+ 和原生构建依赖。当前仓库基于 NetHack 3.6.7，采用 NetHack General Public License；旧版 NLE 的依赖与行为可能不同。", "enables_en": "Train agents on exploration, survival and inventory decisions, or compare policies on defined NetHack tasks.", "enables_zh": "训练探索、生存和物品决策能力，或在明确的 NetHack 任务上比较策略。", "resource_kind": "environment-and-benchmark", "runtime": "Python/Gymnasium with a native NetHack backend; pip install nle; local human-play and random-agent scripts.", "evidence": [ { "url": "https://github.com/NetHack-LE/nle#readme", "supports": "Documents the current NetHack version, Gymnasium interface, installation, macOS/Ubuntu instructions and runnable scripts." }, { "url": "https://github.com/NetHack-LE/nle/blob/main/LICENSE", "supports": "The repository publishes the NetHack General Public License." }, { "url": "https://arxiv.org/abs/2006.13760", "supports": "Original NLE paper describes the game-based RL environment and evaluation tasks." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "minedojo", "name": "MineDojo", "description_en": "A Minecraft research suite that combines survival, resource-gathering, creative and playthrough tasks with a separate knowledge collection.", "description_zh": "基于 Minecraft 的研究套件，提供生存、资源收集、创作和通关任务，并配有独立的知识资料库。", "content_type": "game", "platforms": [ "linux", "macos" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/MineDojo/MineDojo", "repo_url": "https://github.com/MineDojo/MineDojo", "demo_url": null, "paper_url": "https://arxiv.org/abs/2206.08853", "limits_en": "Requires Python 3.9+, JDK 8 and a Minecraft backend; headless runs need the documented display setup. Code is MIT, while the Wiki database is CC BY-NC-SA 3.0 and other databases have separate terms.", "limits_zh": "需要 Python 3.9+、JDK 8 和 Minecraft 后端；无显示器运行需按文档配置。代码采用 MIT；Wiki 数据库采用 CC BY-NC-SA 3.0，其他数据库另有许可。", "enables_en": "Build agents that follow varied Minecraft goals and study how external knowledge supports exploration and crafting.", "enables_zh": "构建遵循不同 Minecraft 目标的 Agent，研究外部知识对探索和制作的帮助。", "resource_kind": "environment-and-benchmark", "runtime": "Python Gym-style API; pip install minedojo; prerequisites and a Docker path are documented.", "evidence": [ { "url": "https://github.com/MineDojo/MineDojo#readme", "supports": "Documents the task families, Python/JDK requirements, tested Ubuntu/macOS setups, API examples and component-specific licenses." }, { "url": "https://docs.minedojo.org/sections/getting_started/install.html", "supports": "Author installation instructions cover prerequisites and headless/Docker operation." }, { "url": "https://arxiv.org/abs/2206.08853", "supports": "Original paper describes the Minecraft simulation suite, knowledge bases and agent-learning setting." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "minerl", "name": "MineRL", "description_en": "A Python interface to Minecraft research tasks, with versioned environments and a separate collection of human demonstrations.", "description_zh": "为 Minecraft 研究任务提供 Python 接口，包含分版本维护的环境，以及独立的人类操作示范数据。", "content_type": "game", "platforms": [ "linux", "macos", "windows" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/minerllabs/minerl", "repo_url": "https://github.com/minerllabs/minerl", "demo_url": null, "paper_url": "https://arxiv.org/abs/1907.13440", "limits_en": "JDK 8 is required; Windows needs bash/WSL and macOS may need extra setup. v1.0 differs from the original dataset environments. Licensing metadata conflicts: root LICENSE says CC BY-NC-SA 4.0, but setup.py says MIT; do not assume a single permissive license.", "limits_zh": "必须使用 JDK 8；Windows 需要 bash/WSL，macOS 可能需额外配置。v1.0 与原始数据对应的环境不同。许可信息存在冲突：根 LICENSE 写为 CC BY-NC-SA 4.0，而 setup.py 标注 MIT，不能直接视为统一的宽松许可。", "enables_en": "Train or evaluate Minecraft control policies while explicitly matching environment and demonstration-data versions.", "enables_zh": "在明确匹配环境与示范数据版本的前提下，训练或评估 Minecraft 控制策略。", "resource_kind": "environment-and-benchmark", "runtime": "Python/Gym and Java JDK 8; install from the author repository; v0 data mirrors were replaced by a linked Zenodo record.", "evidence": [ { "url": "https://github.com/minerllabs/minerl/blob/dev/README.md", "supports": "Documents v1.0/v0.4/v0.3 differences, JDK 8, runnable tasks and the replacement data record." }, { "url": "https://minerl.readthedocs.io/en/latest/tutorials/index.html", "supports": "Author setup page documents Windows bash/WSL, macOS and Debian/Ubuntu requirements." }, { "url": "https://github.com/minerllabs/minerl/blob/dev/LICENSE", "supports": "Root license text is CC BY-NC-SA 4.0." }, { "url": "https://github.com/minerllabs/minerl/blob/dev/setup.py", "supports": "Package metadata separately declares MIT, so this review does not resolve the conflict." }, { "url": "https://arxiv.org/abs/1907.13440", "supports": "Original paper concerns MineRL human demonstrations paired with simulator tasks." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "Windows is documented with bash/WSL requirements; this is not a claim of a dependency-free native Windows build." }, { "id": "balrog", "name": "BALROG", "description_en": "An evaluation harness for language and vision-language agents across existing interactive games, including text adventures and NetHack-based tasks.", "description_zh": "在现有交互游戏上评估语言与视觉语言 Agent 的框架，覆盖文字冒险和基于 NetHack 的任务。", "content_type": "game", "platforms": [ "linux", "macos" ], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/balrog-ai/BALROG", "repo_url": "https://github.com/balrog-ai/BALROG", "demo_url": null, "paper_url": "https://arxiv.org/abs/2411.13543", "limits_en": "Requires the bundled environments and post-install setup, plus a local model server or paid API access where chosen. The harness is MIT; underlying games keep their own licenses. It is not a newly AI-built game.", "limits_zh": "需要安装所依赖的环境并完成后置配置，还需自备本地模型服务或所选付费 API。评测框架采用 MIT，各底层游戏保留自身许可；它并非新生成的 AI 游戏作品。", "enables_en": "Run a common evaluation workflow across games and compare long-horizon reasoning under explicit agent and observation settings.", "enables_zh": "通过统一流程运行多款游戏评测，并在明确 Agent 与观察设置的前提下比较长期推理能力。", "resource_kind": "benchmark-suite", "runtime": "Python 3.10; editable install and balrog-post-install; author Dockerfile uses Ubuntu and README gives macOS notes.", "evidence": [ { "url": "https://github.com/balrog-ai/BALROG#readme", "supports": "Documents the evaluation harness, installation, API/local model options, macOS notes and MIT license." }, { "url": "https://github.com/balrog-ai/BALROG/blob/main/docker/Dockerfile", "supports": "Published Docker configuration uses Ubuntu 22.04 and Python 3.10." }, { "url": "https://arxiv.org/abs/2411.13543", "supports": "Paper defines the multi-environment agent reasoning evaluation setting." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "open-spiel", "name": "OpenSpiel", "description_en": "A framework of game environments and algorithms for single-agent and multi-agent learning, search and planning with different information structures.", "description_zh": "集成多种游戏环境与算法的研究框架，支持不同信息条件下的单 Agent、多 Agent 学习、搜索与规划。", "content_type": "game", "platforms": [ "linux", "macos", "windows" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/google-deepmind/open_spiel", "repo_url": "https://github.com/google-deepmind/open_spiel", "demo_url": null, "paper_url": "https://arxiv.org/abs/1908.09453", "limits_en": "This is a research framework rather than a hosted game. Optional games and algorithms add dependencies. Windows wheels and build instructions are documented, but the Windows page labels support experimental. Repository code is Apache 2.0.", "limits_zh": "这是研究框架，并非在线游戏服务；可选游戏与算法会增加依赖。文档提供 Windows wheel 和构建说明，同时将 Windows 支持标为实验性。仓库代码采用 Apache 2.0。", "enables_en": "Compare decision-making algorithms on games with cooperation, competition, simultaneous actions or hidden information.", "enables_zh": "在合作、竞争、同时行动或隐藏信息等游戏条件下比较决策算法。", "resource_kind": "environment-framework", "runtime": "C++ core with Python bindings; pip installation or source/Docker builds; separate Windows instructions.", "evidence": [ { "url": "https://github.com/google-deepmind/open_spiel#readme", "supports": "Defines supported game types, learning/planning use and C++/Python interfaces." }, { "url": "https://github.com/google-deepmind/open_spiel/blob/master/docs/install.md", "supports": "Documents Linux/macOS pip and source installation." }, { "url": "https://github.com/google-deepmind/open_spiel/blob/master/docs/windows.md", "supports": "Documents Windows wheels and builds and explicitly labels Windows support experimental." }, { "url": "https://github.com/google-deepmind/open_spiel/blob/master/LICENSE", "supports": "Repository license is Apache 2.0." }, { "url": "https://arxiv.org/abs/1908.09453", "supports": "Original OpenSpiel framework paper." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "procgen", "name": "Procgen Benchmark", "description_en": "A suite of sixteen procedurally generated visual games designed to separate learning on familiar levels from generalization to unseen levels.", "description_zh": "由十六款程序化生成的视觉游戏组成的基准，用于区分熟悉关卡上的学习表现和未见关卡上的泛化表现。", "content_type": "game", "platforms": [ "linux", "macos", "windows" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/openai/procgen", "repo_url": "https://github.com/openai/procgen", "demo_url": null, "paper_url": "https://arxiv.org/abs/1912.01588", "limits_en": "The maintenance-mode README lists older Python 3.7–3.10 builds, 64-bit platforms and an AVX-capable CPU. Compatibility with other setups is unverified here. Code is MIT; bundled assets include distinct attribution licenses. Procgen2 is a separate community rewrite; benchmark equivalence has not been established.", "limits_zh": "项目处于维护模式，README 列出较旧的 Python 3.7–3.10 构建、64 位平台和 AVX CPU 要求；本次未验证其他配置的兼容性。代码采用 MIT，内置素材包含独立的署名许可。 Procgen2 是独立的社区重写版本，尚未确认基准等价性。", "enables_en": "Train on selected level seeds and evaluate on new seeds to study generalization in arcade-style control.", "enables_zh": "在指定关卡种子上训练，并在新种子上评估街机操作策略的泛化能力。", "resource_kind": "benchmark-suite", "runtime": "Python Gym/gym3 with a C++/Qt backend; pip install procgen; local interactive player and Docker examples.", "evidence": [ { "url": "https://github.com/openai/procgen#readme", "supports": "Documents sixteen games, level seeds, interfaces, installation, platforms, CPU constraints and maintenance status." }, { "url": "https://github.com/openai/procgen/blob/master/LICENSE", "supports": "Repository code is MIT licensed." }, { "url": "https://github.com/openai/procgen/blob/master/ASSET_LICENSES.md", "supports": "Lists asset-specific CC0, CC BY and OGA BY licenses." }, { "url": "https://arxiv.org/abs/1912.01588", "supports": "Paper defines procedural generation as a means to evaluate RL generalization." }, { "url": "https://github.com/Farama-Foundation/Procgen2#readme", "supports": "Identifies Procgen2 as a separate rewrite; original Procgen results are not automatically applicable." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "arcade-learning-environment", "name": "Arcade Learning Environment", "description_en": "A Stella-based Atari 2600 interface that exposes observations, actions, game scores and episode endings to learning agents.", "description_zh": "基于 Stella 模拟器的 Atari 2600 接口，为学习型 Agent 提供观察、动作、游戏得分和回合终止信号。", "content_type": "game", "platforms": [ "linux", "macos", "windows", "web" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://ale.farama.org/", "repo_url": "https://github.com/Farama-Foundation/Arcade-Learning-Environment", "demo_url": null, "paper_url": "https://jair.org/index.php/jair/article/view/10819", "limits_en": "Current packages include ROMs, unlike older AutoROM-based instructions. Results depend on environment settings and versions. ALE code is GPL 2.0; this review does not establish separate rights for game ROM redistribution.", "limits_zh": "当前安装包已包含 ROM，与旧版 AutoROM 安装流程不同。结果依赖环境设置和版本。ALE 代码采用 GPL 2.0；本次审查未确认游戏 ROM 的独立再分发权利。", "enables_en": "Train and compare agents through standard Atari interfaces while recording the game and evaluation settings.", "enables_zh": "通过标准 Atari 接口训练和比较 Agent，并记录游戏及评测配置。", "resource_kind": "environment-framework", "runtime": "C++/Python/Gymnasium interfaces; pip install ale-py; documented WebAssembly/browser support.", "evidence": [ { "url": "https://github.com/Farama-Foundation/Arcade-Learning-Environment#readme", "supports": "Documents Atari interaction, score signals, native platforms, packaged ROMs and WebAssembly support." }, { "url": "https://ale.farama.org/release_notes/", "supports": "Documents the move to packaged ROMs and changes to registration/setup." }, { "url": "https://github.com/Farama-Foundation/Arcade-Learning-Environment/blob/main/LICENSE.md", "supports": "Published ALE code license is GNU GPL version 2." }, { "url": "https://jair.org/index.php/jair/article/view/10819", "supports": "Original paper introduces ALE as an evaluation platform for general agents." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "The web tag follows explicit WebAssembly support; it does not assert that every Python benchmark configuration runs in a browser." }, { "id": "scienceworld", "name": "ScienceWorld", "description_en": "A text-based simulated world where agents manipulate objects and perform elementary science tasks such as heating, classification and electrical experiments.", "description_zh": "以文字交互的模拟世界，Agent 可操作物体，完成加热、分类和电学实验等基础科学任务。", "content_type": "game", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/allenai/ScienceWorld", "repo_url": "https://github.com/allenai/ScienceWorld", "demo_url": null, "paper_url": "https://arxiv.org/abs/2203.07540", "limits_en": "Requires Python 3.8+ and Java 1.8+. It is a symbolic science simulation, not physical laboratory validation. Version 1.3 changes ordering and can alter trajectories relative to 1.2; pin versions for comparisons. Code is Apache 2.0.", "limits_zh": "需要 Python 3.8+ 与 Java 1.8+。它是符号化科学模拟，不能代替真实实验验证。1.3 版改变对象排序，可能产生不同于 1.2 的轨迹；比较实验应固定版本。代码采用 Apache 2.0。", "enables_en": "Evaluate action sequences that apply scientific knowledge, with controlled task variations and optional environment simplifications.", "enables_zh": "在可控任务变体和可选环境简化条件下，评估 Agent 运用科学知识完成动作序列的能力。", "resource_kind": "environment-and-benchmark", "runtime": "Scala/Java simulator with a Python API; pip install scienceworld; local text console and optional browser console.", "evidence": [ { "url": "https://github.com/allenai/ScienceWorld#readme", "supports": "Documents science tasks, Python/Java requirements, local consoles and the version 1.3 reproducibility change." }, { "url": "https://github.com/allenai/ScienceWorld/blob/main/LICENSE", "supports": "Repository code is Apache 2.0 licensed." }, { "url": "https://arxiv.org/abs/2203.07540", "supports": "Paper describes interactive science tasks and agent evaluation." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "No OS tag is inferred from the Python/Java runtime. The game category denotes an interactive research world, not a commercial game." }, { "id": "virtualhome", "name": "VirtualHome", "description_en": "A household activity simulator that executes action programs, with a Unity visual environment and a separate evolving scene-graph backend.", "description_zh": "可执行家庭活动动作程序的模拟器，提供 Unity 视觉环境和独立的动态场景图后端。", "content_type": "game", "platforms": [ "linux", "macos", "windows" ], "uses": [ "training" ], "implementation": "unknown", "primary_url": "https://github.com/xavierpuigf/virtualhome", "repo_url": "https://github.com/xavierpuigf/virtualhome", "demo_url": null, "paper_url": "https://arxiv.org/abs/1806.07011", "limits_en": "The rendered mode needs a separate Unity executable; headless operation needs additional setup. Graph and Unity backends do not support identical actions. API code is MIT; downloaded simulator/assets were not license-audited here.", "limits_zh": "渲染模式需要另行下载 Unity 可执行文件，无显示器运行需额外配置。场景图和 Unity 后端支持的动作并不完全相同。API 代码采用 MIT；本次未审计下载版模拟器及素材的许可。", "enables_en": "Generate household activity trajectories and build embodied-agent training tasks with object interactions and scene state.", "enables_zh": "生成家庭活动轨迹，并利用物体交互与场景状态构建具身 Agent 训练任务。", "resource_kind": "environment-framework", "runtime": "Python API via pip install virtualhome; author-provided Unity downloads for Linux, macOS and Windows.", "evidence": [ { "url": "https://github.com/xavierpuigf/virtualhome#readme", "supports": "Documents household action programs, embodied training, two backends, platform-specific executables and headless requirements." }, { "url": "https://github.com/xavierpuigf/virtualhome/blob/master/LICENSE", "supports": "API repository publishes the MIT license." }, { "url": "https://arxiv.org/abs/1806.07011", "supports": "Original paper describes program-driven household activity simulation." }, { "url": "https://github.com/xavierpuigf/virtualhome#readme", "supports": "A household-activity virtual environment with documented Linux, macOS and Windows executables." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "minigrid", "name": "MiniGrid", "description_en": "A configurable collection of small grid worlds for navigation, object manipulation and instruction following, including the BabyAI environments.", "description_zh": "可配置的小型网格世界集合，覆盖导航、物体操作和指令执行，并包含 BabyAI 环境。", "content_type": "game", "platforms": [ "linux", "macos" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://minigrid.farama.org/", "repo_url": "https://github.com/Farama-Foundation/Minigrid", "demo_url": null, "paper_url": "https://arxiv.org/abs/2306.13831", "limits_en": "Current README requires Python 3.11+ and does not officially support Windows. Licensing is inconsistent in the reviewed branch: pyproject.toml says MIT, while LICENSE contains incomplete Apache 2.0 text. A definitive license was not established. XLand-MiniGrid is a separate JAX suite with changing rulesets; its included MiniGrid ports overlap existing tasks.", "limits_zh": "当前 README 要求 Python 3.11+，并未正式支持 Windows。所审查分支的许可信息不一致：pyproject.toml 标注 MIT，但 LICENSE 含不完整的 Apache 2.0 文本；本次未确定最终适用许可。 XLand-MiniGrid 是具有变化规则集的独立 JAX 套件，其附带 MiniGrid 移植内容与现有任务重合。", "enables_en": "Create controlled curricula and evaluate navigation, memory and language-conditioned action composition.", "enables_zh": "构建可控课程，评估导航、记忆和语言条件下的动作组合。", "resource_kind": "environment-framework", "runtime": "Python/Gymnasium; pip install minigrid; customizable mission strings, map sizes and task configurations.", "evidence": [ { "url": "https://github.com/Farama-Foundation/Minigrid#readme", "supports": "Documents MiniGrid/BabyAI tasks, Gymnasium, installation and explicitly supported Linux/macOS platforms." }, { "url": "https://github.com/Farama-Foundation/Minigrid/blob/main/pyproject.toml", "supports": "Package metadata declares MIT." }, { "url": "https://github.com/Farama-Foundation/Minigrid/blob/main/LICENSE", "supports": "Reviewed root file instead contains incomplete Apache 2.0 text, so the license conflict is preserved." }, { "url": "https://arxiv.org/abs/2306.13831", "supports": "Author paper describes MiniGrid and MiniWorld design, APIs and goal-oriented tasks." }, { "url": "https://github.com/dunnolab/xland-minigrid#readme", "supports": "Distinguishes changing-ruleset meta-learning tasks from included ports of MiniGrid environments." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "jericho", "name": "Jericho", "description_en": "A Python interface connecting agents to interactive-fiction games, with access to game state and controls for reproducible play.", "description_zh": "将 Agent 连接到交互式文字冒险游戏的 Python 接口，提供游戏状态访问及可复现运行所需的控制。", "content_type": "game", "platforms": [ "linux" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/microsoft/jericho", "repo_url": "https://github.com/microsoft/jericho", "demo_url": null, "paper_url": "https://arxiv.org/abs/1909.05398", "limits_en": "Current requirements include Linux, Python 3.12+, spaCy and build tools; game files are separate. Version 4 changes default seeding, so record seeds and versions. Interface code is GPL 2.0; individual games retain separate rights.", "limits_zh": "当前依赖包括 Linux、Python 3.12+、spaCy 和构建工具，游戏文件需单独准备。第 4 版改变默认种子行为，应记录种子与版本。接口代码采用 GPL 2.0，各游戏保留独立权利。", "enables_en": "Study language-based exploration and planning across existing text adventures with explicit reproducibility settings.", "enables_zh": "在现有文字冒险中研究基于语言的探索和规划，并明确记录复现设置。", "resource_kind": "environment-framework", "runtime": "Python interface to Frotz; pip install jericho plus the documented spaCy model; load a supported game file.", "evidence": [ { "url": "https://github.com/microsoft/jericho#readme", "supports": "Documents the interactive-fiction interface, current Linux/Python requirements and version 4 seed changes." }, { "url": "https://github.com/microsoft/jericho/blob/master/LICENSE", "supports": "Repository license is GNU GPL version 2." }, { "url": "https://arxiv.org/abs/1909.05398", "supports": "Original paper introduces Jericho and its text-game research setting." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "gpt5-escape-the-maze", "name": "Escape the Maze", "description_en": "A single-file browser maze game attributed by OpenAI to its AI-generated coding collection, with keyboard navigation, timing and shortest-path comparison.", "description_zh": "OpenAI 署名的 AI 生成作品集中的单文件浏览器迷宫游戏，包含键盘移动、计时和最短路径比较。", "content_type": "game", "platforms": [ "web" ], "uses": [ "showcase" ], "implementation": "ai-built", "primary_url": "https://github.com/openai/gpt-5-coding-examples/tree/main/apps/escape-the-maze", "repo_url": "https://github.com/openai/gpt-5-coding-examples", "demo_url": "https://gpt5-coding-examples.vercel.app/escape-the-maze/", "paper_url": null, "limits_en": "The game is a coding showcase without a published agent evaluator. The hosted HTML returned 200; gameplay was not tested. Repository code is MIT licensed.", "limits_zh": "属于编程作品展示，未提供公开 Agent 评分器。托管 HTML 返回 200，但未实际操作游戏。仓库代码采用 MIT。", "enables_en": "Inspect a generated maze implementation and compare its code with the published creation prompt.", "enables_zh": "查看生成的迷宫实现，并与公开创作提示对照。", "resource_kind": "ai-generated-output", "runtime": "Published single-file HTML/CSS/JavaScript; inspect the hosted page or run the collection with its documented front-end development server.", "evidence": [ { "url": "https://github.com/openai/gpt-5-coding-examples#readme", "supports": "OpenAI explicitly attributes the collection to single GPT-5 or GPT-5.2 prompts without handwritten application code." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/examples/escape-the-maze.yaml", "supports": "Publisher supplies the specific example name, prompt and category." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/apps/escape-the-maze/index.html", "supports": "Published source contains maze generation, keyboard controls, a timer and shortest-path comparison." }, { "url": "https://gpt5-coding-examples.vercel.app/escape-the-maze/", "supports": "Hosted HTML returned HTTP 200 and matched the published app file during this source review; no JavaScript or app interaction was executed." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/LICENSE", "supports": "Collection publishes an MIT license." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "provenance_source": "https://github.com/openai/gpt-5-coding-examples#readme" }, { "id": "gpt5-typing-rain", "name": "Typing Rain", "description_en": "An AI-generated browser typing game where falling words, rising difficulty, a score and an accuracy counter form the interaction.", "description_zh": "AI 生成的浏览器打字游戏，通过下落单词、逐渐增加的难度、得分和准确率形成交互。", "content_type": "game", "platforms": [ "web" ], "uses": [ "showcase" ], "implementation": "ai-built", "primary_url": "https://github.com/openai/gpt-5-coding-examples/tree/main/apps/typing-rain", "repo_url": "https://github.com/openai/gpt-5-coding-examples", "demo_url": "https://gpt5-coding-examples.vercel.app/typing-rain/", "paper_url": null, "limits_en": "Keyboard-oriented demo with browser-local best-score storage, not a language-learning or agent benchmark. Hosted HTML returned 200; play and accessibility were not tested. Repository code is MIT.", "limits_zh": "以键盘操作为主的演示，最高分保存在当前浏览器；并非语言学习或 Agent 基准。托管 HTML 返回 200，未测试游玩与无障碍体验。仓库代码采用 MIT。", "enables_en": "Inspect how a generated game connects keyboard input, animation and local scoring.", "enables_zh": "观察生成游戏如何连接键盘输入、动画和本地计分。", "resource_kind": "ai-generated-output", "runtime": "Published single-file HTML/CSS/JavaScript; inspect the hosted page or run the collection with its documented front-end development server.", "evidence": [ { "url": "https://github.com/openai/gpt-5-coding-examples#readme", "supports": "OpenAI explicitly attributes the collection to single GPT-5 or GPT-5.2 prompts without handwritten application code." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/examples/typing-rain.yaml", "supports": "Publisher supplies the specific example name, prompt and category." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/apps/typing-rain/index.html", "supports": "Published source implements falling-word generation, score/accuracy state and localStorage for the best score." }, { "url": "https://gpt5-coding-examples.vercel.app/typing-rain/", "supports": "Hosted HTML returned HTTP 200 and matched the published app file during this source review; no JavaScript or app interaction was executed." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/LICENSE", "supports": "Collection publishes an MIT license." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "provenance_source": "https://github.com/openai/gpt-5-coding-examples#readme" }, { "id": "gpt5-tic-tac-toe-game", "name": "Tic Tac Toe — Imperium Edition", "description_en": "A Roman-themed browser tic-tac-toe artifact from OpenAI’s generated examples, with local two-player and computer-opponent modes.", "description_zh": "来自 OpenAI 生成案例集的罗马主题浏览器井字棋作品，提供本地双人和电脑对手模式。", "content_type": "game", "platforms": [ "web" ], "uses": [ "showcase" ], "implementation": "ai-built", "primary_url": "https://github.com/openai/gpt-5-coding-examples/tree/main/apps/tic-tac-toe-game", "repo_url": "https://github.com/openai/gpt-5-coding-examples", "demo_url": "https://gpt5-coding-examples.vercel.app/tic-tac-toe-game/", "paper_url": null, "limits_en": "The computer opponent uses game logic including minimax, not an LLM service. This showcase has no benchmark protocol. A separate browser observation covers one local two-player win; opponent strategy and other modes were not tested. Repository code is MIT.", "limits_zh": "电脑对手使用包含 minimax 的游戏逻辑，并非 LLM 服务。该展示作品没有基准协议。独立浏览器记录覆盖一局本地双人获胜过程；未测试电脑策略及其他模式。仓库代码采用 MIT。", "enables_en": "Examine generated game-state handling, a computer opponent and responsive board UI.", "enables_zh": "研究生成的游戏状态管理、电脑对手和响应式棋盘界面。", "resource_kind": "ai-generated-output", "runtime": "Published single-file HTML/CSS/JavaScript; inspect the hosted page or run the collection with its documented front-end development server.", "evidence": [ { "url": "https://github.com/openai/gpt-5-coding-examples#readme", "supports": "OpenAI explicitly attributes the collection to single GPT-5 or GPT-5.2 prompts without handwritten application code." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/examples/tic-tac-toe-game.yaml", "supports": "Publisher supplies the specific example name, prompt and category." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/apps/tic-tac-toe-game/index.html", "supports": "Published prompt and source describe local modes and contain minimax-based opponent functions." }, { "url": "https://gpt5-coding-examples.vercel.app/tic-tac-toe-game/", "supports": "Hosted HTML was inspected during source review. Later browser interactions are documented separately in data/observations.json; no deployed source commit was established." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/LICENSE", "supports": "Collection publishes an MIT license." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "provenance_source": "https://github.com/openai/gpt-5-coding-examples#readme" }, { "id": "gpt5-online-whiteboard", "name": "Online Whiteboard", "description_en": "A generated browser drawing app with pen and eraser tools, color choices and PNG image export.", "description_zh": "AI 生成的浏览器绘图应用，提供画笔、橡皮擦、颜色选择和 PNG 图像导出。", "content_type": "app", "platforms": [ "web" ], "uses": [ "showcase" ], "implementation": "ai-built", "primary_url": "https://github.com/openai/gpt-5-coding-examples/tree/main/apps/online-whiteboard", "repo_url": "https://github.com/openai/gpt-5-coding-examples", "demo_url": "https://gpt5-coding-examples.vercel.app/online-whiteboard/", "paper_url": null, "limits_en": "The published prompt explicitly limits the app to local use; no shared realtime collaboration service is provided. Hosted HTML returned 200; drawing/export were not exercised. Repository code is MIT.", "limits_zh": "公开提示明确限定为本地使用，未提供共享实时协作服务。托管 HTML 返回 200，未实际测试绘图和导出。仓库代码采用 MIT。", "enables_en": "Inspect a generated canvas interface and adapt its local drawing and export workflow.", "enables_zh": "查看生成的 canvas 界面，并参考其本地绘图与导出流程。", "resource_kind": "ai-generated-output", "runtime": "Published single-file HTML/CSS/JavaScript; inspect the hosted page or run the collection with its documented front-end development server.", "evidence": [ { "url": "https://github.com/openai/gpt-5-coding-examples#readme", "supports": "OpenAI explicitly attributes the collection to single GPT-5 or GPT-5.2 prompts without handwritten application code." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/examples/online-whiteboard.yaml", "supports": "Publisher supplies the specific example name, prompt and category." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/apps/online-whiteboard/index.html", "supports": "Published prompt says local-only; source contains pen/eraser controls and canvas-to-PNG export." }, { "url": "https://gpt5-coding-examples.vercel.app/online-whiteboard/", "supports": "Hosted HTML returned HTTP 200 and matched the published app file during this source review; no JavaScript or app interaction was executed." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/LICENSE", "supports": "Collection publishes an MIT license." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "provenance_source": "https://github.com/openai/gpt-5-coding-examples#readme" }, { "id": "gpt5-audio-step-sequencer", "name": "Audio Step Sequencer", "description_en": "An AI-generated browser drum sequencer with a sixteen-step grid, tempo and swing controls, saved patterns and WAV export code.", "description_zh": "AI 生成的浏览器鼓机音序器，包含十六步网格、速度与摇摆节奏控制、样式保存和 WAV 导出代码。", "content_type": "app", "platforms": [ "web" ], "uses": [ "showcase" ], "implementation": "ai-built", "primary_url": "https://github.com/openai/gpt-5-coding-examples/tree/main/apps/audio-step-sequencer", "repo_url": "https://github.com/openai/gpt-5-coding-examples", "demo_url": "https://gpt5-coding-examples.vercel.app/audio-step-sequencer/", "paper_url": null, "limits_en": "Requires browser WebAudio support and user interaction for audio playback. Pattern storage is local. Hosted HTML returned 200; audio timing and WAV output were not tested. Repository code is MIT.", "limits_zh": "依赖浏览器 WebAudio，播放声音需要用户交互；节奏样式保存在本地。托管 HTML 返回 200，未测试音频时序和 WAV 输出。仓库代码采用 MIT。", "enables_en": "Inspect generated WebAudio scheduling and offline audio-export logic in a standalone app.", "enables_zh": "研究独立应用中生成的 WebAudio 调度与离线音频导出逻辑。", "resource_kind": "ai-generated-output", "runtime": "Published single-file HTML/CSS/JavaScript; inspect the hosted page or run the collection with its documented front-end development server.", "evidence": [ { "url": "https://github.com/openai/gpt-5-coding-examples#readme", "supports": "OpenAI explicitly attributes the collection to single GPT-5 or GPT-5.2 prompts without handwritten application code." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/examples/audio-step-sequencer.yaml", "supports": "Publisher supplies the specific example name, prompt and category." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/apps/audio-step-sequencer/index.html", "supports": "Published source uses AudioContext, a sixteen-step loop, localStorage and OfflineAudioContext/WAV export." }, { "url": "https://gpt5-coding-examples.vercel.app/audio-step-sequencer/", "supports": "Hosted HTML returned HTTP 200 and matched the published app file during this source review; no JavaScript or app interaction was executed." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/main/LICENSE", "supports": "Collection publishes an MIT license." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "provenance_source": "https://github.com/openai/gpt-5-coding-examples#readme" }, { "id": "agentgym", "name": "AgentGym", "description_en": "A common client/server interface for interactive agent environments, accompanied by evaluation tasks, trajectories and agent-development workflows.", "description_zh": "为交互式 Agent 环境提供统一客户端与服务端接口，并配有评测任务、轨迹数据和 Agent 开发流程。", "content_type": "app", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://agentgym.github.io/", "repo_url": "https://github.com/WooooDyy/AgentGym", "demo_url": null, "paper_url": "https://arxiv.org/abs/2406.04151", "limits_en": "Installing agentenv alone does not install every environment: each server has its own dependencies and data. Integrated WebArena, games and other environments overlap existing projects. AgentGym-RL is a companion training stack. Framework code is MIT; upstream components retain their terms.", "limits_zh": "仅安装 agentenv 不会自动部署全部环境，每个服务端都有独立依赖和数据。所集成的 WebArena、游戏等与已有项目存在重合；AgentGym-RL 是配套训练栈。框架代码采用 MIT，上游组件保留自身条款。", "enables_en": "Use consistent reset, observation and action calls across environments and reuse trajectories for evaluation or behavioral training.", "enables_zh": "跨环境使用一致的重置、观察和动作接口，并复用轨迹进行评测或行为训练。", "resource_kind": "environment-framework", "runtime": "Python agentenv package with separately deployed HTTP environment servers; installation and evaluation/training tutorials are provided.", "evidence": [ { "url": "https://github.com/WooooDyy/AgentGym#readme", "supports": "Describes HTTP environment APIs, per-environment setup, AgentTraj/AgentEval and the relationship to AgentGym-RL." }, { "url": "https://github.com/WooooDyy/AgentGym/blob/main/LICENSE", "supports": "Framework code is MIT licensed." }, { "url": "https://arxiv.org/abs/2406.04151", "supports": "Original AgentGym paper explains cross-environment agent development and evaluation." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "A heterogeneous environment API layer, not a single web or desktop target." }, { "id": "agentgym-rl", "name": "AgentGym-RL", "description_en": "A multi-turn reinforcement-learning stack that connects AgentGym environment servers to rollout generation and policy optimization built on verl.", "description_zh": "将 AgentGym 环境服务连接到轨迹生成与策略优化的多轮强化学习栈，训练部分基于 verl 构建。", "content_type": "app", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://agentgym-rl.github.io/", "repo_url": "https://github.com/WooooDyy/AgentGym-RL", "demo_url": null, "paper_url": "https://arxiv.org/abs/2509.08755", "limits_en": "This adds training infrastructure to AgentGym rather than introducing an entirely separate set of environments. The documented recipe pins CUDA 12.4, PyTorch 2.4 and Python 3.10, with a Linux x86-64 attention wheel. Compute and environment servers are separate prerequisites. Code is MIT.", "limits_zh": "它为 AgentGym 增加训练基础设施，并非重新提供一套完全独立的环境。文档方案固定 CUDA 12.4、PyTorch 2.4、Python 3.10，并使用 Linux x86-64 attention wheel。计算资源与环境服务需另外准备。代码采用 MIT。", "enables_en": "Train and evaluate agents over multiple environment turns, including experiments that progressively lengthen the interaction horizon.", "enables_zh": "在多轮环境交互中训练和评估 Agent，并研究逐步延长交互时长的训练方法。", "resource_kind": "environment-framework", "runtime": "GPU-oriented Python/verl/vLLM training stack; install the specified packages, AgentGym servers and training data before running experiments.", "evidence": [ { "url": "https://github.com/WooooDyy/AgentGym-RL#readme", "supports": "Documents the verl/AgentGym relationship, multi-turn algorithms, pinned CUDA/PyTorch/Python setup and training/evaluation workflows." }, { "url": "https://github.com/WooooDyy/AgentGym-RL/blob/main/LICENSE", "supports": "Repository code is MIT licensed." }, { "url": "https://arxiv.org/abs/2509.08755", "supports": "Paper introduces the multi-turn RL framework and progressive interaction-horizon method." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "Linux appears in the documented training recipe, but no application target platform is inferred from the training host." }, { "id": "alfworld", "name": "ALFWorld", "description_en": "Aligned text and embodied household worlds where agents can learn task plans in language before attempting corresponding visual interactions.", "description_zh": "相互对齐的文字与具身家庭环境，Agent 可先通过语言学习任务规划，再尝试对应的视觉交互。", "content_type": "game", "platforms": [ "linux", "macos" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://alfworld.github.io/", "repo_url": "https://github.com/alfworld/alfworld", "demo_url": null, "paper_url": "https://arxiv.org/abs/2010.03768", "limits_en": "Text-only installation differs from visual/full extras, and game files must be downloaded. Visual execution requires rendering support; the documented Docker recipe targets an older release. macOS ARM has a documented x86 environment workaround. ALFWorld is MIT, while Fast Downward is GPLv3.", "limits_zh": "纯文字安装与 visual/full 扩展不同，还需下载游戏文件。视觉模式需要渲染支持；文档中的 Docker 方案针对旧版。macOS ARM 有明确的 x86 环境变通配置。ALFWorld 采用 MIT，Fast Downward 则采用 GPLv3。", "enables_en": "Study transfer between symbolic task planning and embodied action, with separate training and evaluation workflows.", "enables_zh": "通过独立训练与评测流程，研究符号任务规划向具身动作的迁移。", "resource_kind": "environment-and-benchmark", "runtime": "Python 3.9+; alfworld package with optional visual/full extras, downloadable task files and TextWorld/AI2-THOR backends.", "evidence": [ { "url": "https://github.com/alfworld/alfworld#readme", "supports": "Documents aligned TextWorld/ALFRED tasks, installation modes, tested Ubuntu setup, macOS workaround, rendering requirements and component licenses." }, { "url": "https://github.com/alfworld/alfworld/blob/master/LICENSE", "supports": "ALFWorld repository code is MIT licensed." }, { "url": "https://arxiv.org/abs/2010.03768", "supports": "Original paper defines the alignment between text and embodied learning environments." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "Linux and macOS are documented setup targets; visual and text-only modes have different requirements." }, { "id": "api-bank", "name": "API-Bank", "description_en": "A tool-use benchmark with runnable API implementations, evaluation dialogues and separate training data for planning, retrieving and calling tools.", "description_zh": "提供可执行 API 实现、评测对话和独立训练数据的工具使用基准，覆盖规划、检索和调用工具。", "content_type": "app", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/api-bank", "repo_url": "https://github.com/AlibabaResearch/DAMO-ConvAI", "demo_url": null, "paper_url": "https://arxiv.org/abs/2304.08244", "limits_en": "The runnable evaluation tools and the larger training API collection are different resources. Evaluation levels use different scripts. The authors report a googletrans/Gradio dependency conflict that can break the local demo. The enclosing repository is MIT licensed; external services retain their terms.", "limits_zh": "可运行的评测工具与规模更大的训练 API 集合是不同资源，各难度等级使用不同评测脚本。作者指出 googletrans 与 Gradio 的依赖冲突可能导致本地演示失败。所属仓库采用 MIT，外部服务保留自身条款。", "enables_en": "Test tool selection and argument construction against executable APIs, or study the released tool-use dialogue training set.", "enables_zh": "利用可执行 API 检查工具选择与参数构造，或研究公开的工具对话训练集。", "resource_kind": "environment-and-benchmark", "runtime": "Python requirements and local demo.py; evaluator.py/evaluator_by_json.py for lower levels and lv3_evaluator.py for planning tasks.", "evidence": [ { "url": "https://github.com/AlibabaResearch/DAMO-ConvAI/blob/main/api-bank/README.md", "supports": "Separates the runnable evaluation APIs from training data, documents evaluators and warns of the demo dependency conflict." }, { "url": "https://github.com/AlibabaResearch/DAMO-ConvAI/blob/main/api-bank/requirements.txt", "supports": "Lists the Python dependencies used by the published implementation." }, { "url": "https://github.com/AlibabaResearch/DAMO-ConvAI/blob/main/LICENSE", "supports": "Enclosing repository publishes an MIT license." }, { "url": "https://arxiv.org/abs/2304.08244", "supports": "Original API-Bank paper defines tool planning, retrieval and invocation evaluation." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "API-only benchmark; no device or desktop platform is inferred." }, { "id": "bird-interact", "name": "BIRD-Interact", "description_en": "An interactive text-to-SQL environment with database state, supporting knowledge, a user simulator and executable checks for query and update tasks.", "description_zh": "包含数据库状态、辅助知识、用户模拟器和可执行检查的交互式 text-to-SQL 环境，覆盖查询及数据修改任务。", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://bird-interact.github.io/", "repo_url": "https://github.com/bird-bench/BIRD-Interact", "demo_url": null, "paper_url": "https://arxiv.org/abs/2510.05318", "limits_en": "Full evaluation needs ground-truth/test files requested separately from the authors. Legacy and ADK runners, datasets and task counts differ; verify the chosen version and database initialization. ADK is an implementation of this benchmark, not a separate benchmark. Code is MIT; data is CC BY-SA 4.0.", "limits_zh": "完整评测需要向作者单独申请真值与测试文件。旧版和 ADK 运行器、数据集及任务数量不同，应核对所选版本及数据库初始化。ADK 是该基准的一种实现，不另算一套基准。代码采用 MIT，数据采用 CC BY-SA 4.0。", "enables_en": "Test whether agents clarify ambiguous requests, inspect database knowledge and recover from SQL errors during an interaction.", "enables_zh": "测试 Agent 能否在交互中澄清歧义、查阅数据库知识，并从 SQL 错误中恢复。", "resource_kind": "environment-and-benchmark", "runtime": "Docker PostgreSQL databases and Python evaluation services; current ADK path uses Python 3.10+, separate agent/user/database services and provider configuration.", "evidence": [ { "url": "https://github.com/bird-bench/BIRD-Interact#readme", "supports": "Documents interactive SQL modes, Docker setup, initialization checks, separately requested ground truth and dataset licensing." }, { "url": "https://github.com/bird-bench/BIRD-Interact/blob/main/BIRD-Interact-ADK/README.md", "supports": "Describes the official ADK implementation, Python prerequisites, microservices and its relationship to the original benchmark." }, { "url": "https://github.com/bird-bench/BIRD-Interact/blob/main/LICENSE", "supports": "Repository code is MIT licensed." }, { "url": "https://arxiv.org/abs/2510.05318", "supports": "Original paper defines conversational and agent-led database interaction evaluation." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "Database/tool interaction benchmark; no graphical app or host OS platform is inferred." }, { "id": "context-mcp", "name": "Context", "creator": "Indragie Karunaratne", "description_en": "A native macOS app for debugging MCP servers through tool calls, resource previews and live logs.", "description_zh": "原生 macOS MCP 调试应用，可调用工具、预览资源并查看实时日志。", "platforms": [ "macos" ], "implementation": "ai-assisted", "primary_url": "https://github.com/indragiek/Context", "repo_url": "https://github.com/indragiek/Context", "download_url": "https://github.com/indragiek/Context/releases/tag/v1.0.10", "provenance_source": "https://www.indragie.com/blog/i-shipped-a-macos-app-built-entirely-by-claude-code", "enables_en": "Explore several MCP servers in one visual client and inspect their tools, resources and responses.", "enables_zh": "在同一可视化客户端中探索多个 MCP 服务器，检查工具、资源及响应。", "limits_en": "Requires macOS 15+. The reviewed README lists protocol features still unsupported or in progress; prebuilt releases enable Sentry crash reporting. Source code is MIT-licensed.", "limits_zh": "需要 macOS 15 及以上版本。已核查的 README 列有尚未支持或开发中的协议功能；预编译版本默认启用 Sentry 崩溃报告。 源代码采用 MIT 许可。", "provenance_note_en": "The creator estimates that he wrote fewer than 1,000 of roughly 20,000 lines by hand, while Claude Code produced most of the app; human implementation and iteration remain explicit.", "provenance_note_zh": "作者估计约两万行代码中，人工编写不足一千行，大部分由 Claude Code 生成；同时明确记录了人工实现和迭代工作。", "evidence": [ { "url": "https://www.indragie.com/blog/i-shipped-a-macos-app-built-entirely-by-claude-code", "supports": "First-person development account states the substantial Claude-generated code share and the human-written portion." }, { "url": "https://github.com/indragiek/Context#readme", "supports": "Documents native SwiftUI implementation, functions, macOS requirements and protocol limitations." }, { "url": "https://github.com/indragiek/Context/releases/tag/v1.0.10", "supports": "Author release listing contains Context_v1.0.10.dmg; the binary was not downloaded or run." }, { "url": "https://github.com/indragiek/Context/blob/main/LICENSE.md", "supports": "The repository license file licenses its source code under MIT." } ], "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "davit", "name": "Davit", "creator": "Wouter de Bie", "description_en": "A native macOS interface for Apple's container platform, with container controls, image management, logs and resource monitoring.", "description_zh": "面向 Apple 容器平台的原生 macOS 界面，提供容器控制、镜像管理、日志和资源监控。", "platforms": [ "macos" ], "implementation": "ai-assisted", "primary_url": "https://davit.app/", "repo_url": "https://github.com/wouterdebie/davit", "download_url": "https://github.com/wouterdebie/davit/releases/tag/v0.1.38", "provenance_source": "https://news.ycombinator.com/item?id=48821848", "enables_en": "Manage Linux containers on an Apple Silicon Mac from a native desktop interface.", "enables_zh": "在 Apple Silicon Mac 上通过原生桌面界面管理 Linux 容器。", "limits_en": "Requires an Apple Silicon Mac and macOS 15+ plus Apple's container platform. The macOS tag describes the desktop application; the containers it manages run Linux. Source code is MIT-licensed.", "limits_zh": "需要 Apple Silicon Mac、macOS 15 及以上版本和 Apple 容器平台。macOS 标签指桌面应用，受管理的容器运行 Linux。 源代码采用 MIT 许可。", "provenance_note_en": "The Show HN launch under xinit describes Davit as mostly vibe-coded. Separately, maintainer-authored native Swift changes explicitly record Claude co-authorship.", "provenance_note_zh": "署名 xinit 的 Show HN 发布帖称 Davit 主要通过 AI 编程构建；另有维护者提交的原生 Swift 代码变更明确记录 Claude 共同署名。", "evidence": [ { "url": "https://news.ycombinator.com/item?id=48821848", "supports": "The first-person project launch describes Davit as mostly vibe-coded and points to its official site/source." }, { "url": "https://github.com/wouterdebie/davit#readme", "supports": "Documents native SwiftUI implementation, container-management functions and Apple Silicon/macOS requirements." }, { "url": "https://github.com/wouterdebie/davit/releases/tag/v0.1.38", "supports": "Maintainer release listing includes DMG and ZIP artifacts; no executable was downloaded or run." }, { "url": "https://github.com/wouterdebie/davit/commit/68c6984126b7903e94f9e33b11a39c948482cb68", "supports": "Maintainer-authored changes to the native Swift application explicitly record Claude co-authorship." }, { "url": "https://github.com/wouterdebie/davit/blob/main/LICENSE", "supports": "The repository license file licenses its source code under MIT." } ], "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "dog-e-dex", "name": "Dog-e-dex", "creator": "Cynthia Chen", "description_en": "An iPhone app for identifying dogs from photos and saving them, names and notes in a personal collection.", "description_zh": "通过照片识别犬种，并将照片、名字和笔记保存到个人收藏中的 iPhone 应用。", "platforms": [ "ios" ], "implementation": "ai-built", "primary_url": "https://apps.apple.com/us/app/dog-e-dex/id6741032284", "repo_url": null, "download_url": "https://apps.apple.com/us/app/dog-e-dex/id6741032284", "provenance_source": "https://www.producthunt.com/products/dog-e-dex?launch=957816", "enables_en": "Build a personal photo collection of dogs encountered and revisit their saved information.", "enables_zh": "整理遇见过的狗狗照片，并回看保存的信息。", "limits_en": "Requires iOS 17.6+. The reviewed store listing describes cloud-backed photo storage; this review did not test recognition or synchronization, and no source repository was identified.", "limits_zh": "需要 iOS 17.6 及以上版本。已核查的商店说明包含云端照片存储；本轮未测试识别或同步，也未找到公开源码仓库。", "provenance_note_en": "Cynthia Chen's maker statement explicitly says she built Dog-e-dex purely by prompting Claude; the App Store identifies the same creator as seller.", "provenance_note_zh": "Cynthia Chen 在创作者说明中明确表示，Dog-e-dex 完全通过向 Claude 提示构建；App Store 的销售者姓名与之相同。", "evidence": [ { "url": "https://www.producthunt.com/products/dog-e-dex?launch=957816", "supports": "The maker's own pinned comment explicitly attributes construction of the app to prompting Anthropic's Claude." }, { "url": "https://apps.apple.com/us/app/dog-e-dex/id6741032284", "supports": "Current publisher listing identifies Cynthia Chen, the iPhone app, features and iOS minimum; no installation was performed." } ], "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "harbor", "name": "Harbor", "description_en": "An execution framework for agent evaluations and training rollouts, with reusable task formats and local or remote sandbox backends.", "description_zh": "用于 Agent 评测与训练轨迹生成的执行框架，提供可复用任务格式以及本地或远程沙盒后端。", "content_type": "app", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://harborframework.com/docs", "repo_url": "https://github.com/harbor-framework/harbor", "demo_url": null, "paper_url": null, "limits_en": "Harbor is the runner used by Terminal-Bench, not a separate task benchmark. Current code requires Python 3.12+. Local runs need Docker; remote providers and model endpoints need their own configuration. Dataset licenses remain separate from Apache-2.0 framework code.", "limits_zh": "Harbor 是 Terminal-Bench 使用的运行框架，本身不是另一套任务基准。当前代码要求 Python 3.12+；本地运行需要 Docker，远程服务和模型端点需各自配置。框架采用 Apache 2.0，数据集许可另行适用。", "enables_en": "Run the same agent across selected task datasets, or collect trajectories for subsequent reinforcement learning.", "enables_zh": "在选定的任务数据集上运行同一 Agent，或收集轨迹用于后续强化学习。", "resource_kind": "environment-framework", "runtime": "Python package via uv tool install harbor or pip install harbor; Docker locally or configured cloud sandbox integrations.", "evidence": [ { "url": "https://github.com/harbor-framework/harbor#readme", "supports": "Documents evaluation, benchmark authoring, local Docker, remote execution and RL rollout generation." }, { "url": "https://github.com/harbor-framework/harbor/blob/main/pyproject.toml", "supports": "Current package requires Python 3.12+ and declares Apache-2.0 licensing." }, { "url": "https://github.com/harbor-framework/harbor/blob/main/LICENSE", "supports": "Framework license is Apache 2.0." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "Infrastructure framework; platforms depend on the selected task and backend." }, { "id": "headroom-ai", "name": "Headroom AI", "creator": "MariuszT", "description_en": "A macOS menu-bar app that displays usage windows and reset times for multiple Claude Code and Codex accounts.", "description_zh": "在 macOS 菜单栏中展示多个 Claude Code 和 Codex 账户用量窗口及重置时间的应用。", "platforms": [ "macos" ], "implementation": "ai-assisted", "primary_url": "https://github.com/MariuszT/headroom-ai", "repo_url": "https://github.com/MariuszT/headroom-ai", "download_url": "https://github.com/MariuszT/headroom-ai/releases/tag/v1.3.2", "provenance_source": "https://github.com/MariuszT/headroom-ai#how-this-was-built", "enables_en": "Compare remaining account capacity in a single panel while keeping the app's sign-ins separate from CLI sessions.", "enables_zh": "在一个面板中比较多个账户的剩余额度，应用登录与命令行会话分别管理。", "limits_en": "Requires macOS 14+. The maintainer describes it as unofficial and dependent on provider endpoints that may change; usage display does not switch CLI accounts. Source code is MIT-licensed.", "limits_zh": "需要 macOS 14 及以上版本。维护者说明这是依赖可能变动的提供商接口的非官方工具；展示用量不会切换命令行账户。 源代码采用 MIT 许可。", "provenance_note_en": "The maintainer says development used Claude Code from start to finish and credits existing MIT-licensed projects for API/OAuth work, so the entry uses the mixed-scope AI-assisted label.", "provenance_note_zh": "维护者明确表示全程使用 Claude Code 开发，同时注明 API/OAuth 工作参考了现有 MIT 项目，因此使用 AI 辅助开发标签。", "evidence": [ { "url": "https://github.com/MariuszT/headroom-ai#how-this-was-built", "supports": "Maintainer explicitly describes start-to-finish Claude Code development; the adjacent credits identify reused knowledge/components." }, { "url": "https://github.com/MariuszT/headroom-ai#readme", "supports": "Documents features, macOS requirements, separate OAuth sessions and unsupported-endpoint limitations." }, { "url": "https://github.com/MariuszT/headroom-ai/releases/tag/v1.3.2", "supports": "Release listing includes Headroom-AI.dmg; the binary was not downloaded or run." }, { "url": "https://github.com/MariuszT/headroom-ai/blob/main/LICENSE", "supports": "The repository license file licenses its source code under MIT." } ], "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "intercode", "name": "InterCode", "description_en": "An interactive coding environment framework that returns execution feedback during Bash, SQL, Python and capture-the-flag tasks.", "description_zh": "在 Bash、SQL、Python 和夺旗任务执行过程中返回反馈的交互式编程环境框架。", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://intercode-benchmark.github.io/", "repo_url": "https://github.com/princeton-nlp/intercode", "demo_url": null, "paper_url": "https://arxiv.org/abs/2306.14898", "limits_en": "Requires Python 3.8+ and a running Docker service with task images. Included task types and setup paths differ; model-driven experiment scripts require configured provider access. This is execution-based evaluation rather than a static code dataset. Repository code is MIT.", "limits_zh": "需要 Python 3.8+、运行中的 Docker 服务及任务镜像。不同任务类型的配置流程不同；模型驱动的实验脚本需要相应服务访问。它基于实际执行反馈进行评测，并非静态代码数据集。仓库代码采用 MIT。", "enables_en": "Build coding tasks and measure how agents use execution errors and state feedback to refine subsequent actions.", "enables_zh": "构建编程任务，观察 Agent 如何利用执行错误与状态反馈修正后续动作。", "resource_kind": "environment-framework", "runtime": "Python intercode-bench package or source installation; Docker-backed task environments and local interactive demos.", "evidence": [ { "url": "https://github.com/princeton-nlp/intercode#readme", "supports": "Documents Docker/Python requirements, interactive APIs, Bash/SQL/Python/CTF environments and experiment setup." }, { "url": "https://github.com/princeton-nlp/intercode/blob/master/LICENSE.md", "supports": "Repository license is MIT." }, { "url": "https://arxiv.org/abs/2306.14898", "supports": "Original paper describes interactive code evaluation with execution feedback." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "Code tasks run in configured containers; no host OS is inferred as the target platform." }, { "id": "mlgym", "name": "MLGym", "description_en": "A Gym-style research environment where agents modify machine-learning methods, run experiments and receive task-specific feedback.", "description_zh": "采用 Gym 风格接口的研究环境，Agent 可修改机器学习方法、运行实验并获得任务反馈。", "content_type": "app", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/facebookresearch/MLGym", "repo_url": "https://github.com/facebookresearch/MLGym", "demo_url": null, "paper_url": "https://arxiv.org/abs/2502.14499", "limits_en": "The author repository is archived as of this review, despite older README maintenance language. Experiments require Docker/Podman and task-dependent GPU/data resources. Most code is CC BY-NC 4.0, with MIT/Apache components; it is not uniformly licensed for commercial use.", "limits_zh": "截至本次核查，作者仓库已归档，虽然旧 README 仍有维护说明。实验依赖 Docker/Podman，以及具体任务所需的 GPU 和数据。多数代码采用 CC BY-NC 4.0，部分组件采用 MIT/Apache，并非统一允许商业使用。", "enables_en": "Evaluate iterative AI-research workflows or study reinforcement learning for agents that design and test ML experiments.", "enables_zh": "评估迭代式 AI 研究流程，或研究面向机器学习实验设计与测试 Agent 的强化学习。", "resource_kind": "environment-and-benchmark", "runtime": "Source installation, Docker/Podman task containers and configured model access; README documents Linux/macOS host setup and GPU container requirements.", "evidence": [ { "url": "https://github.com/facebookresearch/MLGym#readme", "supports": "Documents ML tasks, Gym-style agent training/evaluation, container/GPU setup and component-specific license terms." }, { "url": "https://api.github.com/repos/facebookresearch/MLGym", "supports": "GitHub repository metadata reports archived=true during the 2026-10-16 source review." }, { "url": "https://github.com/facebookresearch/MLGym/blob/main/LICENSE", "supports": "Root license is CC BY-NC 4.0." }, { "url": "https://arxiv.org/abs/2502.14499", "supports": "Original paper describes the framework and AI research task benchmark." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "Linux/macOS are host setup instructions; the target tasks are ML experiments, so application platform tags remain empty." }, { "id": "onebotan", "name": "OneBotan", "creator": "mirute02", "description_en": "An Android home-screen widget for recording activity timestamps, counting presses and tracking on/off intervals, with history and CSV export.", "description_zh": "用于记录活动时间、统计点击次数和跟踪开关持续时间的 Android 桌面小组件，提供历史记录和 CSV 导出。", "platforms": [ "android" ], "implementation": "ai-built", "primary_url": "https://github.com/mirute02/onebotan", "repo_url": "https://github.com/mirute02/onebotan", "download_url": "https://github.com/mirute02/onebotan/releases/tag/v1.0.0", "provenance_source": "https://zenn.dev/mirute02/articles/android-only-termux-claude-code", "enables_en": "Log recurring activities from the home screen and inspect or export their timing history.", "enables_zh": "从桌面记录重复活动，并查看或导出时间历史。", "limits_en": "Requires Android 12+. Distribution is via a released APK rather than Google Play; records stay on the device and the README documents CSV export for backup. Source code is MIT-licensed.", "limits_zh": "需要 Android 12 及以上版本，通过发布的 APK 分发。记录保存在本机，README 提供 CSV 导出备份说明。 源代码采用 MIT 许可。", "provenance_note_en": "The creator says their work was giving requirements and trying APKs on-device, while Claude Code wrote code, built the app, wrote and ran tests, and committed changes.", "provenance_note_zh": "作者说明自己负责提出需求和在实机上试用 APK，代码编写、构建、测试编写与执行、提交均交由 Claude Code 完成。", "evidence": [ { "url": "https://zenn.dev/mirute02/articles/android-only-termux-claude-code", "supports": "The creator's development diary identifies this repository and explicitly separates human requests/testing from Claude's implementation work." }, { "url": "https://github.com/mirute02/onebotan/blob/main/README.en.md", "supports": "Documents Android 12+, widget modes, history, CSV support and local record storage." }, { "url": "https://github.com/mirute02/onebotan/releases/tag/v1.0.0", "supports": "Author release listing contains onebotan-1.0.0.apk; the APK was not downloaded or run." }, { "url": "https://github.com/mirute02/onebotan/blob/main/LICENSE", "supports": "The repository license file licenses its source code under MIT." } ], "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "openenv", "name": "OpenEnv", "description_en": "A framework for packaging interactive environments behind a shared step/reset API, with Docker deployment, reward interfaces and training integrations.", "description_zh": "将交互环境封装为统一 step/reset API 的框架，提供 Docker 部署、奖励接口和训练集成。", "content_type": "app", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://huggingface.co/docs/openenv", "repo_url": "https://github.com/huggingface/OpenEnv", "demo_url": null, "paper_url": null, "limits_en": "OpenEnv standardizes environment execution; it is not one benchmark or an AI-built software showcase. Python 3.10+ is required, and clients, servers, containers and credentials vary by environment. Some integrations require newer Python. Framework code is BSD 3-Clause; environment terms are separate.", "limits_zh": "OpenEnv 统一环境执行方式，本身不是单一基准，也不是 AI 构建作品。需要 Python 3.10+；具体客户端、服务端、容器和凭据依环境而定，部分集成要求更高版本 Python。框架采用 BSD 3-Clause，环境条款另行适用。", "enables_en": "Build and share agent environments, connect them to RL trainers, or evaluate agents through a consistent interface.", "enables_zh": "构建和共享 Agent 环境、连接强化学习训练器，或通过统一接口进行评测。", "resource_kind": "environment-framework", "runtime": "Python openenv package; WebSocket client/server API with isolated Docker environments and optional hosted deployments.", "evidence": [ { "url": "https://github.com/huggingface/OpenEnv#readme", "supports": "Documents environment creation, step/reset/state APIs, Docker deployment, training integrations and evaluation support." }, { "url": "https://github.com/huggingface/OpenEnv/blob/main/pyproject.toml", "supports": "Package requires Python 3.10+; Harbor integration requires Python 3.12+." }, { "url": "https://github.com/huggingface/OpenEnv/blob/main/LICENSE", "supports": "Current framework code is licensed BSD 3-Clause." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "Transport and hosting are not target application platforms; individual environments determine their own targets." }, { "id": "stabletoolbench", "name": "StableToolBench", "description_en": "A ToolBench-derived environment that combines cached API responses with model-based simulation and revised tool-use evaluation procedures.", "description_zh": "由 ToolBench 衍生的环境，结合缓存 API 响应、模型模拟和调整后的工具使用评测流程。", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/THUNLP-MT/StableToolBench", "repo_url": "https://github.com/THUNLP-MT/StableToolBench", "demo_url": null, "paper_url": "https://arxiv.org/abs/2403.07714", "limits_en": "Simulated or cached API responses do not establish that live services perform the same actions. The repository now offers MirrorAPI and GPT-based paths with different models, caches and evaluators. Setup requires downloaded tool data and either model hosting or API access. Code is Apache 2.0.", "limits_zh": "模拟或缓存的 API 响应不能证明真实服务会执行相同动作。仓库现提供 MirrorAPI 与 GPT 两类方案，模型、缓存和评审方式不同。运行需下载工具数据，并准备模型服务或 API 访问。代码采用 Apache 2.0。", "enables_en": "Evaluate tool-use agents against a controlled virtual API server while recording the simulator, cache and judging configuration.", "enables_zh": "通过可控的虚拟 API 服务评估工具型 Agent，同时记录模拟器、缓存与评分配置。", "resource_kind": "environment-and-benchmark", "runtime": "Python/FastAPI-style virtual server; requirements file, optional Docker, downloadable caches, and MirrorAPI/vLLM or configured GPT service.", "evidence": [ { "url": "https://github.com/THUNLP-MT/StableToolBench#readme", "supports": "Documents ToolBench lineage, MirrorAPI versus GPT-cache paths, downloads, server setup and evaluation variants." }, { "url": "https://github.com/THUNLP-MT/StableToolBench/blob/master/LICENSE", "supports": "Repository code is Apache 2.0 licensed." }, { "url": "https://arxiv.org/abs/2403.07714", "supports": "Original paper introduces virtual API serving and a revised evaluation system." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "The API server transports tool calls; it is not a browser-task environment." }, { "id": "swe-gym", "name": "SWE-Gym", "description_en": "Executable Python-repository tasks and test verification for training software-engineering agents and solution verifiers.", "description_zh": "面向 Python 仓库问题的可执行任务与测试验证环境，用于训练软件工程 Agent 和解法验证器。", "content_type": "app", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/SWE-Gym/SWE-Gym", "repo_url": "https://github.com/SWE-Gym/SWE-Gym", "demo_url": null, "paper_url": "https://arxiv.org/abs/2412.21139", "limits_en": "SWE-Gym supplies training tasks and is distinct from SWE-bench evaluation sets. Reproduction instructions use a specific OpenHands fork; switching versions can change outcomes. Task containers, repository data and model compute are required. Framework code is Apache 2.0; bundled projects retain their licenses.", "limits_zh": "SWE-Gym 提供训练任务，与 SWE-bench 评测集不同。复现说明使用特定 OpenHands 分支，切换版本会影响结果。需要任务容器、仓库数据和模型计算资源。框架代码采用 Apache 2.0，所含项目保留各自许可。", "enables_en": "Collect verified software-repair trajectories, train coding agents and train verifiers that select candidate solutions.", "enables_zh": "收集带测试验证的软件修复轨迹，训练编程 Agent，以及用于选择候选解法的验证器。", "resource_kind": "environment-and-benchmark", "runtime": "Released task data and per-instance x86-64 Docker images; documented OpenHands/MoatlessTools workflows and separate training compute.", "evidence": [ { "url": "https://github.com/SWE-Gym/SWE-Gym#readme", "supports": "Documents executable task environments, tests, training purpose, container images and links to reproduction instructions." }, { "url": "https://github.com/SWE-Gym/SWE-Gym/blob/main/docs/OpenHands.md", "supports": "Requires a specific OpenHands fork and distinguishes rollout/evaluation from model-training setup." }, { "url": "https://github.com/SWE-Gym/SWE-Gym/blob/main/LICENSE", "supports": "Repository code is Apache 2.0 licensed." }, { "url": "https://arxiv.org/abs/2412.21139", "supports": "Original paper distinguishes SWE-Gym training environments from downstream SWE-bench evaluation." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "Repository-repair tasks are not tagged by the operating system of the runner." }, { "id": "terminal-bench", "name": "Terminal-Bench", "description_en": "A versioned collection of executable terminal tasks with containerized starting environments, reference solutions and automated checks for agent outcomes.", "description_zh": "按版本发布的可执行终端任务集合，配有容器化初始环境、参考解法和用于判断 Agent 结果的自动检查。", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://www.tbench.ai/", "repo_url": "https://github.com/harbor-framework/terminal-bench", "demo_url": null, "paper_url": "https://arxiv.org/abs/2601.11868", "limits_en": "The current repository is a continuously updated benchmark, while the linked paper describes Terminal-Bench 2.0. Pin dataset and harness versions for comparisons. Harbor and a supported sandbox backend are required; provider costs and task dependencies vary. Repository license is Apache 2.0.", "limits_zh": "当前仓库是持续更新的基准，所链接论文描述的是 Terminal-Bench 2.0。比较结果时需固定数据集与评测框架版本。运行需要 Harbor 和受支持的沙盒后端；服务费用与任务依赖各不相同。仓库采用 Apache 2.0。", "enables_en": "Test whether an agent can complete multi-step command-line work against executable outcome checks.", "enables_zh": "通过可执行的结果检查，测试 Agent 能否完成多步骤命令行工作。", "resource_kind": "environment-and-benchmark", "runtime": "Runs through Harbor using versioned datasets and container/cloud sandbox providers; current main recommends validating oracle solutions in the chosen backend.", "evidence": [ { "url": "https://github.com/harbor-framework/terminal-bench#readme", "supports": "Current source describes continuous tagged releases, Harbor execution, sandbox validation and the dataset registry." }, { "url": "https://github.com/harbor-framework/terminal-bench-2#readme", "supports": "Preserves the distinct Terminal-Bench 2.0 dataset and documents migration from the legacy harness to Harbor." }, { "url": "https://github.com/harbor-framework/terminal-bench/blob/main/LICENSE", "supports": "Repository license is Apache 2.0." }, { "url": "https://arxiv.org/abs/2601.11868", "supports": "Research paper describes the Terminal-Bench 2.0 task design and evaluation setting, not every subsequent release." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "Terminal/code tasks use isolated backends; no host OS is inferred as a directory platform." }, { "id": "toolsandbox", "name": "ToolSandbox", "description_en": "A stateful tool-use evaluation environment with simulated conversations, dependent tool actions and checks on intermediate and final world states.", "description_zh": "有状态的工具使用评测环境，结合模拟对话、相互依赖的工具动作，以及对中间和最终环境状态的检查。", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://machinelearning.apple.com/research/toolsandbox-stateful-conversational-llm-benchmark", "repo_url": "https://github.com/apple-aiml-research/ToolSandbox", "demo_url": null, "paper_url": "https://arxiv.org/abs/2408.04682", "limits_en": "Despite its name, documented tool execution runs on the host rather than inside Docker. Model credentials are needed, and search scenarios also use RapidAPI. Apple publishes custom license terms plus separately licensed components. It simulates device state; it is not an iOS emulator.", "limits_zh": "虽然名称含 Sandbox，文档中的工具执行发生在主机上，而非 Docker 内。需要模型凭据，搜索场景还使用 RapidAPI。Apple 发布了定制许可条款，部分组件另有许可。它模拟设备状态，并不是 iOS 模拟器。", "enables_en": "Evaluate whether agents handle state dependencies, missing information and conversational tool use across a complete trajectory.", "enables_zh": "评估 Agent 在完整交互轨迹中处理状态依赖、信息缺失和对话式工具调用的能力。", "resource_kind": "environment-and-benchmark", "runtime": "Python package and tool_sandbox CLI; documented setup uses Python 3.9; provider configuration depends on selected user and agent roles.", "evidence": [ { "url": "https://github.com/apple-aiml-research/ToolSandbox#readme", "supports": "Documents stateful tools, user simulation, host execution, provider/RapidAPI requirements and Python setup." }, { "url": "https://github.com/apple-aiml-research/ToolSandbox/blob/main/LICENSE", "supports": "Apple-specific license notice points to separately licensed components." }, { "url": "https://arxiv.org/abs/2408.04682", "supports": "Original paper defines stateful conversational tool-use evaluation." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "Simulated settings/contacts/messages are accessed through tools, not a documented iOS or macOS application target." }, { "id": "vibetunnel", "name": "VibeTunnel", "creator": "VibeTunnel team", "description_en": "A terminal-session manager with a native Mac app and browser interface for viewing output and interacting with running shells remotely.", "description_zh": "提供原生 Mac 应用和浏览器界面的终端会话管理器，可远程查看输出并操作运行中的 Shell。", "platforms": [ "macos", "linux", "web" ], "implementation": "ai-assisted", "primary_url": "https://vibetunnel.sh/", "repo_url": "https://github.com/amantus-ai/vibetunnel", "download_url": "https://github.com/amantus-ai/vibetunnel/releases/tag/v1.0.0-beta.18", "provenance_source": "https://steipete.me/posts/2025/vibetunnel-turn-any-browser-into-your-mac-terminal", "enables_en": "Follow terminal sessions from another device and send input without moving the underlying process to that device.", "enables_zh": "从另一台设备查看终端会话并输入命令，底层进程仍在原主机运行。", "limits_en": "The reviewed release is beta. The Mac bundle requires Apple Silicon and macOS 14+; Linux uses the documented npm/Node setup. The separate iOS client remains a work in progress. Source code is MIT-licensed.", "limits_zh": "已核查版本为 Beta。Mac 安装包要求 Apple Silicon 和 macOS 14 及以上；Linux 使用文档中的 npm/Node 方案，独立 iOS 客户端仍在开发中。 源代码采用 MIT 许可。", "provenance_note_en": "A creator's launch account describes Claude generating implementations, followed by developer testing, refactoring and reuse of existing components.", "provenance_note_zh": "创作者的发布文章记录了 Claude 生成实现，以及开发者随后测试、重构和复用既有组件的过程。", "evidence": [ { "url": "https://steipete.me/posts/2025/vibetunnel-turn-any-browser-into-your-mac-terminal", "supports": "First-party account explains substantial Claude Code generation alongside human debugging, refactoring and component reuse." }, { "url": "https://github.com/amantus-ai/vibetunnel#readme", "supports": "Documents the Mac bundle, Linux npm installation, browser client and unfinished iOS client." }, { "url": "https://github.com/amantus-ai/vibetunnel/releases/tag/v1.0.0-beta.18", "supports": "The prerelease listing contains Mac DMG and ZIP artifacts; no executable was run." }, { "url": "https://github.com/amantus-ai/vibetunnel/blob/main/LICENSE", "supports": "The repository license file licenses its source code under MIT." } ], "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "video-lite-compress-resize", "name": "Video Lite: Compress & Resize", "creator": "Ivan Vdovitsa", "description_en": "An iPhone and iPad app that automatically re-encodes videos into smaller files and supports background processing and sharing.", "description_zh": "自动将视频重新编码为较小文件的 iPhone/iPad 应用，支持后台处理和分享。", "platforms": [ "ios" ], "implementation": "ai-assisted", "primary_url": "https://apps.apple.com/us/app/video-lite-compress-resize/id6776254843", "repo_url": null, "download_url": "https://apps.apple.com/us/app/video-lite-compress-resize/id6776254843", "provenance_source": "https://www.reddit.com/r/vibecoding/comments/1u1crsb/i_shipped_my_first_ios_app_video_compressor_with/", "enables_en": "Reduce the size of large videos or screen recordings before sharing or storing them.", "enables_zh": "在分享或保存前缩小大型视频及录屏文件的体积。", "limits_en": "Requires iOS/iPadOS 26+. Compression is automatic; the reviewed sources provide an App Store app, without an identified public source repository or independent quality tests.", "limits_zh": "需要 iOS/iPadOS 26 及以上版本，压缩由应用自动处理。已核查来源提供 App Store 应用，但未找到公开源码，也未独立测试画质。", "provenance_note_en": "A first-person launch post by u/vdovitsa says this app was built with Claude and Claude Code in Xcode and links the exact App Store ID. Apple lists Ivan Vdovitsa as seller.", "provenance_note_zh": "u/vdovitsa 的第一人称发布帖说明，应用在 Xcode 中借助 Claude 与 Claude Code 构建，并链接到确切的 App Store ID；Apple 将 Ivan Vdovitsa 列为销售者。", "evidence": [ { "url": "https://www.reddit.com/r/vibecoding/comments/1u1crsb/i_shipped_my_first_ios_app_video_compressor_with/", "supports": "Creator u/vdovitsa describes using Claude/Claude Code for the complete app and links App Store ID 6776254843." }, { "url": "https://apps.apple.com/us/app/video-lite-compress-resize/id6776254843", "supports": "Publisher listing identifies Ivan Vdovitsa, documents the video workflow and iOS requirement; no app was run." }, { "url": "https://itunes.apple.com/lookup?id=6776254843&country=us", "supports": "Apple's lookup metadata independently resolves the exact app ID, seller, current title and minimum OS." } ], "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "tau-bench", "name": "τ-Bench", "description_en": "A customer-service simulation framework with domain policies, executable tools and simulated users; the current repository extends τ²-Bench into τ³ text, voice and knowledge tasks.", "description_zh": "结合领域政策、可执行工具和模拟用户的客服环境；当前仓库从 τ²-Bench 扩展为包含文字、语音和知识任务的 τ³ 框架。", "content_type": "app", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/sierra-research/tau2-bench", "repo_url": "https://github.com/sierra-research/tau2-bench", "demo_url": null, "paper_url": "https://arxiv.org/abs/2506.07982", "limits_en": "The original tau-bench repository is explicitly outdated; tau2-bench now hosts τ³. Requires Python 3.12–3.13 and configured model access. Banking results before and after v1.0.1 are not comparable. Text, voice and knowledge modes have different dependencies; code is MIT.", "limits_zh": "原 tau-bench 仓库已明确停止更新任务，tau2-bench 现承载 τ³。需要 Python 3.12–3.13 和模型访问配置。banking 领域在 v1.0.1 前后的结果不可直接比较。文字、语音和知识模式依赖不同；代码采用 MIT。", "enables_en": "Evaluate policy-following and coordination with users who can also change the world, or connect training through the Gymnasium interface.", "enables_zh": "评估政策遵循，以及与同样能改变环境状态的用户协作；也可通过 Gymnasium 接口接入训练。", "resource_kind": "environment-and-benchmark", "runtime": "Python and uv; core text-mode installation plus optional voice, knowledge and Gym extras; provider credentials depend on selected models.", "evidence": [ { "url": "https://github.com/sierra-research/tau2-bench#readme", "supports": "Documents current τ³ domains, Python constraints, optional dependencies and v1.0.1 banking grading incompatibility." }, { "url": "https://github.com/sierra-research/tau-bench#readme", "supports": "Legacy repository warns its tasks are outdated and directs users to the current tau2-bench repository." }, { "url": "https://github.com/sierra-research/tau2-bench/blob/main/src/tau2/gym/README.md", "supports": "Provides step/reset interfaces for agent and user simulation, including RL use." }, { "url": "https://github.com/sierra-research/tau2-bench/blob/main/LICENSE", "supports": "Current framework code is MIT licensed." }, { "url": "https://arxiv.org/abs/2506.07982", "supports": "The τ² paper introduces dual-control evaluation; later τ³ additions are described separately by the repository." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "API/simulated-domain framework; host OS setup is not treated as a target application platform." }, { "id": "cowboy-shooter", "name": "Cowboy Shooter", "creator": "Raymelon (@pseudokid)", "content_type": "game", "implementation": "ai-built", "description_en": "A browser-based 3D runner and shooting game in which players change lanes, jump obstacles and shoot targets as levels progress.", "description_zh": "浏览器中的三维跑酷射击游戏，玩家随关卡推进切换跑道、跳过障碍并射击目标。", "primary_url": "https://cowboy.raymelon.com/", "repo_url": null, "demo_url": "https://cowboy.raymelon.com/", "provenance_source": "https://cowboy.raymelon.com/", "enables_en": "Try a playable example of AI-generated game code and assets, with keyboard and touch controls.", "enables_zh": "体验包含 AI 生成游戏代码及素材的可玩作品，支持键盘和触屏控制。", "limits_en": "The author hosts the game; no public source repository was identified. The interface includes username and optional reward-service flows; gameplay was not independently tested.", "limits_zh": "游戏由作者托管，未找到公开源码仓库。界面包含用户名和可选奖励服务流程，本轮未独立测试游戏操作。", "provenance_note_en": "The official game page explicitly attributes its game assets and code to AI, names Cursor and Claude, and separately credits Three.js as the existing engine.", "provenance_note_zh": "游戏官方页面明确说明游戏素材与代码由 AI 生成，列出 Cursor 和 Claude，并单独注明使用已有的 Three.js 引擎。", "evidence": [ { "url": "https://cowboy.raymelon.com/", "supports": "First-party game page explicitly claims AI-generated assets and code, credits the maker and tools, and documents controls." }, { "url": "https://cowboy.raymelon.com/about.html", "supports": "Maker's About page links back to the exact game and identifies it as the creator's game-jam project." }, { "url": "https://cowboy.raymelon.com/", "supports": "The public artifact returned HTTP 200 on the review date; no account, payment or gameplay interaction was performed." } ], "platforms": [ "web" ], "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "doomscroll", "name": "DOOMscroll", "creator": "David Friedman", "content_type": "game", "implementation": "ai-assisted", "description_en": "A Doom-inspired browser game controlled by scrolling or swiping, with automatic firing, monsters, weapon upgrades and news-headline plaques.", "description_zh": "受 Doom 启发、通过滚动或滑动操作的浏览器游戏，包含自动射击、怪物、武器升级和新闻标题铭牌。", "primary_url": "https://gisnep.com/doomscroll/", "repo_url": null, "demo_url": "https://gisnep.com/doomscroll/", "provenance_source": "https://ironicsans.ghost.io/doomscrolling-the-game/", "enables_en": "Explore how a single scrolling action can drive a complete game and read the creator's AI-assisted prototyping process.", "enables_zh": "体验单一滚动动作如何驱动完整游戏，并了解作者借助 AI 进行原型开发的过程。", "limits_en": "This entry covers the browser version. The game displays externally sourced headlines; neither gameplay performance nor headline freshness was independently verified.", "limits_zh": "本条目仅核查浏览器版本。游戏展示来自外部的新闻标题，本轮未独立验证运行表现或标题时效性。", "provenance_note_en": "Friedman's first-person account links the live game and details using GPT-5 to create its prototype, generate test pages and iteratively implement visual and gameplay changes.", "provenance_note_zh": "Friedman 的第一人称文章链接到游戏，并详细记录了借助 GPT-5 创建原型、生成测试页面及迭代视觉和玩法功能的过程。", "evidence": [ { "url": "https://ironicsans.ghost.io/doomscrolling-the-game/", "supports": "Creator describes substantial GPT-5 implementation work, prompts and iterative design changes, with direct links to the exact public game." }, { "url": "https://gisnep.com/doomscroll/", "supports": "The live page links back to the creator's story and documents scrolling/swiping, automatic firing and upgrades." }, { "url": "https://gisnep.com/doomscroll/", "supports": "The browser-game page returned HTTP 200 on the review date; no game session was started." } ], "platforms": [ "web" ], "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "aldi-prices", "name": "Aldi Prices", "creator": "Jim Lawruk", "content_type": "website", "implementation": "ai-built", "description_en": "A personal grocery-price history website with product tables, per-product timelines and charts generated from a maintained CSV.", "description_zh": "根据维护的 CSV 展示商品表格、单品历史和图表的个人杂货价格记录网站。", "primary_url": "https://aldi-prices.lawruk.com/", "repo_url": "https://github.com/jimlawruk/aldi-prices", "demo_url": "https://aldi-prices.lawruk.com/", "provenance_source": "https://news.ycombinator.com/item?id=45644813", "enables_en": "Inspect a practical AI-generated data website and browse the public interface code behind its historical observations.", "enables_zh": "查看实用的 AI 生成数据网站，并浏览其历史记录展示界面的公开代码。", "limits_en": "The creator describes this as a personal price log. Its CSV is not a comprehensive or live retailer price feed; coverage and freshness depend on the recorded observations. A source-code reuse license was not identified in the reviewed repository.", "limits_zh": "作者将其描述为个人价格记录。CSV 并非完整或实时的零售商价格源，覆盖范围和时效取决于记录的数据。 本轮查阅的仓库中未确认源代码复用许可。", "provenance_note_en": "The maker's own comment states that all application code must be produced by Copilot Agent in VS Code and links both this website and its repository.", "provenance_note_zh": "作者本人明确表示所有应用代码都交由 VS Code 中的 Copilot Agent 编写，并在同一条说明中链接网站和仓库。", "evidence": [ { "url": "https://news.ycombinator.com/item?id=45644813", "supports": "Jim Lawruk explicitly states the all-code-by-Copilot rule and cross-links the exact live site and source repository." }, { "url": "https://hacker-news.firebaseio.com/v0/item/45644813.json", "supports": "Hacker News' official API corroborates the author and text of the specific maker comment." }, { "url": "https://github.com/jimlawruk/aldi-prices#readme", "supports": "Repository documents CSV input, Angular implementation, product views, historical charts and local build instructions." }, { "url": "https://aldi-prices.lawruk.com/", "supports": "The public application shell was inspected. Later search and history-page interactions are recorded separately in data/observations.json; prices and calculations were not validated." } ], "platforms": [ "web" ], "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "social-media-card-cropper", "name": "Social Media Card Cropper", "creator": "Simon Willison", "content_type": "website", "implementation": "ai-assisted", "description_en": "A browser image tool for composing social-media cards with selectable aspect ratios, background colors, crop previews and downloadable image formats.", "description_zh": "在浏览器中制作社交媒体图片卡片的工具，提供画面比例、背景色、裁剪预览和图片下载。", "primary_url": "https://tools.simonwillison.net/social-media-cropper", "repo_url": "https://github.com/simonw/tools", "demo_url": "https://tools.simonwillison.net/social-media-cropper", "source_file_url": "https://github.com/simonw/tools/blob/main/social-media-cropper.html", "provenance_source": "https://gist.github.com/simonw/12b8f88932a71450071190e1289a17e9", "enables_en": "Crop and frame an image for sharing while inspecting an openly documented sequence of AI-generated implementation changes.", "enables_zh": "裁剪和排版用于分享的图片，同时查看公开记录的 AI 生成实现与迭代过程。", "limits_en": "The tool uses third-party browser libraries and may load image-encoder code from a CDN. Export support and speed depend on the browser; no image was uploaded or processed in this review. Repository code is Apache-2.0 licensed; external libraries retain their own terms.", "limits_zh": "工具使用第三方浏览器库，并可能从 CDN 加载图片编码器。导出支持及速度取决于浏览器；本轮未上传或处理图片。 仓库代码采用 Apache 2.0 许可，外部库保留各自条款。", "provenance_note_en": "The creator publishes the initial request and generated HTML, links that transcript from the introducing commit, and publishes later Claude Code editing transcripts; the app also incorporates existing libraries.", "provenance_note_zh": "作者公开了最初的需求和生成的 HTML，在引入该文件的提交中引用对话，并公开后续 Claude Code 修改记录；应用同时使用现有库。", "evidence": [ { "url": "https://gist.github.com/simonw/12b8f88932a71450071190e1289a17e9", "supports": "Maker-owned transcript contains the request for the complete image-cropping app and the assistant's generated HTML implementation." }, { "url": "https://github.com/simonw/tools/commit/b4a2bc98753c3b6592a7cd86a9510bc6bf552efb", "supports": "The commit adding social-media-cropper.html links directly to the initial generation transcript." }, { "url": "https://gist.github.com/simonw/a3e7f3ce34e26ab7c207fe641853e4a0", "supports": "Maker-published Claude Code transcript shows the named tool being read and modified for zoom/background behavior." }, { "url": "https://tools.simonwillison.net/social-media-cropper", "supports": "Live tool documents current crop/export controls and encoder fallbacks; the page returned HTTP 200." }, { "url": "https://github.com/simonw/tools/blob/main/LICENSE", "supports": "Repository code is licensed under Apache 2.0." } ], "platforms": [ "web" ], "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "content_type": "website", "platforms": [ "web" ], "uses": [ "evaluation" ], "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "interaction2code", "name": "Interaction2Code", "description_en": "A benchmark of webpage interaction transitions, pairing before-and-after screenshots with action annotations and public generation and evaluation code.", "description_zh": "面向网页交互状态变化的基准，将交互前后截图与动作标注配对，并公开代码生成和评估实现。", "primary_url": "https://github.com/WebPAI/Interaction2Code", "repo_url": "https://github.com/WebPAI/Interaction2Code", "paper_url": "https://arxiv.org/abs/2411.03292", "enables_en": "Compare generated pages on visual similarity and specified interactions, using the released screenshot pairs, metadata and browser-based metrics.", "enables_zh": "利用公开截图对、元数据和浏览器评估代码，比较生成页面的视觉相似度与指定交互表现。", "limits_en": "Evaluates specified interaction transitions rather than complete site workflows or backend equivalence. Code and data are public, but no project-level code license or dataset license was found in the reviewed repository and data card.", "limits_zh": "评估指定交互的状态变化，不能证明完整网站流程或后端等价。代码与数据可公开查看，但已核查的仓库及数据卡未注明项目级代码许可或数据许可。", "resource_kind": "dataset-and-benchmark", "runtime": "Python generation and metric scripts; browser automation, vision metrics and configured model access. No upstream code was executed.", "platform_note": "Web describes the reconstruction target, not the host operating system.", "evidence": [ { "url": "https://github.com/WebPAI/Interaction2Code#readme", "supports": "Documents paired interaction screenshots, action metadata, generation scripts and automatic/human evaluation; the public tree exposes these artifacts without a project-level license file." }, { "url": "https://github.com/WebPAI/Interaction2Code/blob/main/code/metric/calculate_metric.py", "supports": "Public evaluation source uses Selenium to exercise generated interaction elements and computes visual/text metrics." }, { "url": "https://huggingface.co/datasets/whale99/Interaction2Code", "supports": "Author-linked dataset publicly exposes page-state images and action.json annotations; its card does not specify a dataset license." }, { "url": "https://arxiv.org/abs/2411.03292", "supports": "Current arXiv version describes the interaction-prototyping benchmark and its scope." } ] }, { "content_type": "website", "platforms": [ "web" ], "uses": [ "evaluation" ], "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "iwr-bench", "name": "IWR-Bench", "description_en": "A video-to-webpage reconstruction benchmark with user-interaction recordings, static assets, action sequences and public code that evaluates generated pages.", "description_zh": "从交互视频重建网页的基准，提供用户操作录像、静态资源、动作序列，以及评估生成页面的公开代码。", "primary_url": "https://github.com/SIGMME/IWR-Bench", "repo_url": "https://github.com/SIGMME/IWR-Bench", "paper_url": "https://arxiv.org/abs/2509.24709", "enables_en": "Evaluate whether generated HTML reproduces the visible states and interactions demonstrated in a supplied video.", "enables_zh": "评估生成的 HTML 能否复现给定视频展示的可见状态与交互。", "limits_en": "Uses predefined trajectories and model-assisted judging, not proof of complete site or backend equivalence. The public dataset is marked Apache 2.0; no project-level license was found for the benchmark code. Requires browser and model setup.", "limits_zh": "采用预设轨迹和模型辅助评审，不能证明完整网站或后端等价。公开数据集标注 Apache 2.0；未找到基准代码的项目级许可。运行需配置浏览器与模型。", "resource_kind": "dataset-and-benchmark", "runtime": "Python generation/evaluation scripts with Playwright and the bundled browser-use integration; models and data require configuration.", "platform_note": "Web is the generated artifact target; the benchmark is not a hosted browser-task service.", "evidence": [ { "url": "https://github.com/SIGMME/IWR-Bench#readme", "supports": "Documents video/assets inputs, public generation and evaluation scripts, browser setup and the agent-as-judge procedure." }, { "url": "https://github.com/SIGMME/IWR-Bench/blob/main/evaluate.py", "supports": "Evaluation source loads generated index.html, executes recorded actions and invokes visual/assertion scoring." }, { "url": "https://huggingface.co/datasets/IWR-Bench/IWR-Bench", "supports": "Public ungated dataset provides data.jsonl, videos, assets and checkpoint images, with Apache-2.0 metadata." }, { "url": "https://arxiv.org/abs/2509.24709", "supports": "Current arXiv record defines video-based reconstruction and supplies the bibliography used here." } ] }, { "content_type": "website", "platforms": [ "web" ], "uses": [ "evaluation" ], "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "dashboard2code", "name": "Dashboard2Code (DashboardMimic)", "description_en": "A reconstruction benchmark with public code for agents to explore Plotly+Dash dashboards and reproduce their visual design and interactive behavior; its DashboardMimic data is gated.", "description_zh": "公开代码的仪表盘重建基准，要求 Agent 探索 Plotly+Dash 仪表盘并复现外观和交互；配套 DashboardMimic 数据需要申请访问。", "primary_url": "https://github.com/nth2000/Dashboard2Code", "repo_url": "https://github.com/nth2000/Dashboard2Code", "paper_url": "https://arxiv.org/abs/2607.04727", "enables_en": "Inspect the exploration, generation and evaluation implementation, then reproduce experiments when the required dataset and model integrations are available.", "enables_zh": "查看探索、代码生成与评估实现，并在获得所需数据和模型集成后复现实验。", "limits_en": "Dataset access requires shared contact details and manual approval; gated contents were not accessed. Model adapters and evaluation scripts need configuration, including an empty batch-run list. No project-level code or publicly visible dataset license was found.", "limits_zh": "数据访问要求共享联系信息并经人工批准，本次未访问受限内容。需配置模型适配器与评估脚本，包括默认空的批处理运行列表。未找到项目级代码许可或公开可见的数据许可。", "resource_kind": "benchmark-suite", "runtime": "Python dashboard server and agent/evaluation scripts; Plotly+Dash tasks; gated dataset and configured model adapters required.", "platform_note": "Web dashboards are the evaluation target; no host OS support is inferred.", "evidence": [ { "url": "https://github.com/nth2000/Dashboard2Code#readme", "supports": "Public repository contains exploration, generation and evaluation code and documents custom model-call adapters and data setup." }, { "url": "https://github.com/nth2000/Dashboard2Code/blob/main/evaluation/run_evals.py", "supports": "The shipped batch driver has an empty configuration list and requires editing to select runs and evaluation stages." }, { "url": "https://huggingface.co/datasets/nth2000/Dashboard2Code", "supports": "Public access notice requires agreement to share contact information; no public dataset card or license is displayed." }, { "url": "https://huggingface.co/api/datasets/nth2000/Dashboard2Code", "supports": "Public Hub metadata marks dataset access as manually gated." }, { "url": "https://arxiv.org/abs/2607.04727", "supports": "Paper identifies DashboardMimic as the Plotly+Dash benchmark and describes code-semantic and dynamic interaction evaluation." } ] }, { "content_type": "website", "platforms": [ "web" ], "uses": [ "evaluation" ], "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "webuibench", "name": "WebUIBench", "description_en": "A public diagnostic dataset for webpage perception, HTML code tasks and image–HTML understanding, with eight released evaluation subsets.", "description_zh": "公开的网页能力诊断数据集，覆盖页面感知、HTML 代码任务和图像与 HTML 对应理解，目前发布八个评测子集。", "primary_url": "https://huggingface.co/datasets/Tele-AI-MAIL/WebUIBench", "repo_url": null, "paper_url": "https://arxiv.org/abs/2506.07818", "enables_en": "Measure component skills behind screenshot-to-code systems, such as OCR, visual grounding, code editing and matching page images to HTML.", "enables_zh": "评估截图到代码系统所需的基础能力，包括 OCR、视觉定位、代码编辑和页面图像与 HTML 匹配。", "limits_en": "The public repository has README/images but lacks its advertised evaluator scripts. The Hub currently exposes eight diagnostic subsets, not a complete runnable nine-task release. Data is CC BY 4.0; the data card's arXiv link is incorrect, so use the verified paper link here.", "limits_zh": "公开仓库仅有 README 和图片，缺少说明中列出的评估脚本。Hub 当前提供八个诊断子集，并非可直接运行的完整九任务版本。数据采用 CC BY 4.0；数据卡中的 arXiv 链接有误，应使用此处已核实的论文链接。", "resource_kind": "dataset-and-benchmark", "runtime": "Public Parquet test datasets; an evaluation implementation must be supplied separately.", "platform_note": "Web is the dataset's source/target domain, not a live agent environment.", "evidence": [ { "url": "https://huggingface.co/datasets/Tele-AI-MAIL/WebUIBench", "supports": "Official public dataset provides eight test subsets and declares CC BY 4.0; its paper link incorrectly points to VisualWebBench." }, { "url": "https://github.com/MAIL-Tele-AI/WebUIBench", "supports": "Author repository links the correct paper and dataset, but its current file tree contains only README and images." }, { "url": "https://arxiv.org/abs/2506.07818", "supports": "Verified WebUIBench arXiv record, distinct from VisualWebBench at 2404.05955." }, { "url": "https://aclanthology.org/2025.findings-acl.815/", "supports": "Publisher record confirms the WebUIBench title and authors." } ] }, { "id": "rhounpacker", "name": "RhoUnpacker", "creator": "w1426421352", "description_en": "A Windows command-line utility for extracting KartRider .rho and .rho5 asset archives, with a published standalone executable.", "description_zh": "用于提取 KartRider .rho 和 .rho5 素材归档的 Windows 命令行工具，提供独立可执行文件。", "platforms": [ "windows" ], "implementation": "ai-assisted", "primary_url": "https://github.com/w1426421352/RhoUnpacker", "repo_url": "https://github.com/w1426421352/RhoUnpacker", "download_url": "https://github.com/w1426421352/RhoUnpacker/releases/tag/V1.0.0", "provenance_source": "https://github.com/w1426421352/RhoUnpacker#readme", "enables_en": "Extract supported game-asset archives through a scriptable Windows command-line interface.", "enables_zh": "通过便于脚本调用的 Windows 命令行界面提取受支持的游戏素材归档。", "limits_en": "The AI-generated part is the command-line layer; the core KartLibrary unpacking library is human-authored. The documented Native AOT target is Windows x64, and the project is GPLv3.", "limits_zh": "AI 生成的是命令行层，核心解包库 KartLibrary 由人工编写。文档中的 Native AOT 目标为 Windows x64，项目采用 GPLv3。", "provenance_note_en": "The maintainer explicitly attributes the CLI, README and release notes to Google Gemini and explicitly excludes the reused human-authored KartLibrary from that claim.", "provenance_note_zh": "维护者明确将 CLI、README 和发布说明归于 Google Gemini，并明确说明复用的人工编写 KartLibrary 不属于 AI 生成部分。", "license_en": "GPL-3.0, including the obligations inherited from the reused unpacking-library code.", "license_zh": "GPL-3.0，包含复用解包库代码带来的许可要求。", "license_url": "https://github.com/w1426421352/RhoUnpacker/blob/main/LICENSE", "evidence": [ { "url": "https://github.com/w1426421352/RhoUnpacker#readme", "supports": "Maintainer disclosure identifies the Gemini-generated CLI and distinguishes the human-authored KartLibrary; README documents win-x64 and GPLv3." }, { "url": "https://github.com/w1426421352/RhoUnpacker/releases/tag/V1.0.0", "supports": "Published assets include Windows x64 Native AOT and x86 self-contained ZIP packages; none was downloaded or run." } ], "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "kunkel321-sticky-notes", "name": "Sticky Notes (kunkel321)", "creator": "kunkel321", "description_en": "A portable Windows sticky-note application with checkboxes, reminders and notes that can follow a selected application window.", "description_zh": "便携式 Windows 桌面便签应用，支持复选框、提醒，以及跟随指定应用窗口显示的便签。", "platforms": [ "windows" ], "implementation": "ai-assisted", "primary_url": "https://github.com/kunkel321/Stickies", "repo_url": "https://github.com/kunkel321/Stickies", "download_url": null, "provenance_source": "https://github.com/kunkel321/Stickies#development-note", "enables_en": "Keep desktop notes and alarms, including notes associated with a particular window.", "enables_zh": "在桌面保存便签和提醒，并将特定便签关联到应用窗口。", "limits_en": "This is an AutoHotkey v2 script, paired with a renamed AutoHotkey executable; both files must remain together. No repository license file was identified, so source visibility is not labeled an open-source license.", "limits_zh": "这是 AutoHotkey v2 脚本，配套 EXE 是重命名的 AutoHotkey 运行程序，二者需放在一起。未找到仓库许可文件，因此不将源码可见等同于开源许可。", "provenance_note_en": "The maintainer says Claude generated most of the initial structure and functionality, and that later versions added human-written enhancements and fixes.", "provenance_note_zh": "维护者说明 Claude 生成了大部分初始结构和功能，后续版本加入了人工编写的增强功能和修复。", "license_en": "Not specified in the reviewed repository; bundled runtime and application-script licensing should be checked separately.", "license_zh": "已核查仓库未明确指定许可；配套运行程序与应用脚本的许可应分别确认。", "license_url": null, "runtime_note_en": "Runs as a native Windows AutoHotkey script; the repository's EXE is the interpreter, not a compilation of StickyNotes.ahk.", "runtime_note_zh": "作为原生 Windows AutoHotkey 脚本运行；仓库 EXE 是解释器，并非 StickyNotes.ahk 的编译产物。", "evidence": [ { "url": "https://github.com/kunkel321/Stickies#development-note", "supports": "Maintainer describes substantial Claude-generated base code and later manual code contributions." }, { "url": "https://github.com/kunkel321/Stickies#readme", "supports": "Documents Windows operation, note features, portable packaging, and the exact relationship between the AHK script and renamed interpreter." }, { "url": "https://github.com/kunkel321/Stickies/blob/main/StickyNotes.ahk", "supports": "Public runnable application script is present in the repository." }, { "url": "https://github.com/kunkel321/Stickies/blob/main/StickyNotes.exe", "supports": "Public runtime artifact is present; no binary was downloaded or executed." } ], "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "lazywords", "name": "LazyWords", "creator": "DemonazGH", "description_en": "A Windows vocabulary tool that displays word-and-definition cards as a floating overlay while other applications remain in use.", "description_zh": "在使用其他应用时，通过悬浮卡片展示单词及释义的 Windows 词汇工具。", "platforms": [ "windows" ], "implementation": "ai-assisted", "primary_url": "https://github.com/DemonazGH/LazyWords-Tauri", "repo_url": "https://github.com/DemonazGH/LazyWords-Tauri", "download_url": "https://github.com/DemonazGH/LazyWords-Tauri/releases/tag/v1.0.1", "provenance_source": "https://www.reddit.com/r/OnlyAICoding/comments/1tnouuj/i_built_a_desktop_app_almost_entirely_with_claude/", "enables_en": "Review vocabulary or imported flashcards through periodic desktop overlays and keyboard shortcuts.", "enables_zh": "通过定时桌面浮层和快捷键复习词汇或导入的卡片内容。", "limits_en": "Windows is the documented supported platform; macOS/Linux are planned. The README declares MIT for code and CC BY 4.0 for dictionary data, but its linked LICENSE file returned 404 during review.", "limits_zh": "文档目前支持 Windows，macOS/Linux 尚属计划。README 声明代码为 MIT、词库数据为 CC BY 4.0，但其 LICENSE 文件链接在核查时返回 404。", "provenance_note_en": "The creator's post links this exact repository and describes Claude drafting feature specifications, Claude Code implementing them, and the creator testing and iterating.", "provenance_note_zh": "创作者帖子链接到这一仓库，并说明由 Claude 起草功能规格、Claude Code 实现，再由作者测试和迭代。", "license_en": "MIT claimed in README; standalone LICENSE file unresolved. NGSL-derived dictionary data is declared CC BY 4.0.", "license_zh": "README 声明 MIT，但独立 LICENSE 文件尚未核实；NGSL 衍生词库数据声明为 CC BY 4.0。", "license_url": "https://github.com/DemonazGH/LazyWords-Tauri#license", "evidence": [ { "url": "https://www.reddit.com/r/OnlyAICoding/comments/1tnouuj/i_built_a_desktop_app_almost_entirely_with_claude/", "supports": "Creator explicitly describes a specification-to-implementation Claude workflow and links DemonazGH/LazyWords-Tauri." }, { "url": "https://github.com/DemonazGH/LazyWords-Tauri#readme", "supports": "Documents Windows-only support, overlay behavior, custom dictionary import and source build instructions." }, { "url": "https://github.com/DemonazGH/LazyWords-Tauri/releases/tag/v1.0.1", "supports": "Release listing contains x64 Windows EXE and MSI installers; no installer was run." }, { "url": "https://github.com/DemonazGH/LazyWords-Tauri#license", "supports": "README states MIT for application code and CC BY 4.0 for dictionary data; the separately linked LICENSE path could not be retrieved." } ], "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "claude-code", "name": "Claude Code", "creator": "Anthropic", "description_en": "A coding-agent application that works in a terminal to inspect repositories, edit files and run development tasks, with documented native Windows support.", "description_zh": "在终端中检查仓库、编辑文件和执行开发任务的编程 Agent 应用，官方文档明确支持原生 Windows。", "platforms": [ "windows", "macos", "linux" ], "implementation": "ai-assisted", "primary_url": "https://github.com/anthropics/claude-code", "repo_url": null, "project_repository_url": "https://github.com/anthropics/claude-code", "download_url": "https://code.claude.com/docs/en/setup", "provenance_source": "https://x.com/bcherny/status/2004897269674639461", "enables_en": "Delegate repository changes and development workflows from a supported terminal environment.", "enables_zh": "在受支持的终端环境中委派仓库修改和开发流程。", "limits_en": "Requires supported account or API access and an internet connection. Official documentation distinguishes native Windows from WSL; command sandboxing is not supported on native Windows. The public repository is not the full CLI source.", "limits_zh": "需要受支持的账户或 API 访问及联网。官方文档区分原生 Windows 与 WSL；原生 Windows 不支持命令沙盒。公开仓库并非完整 CLI 源码。", "provenance_note_en": "On 2025-12-27, maintainer Boris Cherny said all of his own Claude Code contributions in the preceding 30 days were written by Claude Code. This documents one maintainer's scoped contributions; it does not establish a product-wide authorship percentage.", "provenance_note_zh": "2025 年 12 月 27 日，维护者 Boris Cherny 表示自己此前 30 天对 Claude Code 的贡献全部由 Claude Code 编写。这仅证明该维护者在特定时期内的贡献范围，不代表整个产品的代码比例。", "license_en": "Proprietary; the repository notice reserves rights to Anthropic and subjects use to its Commercial Terms of Service.", "license_zh": "专有软件；仓库许可说明保留 Anthropic 权利，并规定使用受其商业服务条款约束。", "license_url": "https://github.com/anthropics/claude-code/blob/main/LICENSE.md", "runtime_note_en": "Official setup documents PowerShell/CMD installation on Windows 10 1809+ or Windows Server 2019+ separately from installation inside WSL.", "runtime_note_zh": "官方安装文档单独列出 Windows 10 1809+/Windows Server 2019+ 的 PowerShell/CMD 安装方式，与 WSL 内安装分别说明。", "evidence": [ { "url": "https://x.com/bcherny/status/2004897269674639461", "supports": "Maintainer's dated first-party statement is explicitly limited to his preceding 30 days of contributions." }, { "url": "https://publish.twitter.com/oembed?url=https%3A%2F%2Ftwitter.com%2Fbcherny%2Fstatus%2F2004897269674639461", "supports": "X's public official oEmbed response returned the original author, dated post and scoped contribution statement when direct page retrieval was blocked." }, { "url": "https://code.claude.com/docs/en/setup", "supports": "Official documentation provides native Windows PowerShell/CMD installation, distinguishes WSL, and lists platform/account requirements and sandbox limitations." }, { "url": "https://github.com/anthropics/claude-code/blob/main/LICENSE.md", "supports": "First-party license notice states all rights reserved and references Anthropic's Commercial Terms." } ], "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "shelly-android", "name": "Shelly", "creator": "RYO ITABASHI", "description_en": "An Android terminal and development workspace combining an on-device shell with editor, browser and AI interaction panes.", "description_zh": "将本机 Shell、编辑器、浏览器和 AI 交互面板结合的 Android 终端开发工作区。", "platforms": [ "android" ], "implementation": "ai-built", "primary_url": "https://github.com/RYOITABASHI/Shelly", "repo_url": "https://github.com/RYOITABASHI/Shelly", "download_url": "https://github.com/RYOITABASHI/Shelly/releases/tag/android-latest", "provenance_source": "https://github.com/RYOITABASHI/Shelly#the-story", "enables_en": "Use a terminal and development tools on an Android device, alongside local or configured remote AI providers.", "enables_zh": "在 Android 设备上使用终端与开发工具，并连接本地或自行配置的远程 AI 提供方。", "limits_en": "The maintainer calls it prerelease software and limits unattended-cycle verification to one Galaxy Z Fold6 observation. The rolling android-latest release is the documented APK source; its reviewed manifest identifies version 8.1.1. Repository code is GPLv3.", "limits_zh": "维护者将其列为预发布软件，后台自主执行循环仅记录过一次 Galaxy Z Fold6 验证。文档指定 android-latest 滚动发布为 APK 来源；本次核查的清单版本为 8.1.1。仓库代码采用 GPLv3。", "provenance_note_en": "The maintainer explicitly says every repository line was AI-generated under his direction, with human architectural decisions, review and on-device testing.", "provenance_note_zh": "维护者明确表示仓库代码由 AI 在其指导下生成，架构决策、审查及实机测试由本人负责。", "license_en": "GPL-3.0 as declared by the repository; bundled runtimes and optional components retain their respective licenses.", "license_zh": "仓库声明采用 GPL-3.0；配套运行时及可选组件保留各自许可。", "license_url": "https://github.com/RYOITABASHI/Shelly/blob/main/LICENSE", "evidence": [ { "url": "https://github.com/RYOITABASHI/Shelly#the-story", "supports": "Maintainer explicitly states AI-generated repository code and identifies his architectural and testing role." }, { "url": "https://github.com/RYOITABASHI/Shelly#readme", "supports": "Documents the native Android terminal, panes, APK distribution, provider configuration and validation limits." }, { "url": "https://github.com/RYOITABASHI/Shelly/releases/tag/android-latest", "supports": "The documented rolling release contains APK assets; its latest.json manifest identifies versionName 8.1.1 and versionCode 2612 at review." }, { "url": "https://github.com/RYOITABASHI/Shelly/blob/main/LICENSE", "supports": "Repository contains the GNU GPL version 3 license." }, { "url": "https://github.com/RYOITABASHI/Shelly/releases/download/android-latest/latest.json", "supports": "The rolling release manifest identifies the current APK version and asset location." } ], "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "pourcraft-coffee", "name": "PourCraft Coffee", "creator": "Vinny Carpenter", "description_en": "A native iPhone and iPad app that calculates coffee-to-water amounts and presents a step-by-step pour-over brewing guide.", "description_zh": "计算咖啡与水用量、提供分步手冲流程的原生 iPhone/iPad 应用。", "platforms": [ "ios" ], "implementation": "ai-assisted", "primary_url": "https://apps.apple.com/us/app/pourcraft-coffee/id6759871953", "repo_url": "https://github.com/vscarpenter/PourCraft", "download_url": "https://apps.apple.com/us/app/pourcraft-coffee/id6759871953", "provenance_source": "https://vinny.dev/blog/2026-03-01-vibe-coding-with-coffee/", "enables_en": "Calculate water amounts from a coffee dose and follow the app's selected brewing steps.", "enables_zh": "根据咖啡粉量计算用水量，并跟随应用选定的冲煮步骤。", "limits_en": "Requires iOS/iPadOS 17+. Calculations follow the recipe rules built into the app; the catalog has not independently tested brewing outcomes or the application.", "limits_zh": "需要 iOS/iPadOS 17 及以上版本。计算遵循应用内置配方规则；目录未独立测试冲煮结果或应用功能。", "provenance_note_en": "The creator describes supplying architecture, design and a detailed specification, then having Claude Code turn it into the working SwiftUI application.", "provenance_note_zh": "作者说明自己提供架构、设计及详细需求规格，再由 Claude Code 将其实现为可运行的 SwiftUI 应用。", "license_en": "MIT for the published source repository.", "license_zh": "公开源码仓库采用 MIT 许可。", "license_url": "https://github.com/vscarpenter/PourCraft/blob/main/LICENSE", "evidence": [ { "url": "https://vinny.dev/blog/2026-03-01-vibe-coding-with-coffee/", "supports": "First-person build account identifies substantial Claude Code implementation while retaining human architecture and product decisions." }, { "url": "https://github.com/vscarpenter/PourCraft#readme", "supports": "Documents native iOS implementation, features, iOS 17 requirement and source build steps." }, { "url": "https://apps.apple.com/us/app/pourcraft-coffee/id6759871953", "supports": "Public App Store listing identifies Vinny Carpenter and an installable iPhone/iPad app." }, { "url": "https://github.com/vscarpenter/PourCraft/blob/main/LICENSE", "supports": "Repository provides an MIT license file." } ], "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "anchor-bet-blocker", "name": "Anchor - Bet Blocker", "creator": "Anchor / Branko Popovic", "description_en": "An iPhone and iPad app using Apple's Screen Time controls to restrict selected gambling apps and websites, with check-ins and progress tracking.", "description_zh": "使用 Apple 屏幕使用时间功能限制选定博彩应用和网站，并提供签到与进度记录的 iPhone 与 iPad 应用。", "platforms": [ "ios" ], "implementation": "ai-assisted", "primary_url": "https://heyanchor.app/", "repo_url": null, "download_url": "https://apps.apple.com/us/app/anchor-bet-blocker/id6760262304", "provenance_source": "https://heyanchor.app/blog/built-ios-app-7-days-claude-code.html", "enables_en": "Configure device-level access restrictions and keep a personal record of check-ins.", "enables_zh": "配置设备层面的访问限制，并保存个人签到记录。", "limits_en": "Requires iOS/iPadOS 18+ and the necessary Screen Time authorization. The store lists app/website blocking and contact features under a paid Premium subscription; no public source repository was identified.", "limits_zh": "需要 iOS/iPadOS 18 及以上版本及相应屏幕使用时间授权。商店将应用/网站限制和联系人功能列入付费 Premium 订阅；未找到公开源码仓库。", "provenance_note_en": "The developer's own build account describes Claude Code generating the SwiftUI structure and implementing the blocking integrations, then refactoring storage and authentication under human guidance.", "provenance_note_zh": "开发者在自己的开发记录中说明，Claude Code 生成 SwiftUI 结构、实现应用限制接口，并在人类指导下重构存储和认证。", "license_en": "Commercial App Store application with subscription features; no open-source license was identified.", "license_zh": "带订阅功能的商业 App Store 应用，未找到开源许可。", "license_url": null, "evidence": [ { "url": "https://heyanchor.app/blog/built-ios-app-7-days-claude-code.html", "supports": "First-party account gives concrete AI-generated architecture/integration work and human decisions, and links the exact App Store app." }, { "url": "https://apps.apple.com/us/app/anchor-bet-blocker/id6760262304", "supports": "Public store listing identifies seller Branko Popovic, iOS 18 minimum and Premium subscription feature gates." }, { "url": "https://itunes.apple.com/lookup?id=6760262304&country=us", "supports": "Apple's public lookup confirms the exact app ID, title, seller and minimum OS." } ], "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "gaia", "name": "GAIA", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "description_en": "A multimodal assistant benchmark whose tasks combine reasoning, web research, file handling and tool use, released as questions with supporting attachments.", "description_zh": "多模态助手基准，任务结合推理、网页检索、文件处理和工具使用，以问题及配套附件的形式发布。", "primary_url": "https://huggingface.co/datasets/gaia-benchmark/GAIA", "repo_url": null, "paper_url": "https://arxiv.org/abs/2311.12983", "enables_en": "Evaluate a tool-equipped assistant on tasks that require finding and combining information across software tools and media.", "enables_zh": "评估配备工具的助手能否跨软件工具和媒体查找、整合信息。", "limits_en": "Provides task data, not an execution environment. Access requires contact sharing and agreement to restricted redistribution; test answers are hidden. No general dataset license is stated in the public card. Gated files were not accessed.", "limits_zh": "提供任务数据，并非执行环境。访问要求共享联系信息并同意限制再分发；测试集答案不公开。公开数据卡未注明通用数据许可，本次未访问受限文件。", "resource_kind": "dataset-and-benchmark", "runtime": "Parquet task metadata and attachments; bring a browser, file tools and an agent. Dataset download is gated.", "platform_note": "Tool and media tasks do not establish support for a particular operating-system target.", "evidence": [ { "url": "https://huggingface.co/datasets/gaia-benchmark/GAIA", "supports": "Official card documents tool-oriented tasks, file attachments, hidden test answers, contact-sharing gate and redistribution restrictions." }, { "url": "https://arxiv.org/abs/2311.12983", "supports": "Original paper explicitly includes multimodal understanding, web browsing and tool use rather than only closed-book questions." } ] }, { "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "gaia2", "name": "Gaia2 (ARE)", "content_type": "app", "platforms": [], "uses": [ "training", "evaluation" ], "description_en": "An asynchronous benchmark on Meta's Agents Research Environments, where agents use simulated apps while events, user requests and time constraints change the shared state.", "description_zh": "基于 Meta Agents Research Environments 的异步基准，Agent 操作模拟应用，同时应对事件、用户请求和时间约束带来的状态变化。", "primary_url": "https://github.com/facebookresearch/meta-agents-research-environments", "repo_url": "https://github.com/facebookresearch/meta-agents-research-environments", "paper_url": "https://arxiv.org/abs/2602.11964", "related_paper_urls": [ "https://arxiv.org/abs/2509.17158" ], "enables_en": "Evaluate adaptation and coordination in changing app workflows, or use verifiable write-action outcomes for reinforcement learning.", "enables_zh": "在不断变化的应用流程中评估适应与协作能力，或利用可验证的写操作结果进行强化学习。", "limits_en": "A simulated app/API world, not native phone software. Core ARE requires Python 3.10+ and model configuration. Pin scenario and runner versions, including CLI or multilingual variants. Code is MIT; the public Gaia2 dataset is CC BY 4.0.", "limits_zh": "这是模拟应用/API 世界，并非原生手机软件。ARE 核心要求 Python 3.10+ 和模型配置。应固定场景与运行器版本，包括 CLI 或多语言变体。代码采用 MIT，公开 Gaia2 数据采用 CC BY 4.0。", "resource_kind": "environment-and-benchmark", "runtime": "Python ARE package and scenario CLI; optional GUI and container-based Gaia2-CLI have separate setup.", "platform_note": "Optional web exploration UI is not treated as a native app target platform.", "evidence": [ { "url": "https://github.com/facebookresearch/meta-agents-research-environments#readme", "supports": "Documents simulated apps, events, scenarios, the Gaia2 benchmark, current CLI/multilingual extensions and MIT licensing." }, { "url": "https://github.com/facebookresearch/meta-agents-research-environments/blob/main/pyproject.toml", "supports": "Current package requires Python 3.10 or later." }, { "url": "https://huggingface.co/datasets/meta-agents-research-environments/gaia2", "supports": "Official ungated scenario dataset declares CC BY 4.0." }, { "url": "https://arxiv.org/abs/2602.11964", "supports": "Gaia2 paper describes asynchronous scenarios and write-action verifiers suitable for evaluation and RL." } ] }, { "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "assistantbench", "name": "AssistantBench", "content_type": "website", "platforms": [ "web" ], "uses": [ "evaluation" ], "description_en": "A live-web research benchmark with realistic information-gathering tasks, public development answers and a SeePlanAct browser-agent implementation.", "description_zh": "面向在线网页研究的基准，包含实际信息收集任务、公开开发集答案和 SeePlanAct 浏览器 Agent 实现。", "primary_url": "https://assistantbench.github.io/", "repo_url": "https://github.com/oriyor/assistantbench", "paper_url": "https://arxiv.org/abs/2407.15711", "enables_en": "Test whether an agent can plan a web investigation, gather evidence across sites and return a supported answer.", "enables_zh": "测试 Agent 能否规划网页调查、跨站点收集证据并返回有依据的答案。", "limits_en": "Live websites can change, and test answers are withheld for submission-based evaluation. It is not a resettable website replica. Dataset licensing is Apache 2.0; SeePlanAct code uses OpenRAIL-S with use restrictions. Browser and model access are required.", "limits_zh": "在线网站可能变化，测试答案保留用于提交式评测；它不是可重置的网站副本。数据采用 Apache 2.0，SeePlanAct 代码采用带使用限制的 OpenRAIL-S。运行需要浏览器与模型访问。", "resource_kind": "dataset-and-benchmark", "runtime": "Public task JSONL and Python 3.10/Playwright SeePlanAct implementation; provider configuration required.", "evidence": [ { "url": "https://assistantbench.github.io/", "supports": "Author project defines realistic web tasks and distinguishes public development answers from withheld test answers." }, { "url": "https://github.com/oriyor/assistantbench/tree/main/SeePlanAct", "supports": "Public code and setup instructions are present despite the older root README release wording." }, { "url": "https://github.com/oriyor/assistantbench/blob/main/SeePlanAct/src/seeplanact.py", "supports": "Implementation uses Playwright and model-backed planning, memory and action grounding." }, { "url": "https://huggingface.co/datasets/AssistantBench/AssistantBench", "supports": "Official task dataset is public and declares Apache 2.0." }, { "url": "https://github.com/oriyor/assistantbench/blob/main/LICENSE", "supports": "Repository publishes the AI PUBS OpenRAIL-S license." } ] }, { "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "agentboard", "name": "AgentBoard", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "description_en": "A multi-environment evaluation suite that tracks intermediate progress as well as final success across web browsing, tool operation, games and simulated-world tasks.", "description_zh": "跨环境评测套件，在网页浏览、工具操作、游戏和模拟世界任务中，同时跟踪中间进展与最终成功情况。", "primary_url": "https://github.com/hkust-nlp/AgentBoard", "repo_url": "https://github.com/hkust-nlp/AgentBoard", "paper_url": "https://arxiv.org/abs/2401.13178", "enables_en": "Inspect where an agent makes or loses progress using shared evaluation and analysis across task families.", "enables_zh": "通过跨任务类型的统一评测和分析，检查 Agent 在哪些环节取得或失去进展。", "limits_en": "Reuses environments such as WebArena and WebShop rather than supplying entirely new worlds. Setup is task-specific and some tools need live service credentials. The documented runner pins Python 3.8.13. Code is Apache 2.0; the dataset is GPL 2.0.", "limits_zh": "复用 WebArena、WebShop 等环境，并非全部新建的任务世界。各任务有独立配置，部分工具需要在线服务凭据；文档固定 Python 3.8.13。代码采用 Apache 2.0，数据采用 GPL 2.0。", "resource_kind": "benchmark-suite", "runtime": "Python evaluation toolkit with per-environment services and optional analysis dashboards.", "platform_note": "A mixed-domain evaluation layer; host operating systems are not application-platform tags.", "evidence": [ { "url": "https://github.com/hkust-nlp/AgentBoard#readme", "supports": "Documents analytical progress metrics, heterogeneous environments, per-task setup and separate code/data licenses." }, { "url": "https://huggingface.co/datasets/hkust-nlp/agentboard", "supports": "Official data lists nine task families and GPL-2.0 licensing; the full archive contains more than the preview." }, { "url": "https://arxiv.org/abs/2401.13178", "supports": "Paper introduces fine-grained progress-rate analysis for multi-turn agents." } ] }, { "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "agentdojo", "name": "AgentDojo", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "description_en": "A tool-use evaluation environment with simulated applications and untrusted tool data, designed to measure task utility and resistance to prompt injection.", "description_zh": "包含模拟应用和不可信工具数据的工具使用评测环境，用于衡量任务完成能力及对提示注入的抵抗能力。", "primary_url": "https://github.com/ethz-spylab/agentdojo", "repo_url": "https://github.com/ethz-spylab/agentdojo", "paper_url": "https://arxiv.org/abs/2406.13352", "enables_en": "Compare agent defenses while checking whether legitimate tool-based tasks still succeed.", "enables_zh": "比较 Agent 防御方法，同时检查正常的工具任务是否仍能成功完成。", "limits_en": "A specialized security and utility benchmark using simulated app tools, not native banking or workplace software. Requires Python 3.10+ and model-provider configuration. Package APIs are still evolving. Code is MIT-licensed.", "limits_zh": "这是采用模拟应用工具的专门安全与实用性基准，并非原生银行或办公软件。需要 Python 3.10+ 和模型服务配置，包接口仍在演进。代码采用 MIT。", "resource_kind": "environment-and-benchmark", "runtime": "Python agentdojo package with configurable task suites, agents and defenses.", "platform_note": "Tool APIs represent application workflows; no native desktop or phone platform is asserted.", "evidence": [ { "url": "https://github.com/ethz-spylab/agentdojo#readme", "supports": "Official code documents task suites, benchmarking and the evolving package API." }, { "url": "https://github.com/ethz-spylab/agentdojo/blob/main/pyproject.toml", "supports": "Package requires Python 3.10+ and declares MIT licensing." }, { "url": "https://arxiv.org/abs/2406.13352", "supports": "Original paper distinguishes legitimate task utility from robustness to injected tool data." } ] }, { "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "omniact", "name": "OmniACT", "content_type": "app", "platforms": [ "web", "macos", "windows", "linux" ], "uses": [ "training", "evaluation" ], "description_en": "A screenshot-to-action-program dataset pairing desktop and web task instructions with UI images, target annotations and reference PyAutoGUI scripts.", "description_zh": "截图到动作程序的数据集，将桌面与网页任务指令、界面图像、目标标注和参考 PyAutoGUI 脚本配对。", "primary_url": "https://huggingface.co/datasets/Writer/omniact", "repo_url": "https://huggingface.co/datasets/Writer/omniact/tree/main", "paper_url": "https://arxiv.org/abs/2402.17553", "enables_en": "Train and evaluate visual models that translate a screen and instruction into a mouse-and-keyboard action program.", "enables_zh": "训练和评估将屏幕图像及指令转换为鼠标键盘动作程序的视觉模型。", "limits_en": "Static examples and reference-based action metrics, not a packaged desktop execution environment. Evaluation target boxes must not be used as test-time inputs. The dataset card declares MIT; platform tags identify screenshot sources.", "limits_zh": "提供静态样本和基于参考的动作指标，并非打包好的桌面执行环境。测试时不能将评估用目标框作为输入。数据卡标注 MIT；平台标签表示截图来源。", "resource_kind": "dataset-and-benchmark", "runtime": "Public train/validation/test JSON, screenshot archives and Python evaluation scripts.", "platform_note": "The paper explicitly covers screenshots from macOS, Windows, Linux and web applications.", "evidence": [ { "url": "https://huggingface.co/datasets/Writer/omniact", "supports": "Author dataset defines screenshot/task/script pairs, train/validation/test splits, test-metadata restrictions and MIT licensing." }, { "url": "https://huggingface.co/datasets/Writer/omniact/blob/main/eval.py", "supports": "Public evaluation script computes action/sequence comparison scores." }, { "url": "https://arxiv.org/html/2402.17553", "supports": "Paper specifies screenshot platforms and reference-based action metrics rather than a supplied desktop VM." } ] }, { "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "the-agent-company", "name": "TheAgentCompany", "content_type": "app", "platforms": [ "web" ], "uses": [ "evaluation" ], "description_en": "A self-hosted simulated software company where agents use internal web services, files, code and conversations with simulated coworkers to complete workplace tasks.", "description_zh": "可自行部署的模拟软件公司，Agent 通过内部网页服务、文件、代码及与模拟同事的对话完成工作任务。", "primary_url": "https://github.com/TheAgentCompany/TheAgentCompany", "repo_url": "https://github.com/TheAgentCompany/TheAgentCompany", "paper_url": "https://arxiv.org/abs/2412.14161", "enables_en": "Evaluate multi-tool workplace workflows with task initialization, intermediate checkpoints and outcome grading.", "enables_zh": "通过任务初始化、中间检查点和结果评分，评估涉及多种工具的办公流程。", "limits_en": "Requires Docker/Compose, service data and at least 30 GB free disk space according to setup documentation. Agent and simulated-coworker models need configuration. Initial states and benchmark versions matter. Repository code is MIT; bundled services retain their own terms.", "limits_zh": "按配置文档需要 Docker/Compose、服务数据及至少 30 GB 空闲磁盘；Agent 和模拟同事模型需单独配置。初始状态及基准版本会影响比较。仓库代码采用 MIT，配套服务仍适用各自条款。", "resource_kind": "environment-and-benchmark", "runtime": "Docker-hosted company services plus task-specific workspaces and evaluators.", "platform_note": "Web covers the internal browser applications; code and file tasks also occur, with no host OS inferred.", "evidence": [ { "url": "https://github.com/TheAgentCompany/TheAgentCompany#readme", "supports": "Documents self-contained workplace services, Docker requirements, task initialization, agent/environment model configuration and grading." }, { "url": "https://github.com/TheAgentCompany/TheAgentCompany/blob/main/LICENSE", "supports": "Repository code is MIT-licensed." }, { "url": "https://arxiv.org/abs/2412.14161", "supports": "Original paper defines simulated digital-worker tasks across websites, programs and coworker communication." } ] }, { "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "spa-bench", "name": "SPA-Bench", "content_type": "app", "platforms": [ "android" ], "uses": [ "evaluation" ], "description_en": "An Android-agent evaluation framework with single-app and cross-app tasks in English and Chinese, trajectory annotations and an automated scoring pipeline.", "description_zh": "Android Agent 评测框架，提供中英文单应用及跨应用任务、轨迹标注和自动评分流程。", "primary_url": "https://github.com/ai-agents-2030/SPA-Bench", "repo_url": "https://github.com/ai-agents-2030/SPA-Bench", "paper_url": "https://arxiv.org/abs/2410.15164", "enables_en": "Compare mobile agents across varied apps and measure task completion together with execution and resource-use metrics.", "enables_zh": "跨不同应用比较移动端 Agent，并结合执行过程与资源消耗衡量任务完成情况。", "limits_en": "Requires an Android device/emulator, ADB, app setup and configured model backends. The preconfigured emulator snapshot is still marked coming soon. Evaluation uses model-based judgments and annotated components. Framework code is MIT; individual apps have separate terms.", "limits_zh": "需要 Android 设备或模拟器、ADB、应用安装及模型后端配置；预配置模拟器快照仍标注待发布。评估结合模型评审与标注组件。框架代码采用 MIT，各应用有独立条款。", "resource_kind": "environment-and-benchmark", "runtime": "Python/Conda runner communicating with Android through ADB; selected model integrations may require GPU hosting.", "evidence": [ { "url": "https://github.com/ai-agents-2030/SPA-Bench#readme", "supports": "Official repository exposes task collections, agent integrations and evaluation code while the emulator-snapshot checklist remains incomplete." }, { "url": "https://github.com/ai-agents-2030/SPA-Bench/blob/main/Documentation.md", "supports": "Documents device/emulator setup, ADB, model configuration and model-based evaluation modes." }, { "url": "https://github.com/ai-agents-2030/SPA-Bench/blob/main/LICENSE", "supports": "Framework is MIT-licensed." } ] }, { "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "mobileagentbench", "name": "MobileAgentBench", "content_type": "app", "platforms": [ "android" ], "uses": [ "evaluation" ], "description_en": "A mobile-agent benchmark library with task-specific setup, completion checks and resets for everyday Android apps such as calendar, contacts and file management.", "description_zh": "移动端 Agent 基准库，为日历、联系人和文件管理等日常 Android 应用提供任务专用初始化、完成检查及重置逻辑。", "primary_url": "https://github.com/MobileAgentBench/mobile-agent-bench", "repo_url": "https://github.com/MobileAgentBench/mobile-agent-bench", "paper_url": "https://arxiv.org/abs/2406.08184", "enables_en": "Integrate a mobile agent with programmatic checks instead of manually reviewing every task attempt.", "enables_zh": "将移动端 Agent 接入程序化完成检查，减少逐次人工审阅任务尝试的工作。", "limits_en": "Requires ADB and separately installed SimpleMobileTools apps; task checks depend on the expected app UI/state. The package requires Python 3.8+ and pins AndroidViewClient. No explicit repository license was found in the reviewed source.", "limits_zh": "需要 ADB 及另行安装的 SimpleMobileTools 应用；检查逻辑依赖预期的应用界面和状态。包要求 Python 3.8+，并固定 AndroidViewClient 版本。已核查源码未注明仓库许可。", "resource_kind": "environment-and-benchmark", "runtime": "Python library with AndroidViewClient/ADB and per-app task modules; emulator preferred in upstream guidance.", "evidence": [ { "url": "https://github.com/MobileAgentBench/mobile-agent-bench#readme", "supports": "Documents Android Studio/ADB setup, separately installed benchmark apps and the benchmark library." }, { "url": "https://github.com/MobileAgentBench/mobile-agent-bench/blob/main/src/mobile_agent_benchmark/bench_task.py", "supports": "Public task interface includes setup, teardown and completion checking." }, { "url": "https://github.com/MobileAgentBench/mobile-agent-bench/blob/main/pyproject.toml", "supports": "Package requires Python 3.8+, pins AndroidViewClient and does not declare a license." }, { "url": "https://arxiv.org/abs/2406.08184", "supports": "Paper defines the mobile-app task benchmark." } ] }, { "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "webmall", "name": "WebMall", "content_type": "website", "platforms": [ "web" ], "uses": [ "evaluation" ], "description_en": "A self-hostable shopping benchmark with four different stores, designed for cross-shop product comparison, vague requirements, compatible alternatives and simulated checkout.", "description_zh": "可自行部署的购物基准，包含四个不同商店，覆盖跨店商品比较、模糊需求、兼容替代品和模拟结账。", "primary_url": "https://wbsg-uni-mannheim.github.io/WebMall/", "repo_url": "https://github.com/wbsg-uni-mannheim/WebMall", "paper_url": "https://arxiv.org/abs/2508.13024", "enables_en": "Test whether a web agent can compare heterogeneous offers and carry a shopping request through several stores.", "enables_zh": "测试网页 Agent 能否比较异构商品信息，并跨多个商店完成购物请求。", "limits_en": "Requires Docker/Compose and the project's modified BrowserGym/AgentLab submodules, with Python 3.11 or 3.12. The setup documents at least 4 GB container RAM. No project-level license was found in the reviewed repository; component and product-data terms remain separate.", "limits_zh": "需要 Docker/Compose、项目修改版 BrowserGym/AgentLab 子模块，以及 Python 3.11 或 3.12。配置文档要求容器至少使用 4 GB 内存。已核查仓库未注明项目级许可，组件和商品数据条款应分别确认。", "resource_kind": "environment-and-benchmark", "runtime": "Four WooCommerce/WordPress stores with databases and Elasticsearch, plus BrowserGym/AgentLab evaluation.", "evidence": [ { "url": "https://wbsg-uni-mannheim.github.io/WebMall/", "supports": "Official project defines multi-shop tasks, heterogeneous product data and simulated store workflows." }, { "url": "https://github.com/wbsg-uni-mannheim/WebMall#readme", "supports": "Documents Python requirements, modified submodules and local benchmark scripts." }, { "url": "https://github.com/wbsg-uni-mannheim/WebMall/blob/main/docker_all/README.md", "supports": "Documents Docker hosting for four stores and the container memory requirement." } ] }, { "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "webchorearena", "name": "WebChoreArena", "content_type": "website", "platforms": [ "web" ], "uses": [ "evaluation" ], "description_en": "A distinct task suite built on WebArena's websites, emphasizing tedious workflows that require retaining many details, performing calculations and tracking information across pages.", "description_zh": "基于 WebArena 网站构建的独立任务套件，聚焦需要记忆大量细节、进行计算及跨页面跟踪信息的繁琐流程。", "primary_url": "https://github.com/WebChoreArena/WebChoreArena", "repo_url": "https://github.com/WebChoreArena/WebChoreArena", "paper_url": "https://arxiv.org/abs/2506.01952", "enables_en": "Measure web-agent performance on demanding information-heavy chores using the released task configurations and evaluators.", "enables_zh": "利用公开任务配置和评分器，评估网页 Agent 对高信息量、复杂日常流程的处理能力。", "limits_en": "Adds tasks rather than a new set of websites. Requires separately deployed WebArena services and controlled resets, including before cross-site evaluation. Model/API configuration affects comparisons. Repository code is Apache 2.0; upstream components retain their own terms.", "limits_zh": "新增的是任务，而非另一套网站。需要另外部署 WebArena 服务并控制重置，包括跨站点评测前的重置；模型和 API 配置会影响比较。仓库代码采用 Apache 2.0，上游组件保留各自条款。", "resource_kind": "benchmark-suite", "runtime": "WebArena service environment with the supplied BrowserGym or AgentOccam task integration.", "evidence": [ { "url": "https://github.com/WebChoreArena/WebChoreArena#readme", "supports": "Documents the new task collection, information/memory/calculation demands, WebArena dependency and reset protocol." }, { "url": "https://github.com/WebChoreArena/WebChoreArena/blob/main/LICENSE", "supports": "Repository declares Apache-2.0 licensing." }, { "url": "https://arxiv.org/abs/2506.01952", "supports": "Paper establishes the task suite as a substantial extension on the existing WebArena environments." } ] }, { "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "crmarena", "name": "CRMArena", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "description_en": "A Salesforce-based professional CRM task benchmark with public evaluation code and datasets; its Pro extension adds sales, service, quotation workflows and multi-turn user interactions.", "description_zh": "基于 Salesforce 的专业 CRM 任务基准，公开评测代码和数据；Pro 扩展覆盖销售、服务、报价流程及多轮用户交互。", "primary_url": "https://github.com/SalesforceAIResearch/CRMArena", "repo_url": "https://github.com/SalesforceAIResearch/CRMArena", "paper_url": "https://arxiv.org/abs/2411.02305", "related_paper_urls": [ "https://arxiv.org/abs/2505.18878" ], "enables_en": "Evaluate agents on linked customer and business records, workflow reasoning and interactive professional scenarios.", "enables_zh": "在相互关联的客户及业务记录、流程推理和交互式专业场景中评估 Agent。", "limits_en": "Execution depends on access to Salesforce research orgs and model providers; live access was not tested. GUI access now requires author approval. Original, Pro, B2B/B2C and interactive settings differ. Code and datasets use CC BY-NC 4.0 and are described as research-only.", "limits_zh": "执行依赖 Salesforce 研究组织及模型服务访问，本次未测试在线权限；GUI 访问目前需作者批准。原始、Pro、B2B/B2C 及交互模式有所不同。代码和数据采用 CC BY-NC 4.0，并声明限研究用途。", "resource_kind": "environment-and-benchmark", "runtime": "Python Simple Salesforce integration and evaluation runners; hosted research-org access and provider configuration required.", "platform_note": "The reviewed public execution path is API-based; no native desktop or mobile platform is inferred.", "evidence": [ { "url": "https://github.com/SalesforceAIResearch/CRMArena#readme", "supports": "Documents original and Pro evaluation, hosted research-org dependencies and the change to approval-only GUI access." }, { "url": "https://github.com/SalesforceAIResearch/CRMArena/blob/main/LICENSE.txt", "supports": "Repository license is CC BY-NC 4.0." }, { "url": "https://huggingface.co/datasets/Salesforce/CRMArena", "supports": "Official original dataset publishes tasks/schema under CC BY-NC 4.0." }, { "url": "https://huggingface.co/datasets/Salesforce/CRMArenaPro", "supports": "Official Pro dataset separates B2B/B2C and interactive modes under CC BY-NC 4.0." } ] }, { "content_type": "game", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "gamedevbench", "name": "GameDevBench", "description_en": "An executable benchmark of scoped Godot development tasks involving gameplay logic, interfaces, graphics and animation, with starting projects and engine-level validation.", "description_zh": "由限定范围的 Godot 开发任务组成的可执行基准，覆盖玩法逻辑、界面、图形和动画，并提供初始项目及引擎级验证。", "primary_url": "https://github.com/waynchi/gamedevbench", "repo_url": "https://github.com/waynchi/gamedevbench", "paper_url": "https://arxiv.org/abs/2602.11103", "enables_en": "Evaluate whether a coding agent can modify an existing game project correctly, including tasks that benefit from screenshot or runtime-video feedback.", "enables_zh": "评估编程 Agent 能否正确修改现有游戏项目，包括需要截图或运行视频反馈的任务。", "limits_en": "Tests game development, not an agent's ability to play a game. Official comparisons require exactly Godot 4.4.1 and a pinned task set. Python 3.10+ is required; strict runs document Linux Bubblewrap/Xvfb dependencies. Repository license is Apache 2.0.", "limits_zh": "测试游戏开发，而非 Agent 玩游戏的能力。正式比较要求严格使用 Godot 4.4.1 并固定任务集；需要 Python 3.10+，严格运行模式注明 Linux Bubblewrap/Xvfb 依赖。仓库采用 Apache 2.0。", "resource_kind": "environment-and-benchmark", "runtime": "Godot 4.4.1 plus a Python runner and configured coding agent; optional screenshot/video feedback.", "platform_note": "Godot is the task runtime. Host support and engine export possibilities are not asserted as deployment-platform tags.", "evidence": [ { "url": "https://github.com/waynchi/gamedevbench#readme", "supports": "Documents Godot project-editing tasks, starting/reference artifacts, exact engine version, Python requirement and confinement/display dependencies." }, { "url": "https://github.com/waynchi/gamedevbench/blob/main/LICENSE", "supports": "Repository declares Apache 2.0." }, { "url": "https://arxiv.org/abs/2602.11103", "supports": "Paper distinguishes multimodal game development tasks and visual feedback from game-playing evaluation." } ] }, { "content_type": "game", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "gamecraft-bench", "name": "GameCraft-Bench", "description_en": "A benchmark for generating complete playable 2D Godot projects from specifications, evaluated through launch checks, replayed player inputs and multimodal rubric scoring.", "description_zh": "根据规格生成完整可玩二维 Godot 项目的基准，通过启动检查、玩家输入重放和多模态评分标准进行评估。", "primary_url": "https://github.com/FreedomIntelligence/gamecraft-bench", "repo_url": "https://github.com/FreedomIntelligence/gamecraft-bench", "paper_url": "https://arxiv.org/abs/2606.17861", "enables_en": "Measure whether a coding agent delivers a coherent game artifact whose mechanics and presentation work during observed interaction.", "enables_zh": "衡量编程 Agent 能否交付完整一致的游戏，并在实际观察的交互中实现玩法与表现要求。", "limits_en": "Covers 2D Godot generation, not general 3D/Unity/Unreal development or subjective fun. The documented setup pins Godot 4.6.2, Python 3.12+, Harbor and Linux display/recording tools. Scoring uses a model judge. Code is Apache 2.0; asset licenses are recorded separately.", "limits_zh": "覆盖二维 Godot 游戏生成，并不代表通用三维、Unity/Unreal 开发或主观乐趣评测。文档固定 Godot 4.6.2，要求 Python 3.12+、Harbor 及 Linux 显示和录制工具；评分使用模型评审。代码采用 Apache 2.0，素材许可分别记录。", "resource_kind": "environment-and-benchmark", "runtime": "Harbor with a local-subprocess environment, Godot, Xvfb/xdotool/ffmpeg and configured solver/judge providers.", "platform_note": "The browser dashboard is an inspection interface, not the generated game's platform classification.", "evidence": [ { "url": "https://github.com/FreedomIntelligence/gamecraft-bench#readme", "supports": "Documents complete game submission, replayable traces, judge-based evaluation, pinned Godot setup and explicit 2D scope." }, { "url": "https://github.com/FreedomIntelligence/gamecraft-bench/blob/main/pyproject.toml", "supports": "Package requires Python 3.12+ and declares Apache 2.0." }, { "url": "https://github.com/FreedomIntelligence/gamecraft-bench/blob/main/LICENSE", "supports": "Repository code uses Apache 2.0." }, { "url": "https://arxiv.org/abs/2606.17861", "supports": "Paper defines end-to-end artifact completeness and interaction-grounded evaluation." } ] }, { "content_type": "game", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "v-gamegym", "name": "V-GameGym (SKYLENAGE-GameCodeGym)", "description_en": "A Pygame-generation dataset and evaluation toolkit with natural-language requirements, reference code and model-based scoring of generated code, screenshots and short gameplay recordings.", "description_zh": "Pygame 生成数据集与评估工具，包含自然语言要求、参考代码，并通过模型评估生成代码、截图及短时游戏录像。", "primary_url": "https://github.com/alibaba/SKYLENAGE-GameCodeGym", "repo_url": "https://github.com/alibaba/SKYLENAGE-GameCodeGym", "paper_url": "https://arxiv.org/abs/2509.20136", "enables_en": "Compare code models on generating visual game programs and inspect how code-level and recorded visual quality differ.", "enables_zh": "比较代码模型生成可视化游戏程序的能力，并分析代码质量与录制画面质量的差异。", "limits_en": "Measures generated artifacts with model judges, not long-horizon game-playing policies. README command examples differ from the shipped script arguments, so paths and judge endpoints need configuration. Requires Python 3.10+, Pygame and model access. Code and dataset declare Apache 2.0.", "limits_zh": "使用模型评审生成产物，并非评测长时游戏策略。README 命令示例与已发布脚本参数不同，需要配置路径和评审端点。要求 Python 3.10+、Pygame 和模型访问。代码及数据均声明 Apache 2.0。", "resource_kind": "dataset-and-benchmark", "runtime": "Public JSONL requirements/code dataset plus Python generation, recording and evaluation scripts; text and visual judge endpoints required.", "platform_note": "No operating system is inferred from Pygame; this is not a browser-game benchmark.", "evidence": [ { "url": "https://github.com/alibaba/SKYLENAGE-GameCodeGym#readme", "supports": "Official repository identifies V-GameGym, documents Pygame generation/recording/evaluation and declares Apache 2.0." }, { "url": "https://github.com/alibaba/SKYLENAGE-GameCodeGym/blob/main/game_evaluator.py", "supports": "Public source implements code/screenshot/video model scoring and requires model-name, visual/text endpoints rather than the README's illustrative input/output flags." }, { "url": "https://huggingface.co/datasets/alibabagroup/SKYLENAGE-GameCodeGym", "supports": "Author-linked ungated dataset exposes requirements, reference code and metadata under Apache 2.0." }, { "url": "https://arxiv.org/abs/2509.20136", "supports": "Original paper defines the visual game-generation evaluation setting." } ] }, { "content_type": "app", "platforms": [ "android" ], "uses": [ "evaluation" ], "implementation": "ai-assisted", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "appsim-bench", "name": "AppSim-Bench", "description_en": "A mobile GUI benchmark built from controllable simulated Android apps, with task-relevant interaction flows, seeded backend data and outcome checks.", "description_zh": "基于可控模拟 Android 应用的移动 GUI 基准，提供与任务相关的交互流程、预置后端数据和结果检查。", "primary_url": "https://github.com/Acrab-Agentic-Labs/AppSim", "repo_url": "https://github.com/Acrab-Agentic-Labs/AppSim", "paper_url": "https://arxiv.org/abs/2609.07712", "provenance_source": "https://arxiv.org/html/2609.07712v1", "provenance_note_en": "The authors explicitly describe simulated-app implementation as coding-agent-assisted, with human requirements, inspection and verification. This is not a product-wide autonomous-authorship claim.", "provenance_note_zh": "作者明确说明模拟应用由编程 Agent 辅助实现，人类负责需求、检查和验证；这不代表整个项目由 AI 自主开发。", "enables_en": "Evaluate phone agents on repeatable app workflows without relying on changing commercial-app content and accounts.", "enables_zh": "在可重复的应用流程上评估手机 Agent，减少对不断变化的商业应用内容和账户状态的依赖。", "limits_en": "These are research simulations, not the named commercial services. Build/install the separate app APKs and configure an Android device/emulator; the documented benchmark uses Android 13/API 33 and Python 3.11. Windows/macOS emulator guidance remains a TODO. No project-level framework license was found; app terms must be checked separately.", "limits_zh": "这些是研究模拟应用，不是对应名称的商业服务。需单独构建或安装应用 APK 并配置 Android 设备或模拟器；文档基准使用 Android 13/API 33 和 Python 3.11。Windows/macOS 模拟器说明仍为 TODO。未找到框架项目级许可，各应用条款需分别检查。", "resource_kind": "environment-and-benchmark", "runtime": "Python runner using ADB with separately published Android app projects and controlled task state.", "platform_note": "Android is the task target; Windows/macOS/Linux host setup does not imply tasks for those operating systems.", "evidence": [ { "url": "https://github.com/Acrab-Agentic-Labs/AppSim#readme", "supports": "Public framework documents evaluation, Android 13 emulator setup, Python 3.11 and separately linked app repositories; Windows/macOS emulator instructions are incomplete." }, { "url": "https://github.com/Acrab-Agentic-Labs/AppSim-JD", "supports": "Author-linked app repository exposes an Android Gradle project and evaluation assets." }, { "url": "https://github.com/Acrab-Agentic-Labs/AppSim-Amazon", "supports": "Author-linked simulated app repository exposes Android source and release materials." }, { "url": "https://arxiv.org/html/2609.07712v1", "supports": "Authors describe human-led coding-agent assistance, controlled simulated apps, Android runtime and outcome verification." } ] }, { "content_type": "game", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "gamecode4d", "name": "GameCode4D (Code2Games)", "description_en": "A game-world generation benchmark with public prompts and Blender scene artifacts, accompanied by Code2Games code for scene analysis, asset placement and gameplay-world construction.", "description_zh": "公开提示与 Blender 场景产物的游戏世界生成基准，配有 Code2Games 的场景分析、素材放置和玩法世界构建代码。", "primary_url": "https://huggingface.co/datasets/AIGeeksGroup/GameCode4D", "repo_url": "https://github.com/AIGeeksGroup/Code2Games", "paper_url": "https://arxiv.org/abs/2610.05033", "enables_en": "Inspect generated game-world assets and study how a shared scene representation connects spatial layout with gameplay intentions.", "enables_zh": "查看生成的游戏世界素材，并研究共享场景表示如何连接空间布局与玩法意图。", "limits_en": "The current data card lists 20 prompt variants but exposes 16 .blend files; the paper evaluated 10 prompts. Public code documents Blender outputs, not a complete released UE5 evaluation pipeline. Requires external scenes, animated characters and asset/model services. No code or dataset license was stated in the reviewed sources.", "limits_zh": "当前数据卡列出 20 个提示变体，但公开 16 个 .blend 文件；论文评估的是 10 个提示。公开代码说明 Blender 输出，不能视为完整发布的 UE5 评测流程。需要外部场景、动画角色及素材或模型服务。已核查来源未注明代码或数据许可。", "resource_kind": "dataset-and-benchmark", "runtime": "Python 3.11, Blender 4.2, Code2Worlds scene input, animated FBX character and Hunyuan3D/DashScope components for the documented generation pipeline.", "platform_note": "Blender/Unreal are research tooling; no desktop deployment platform is inferred from them.", "evidence": [ { "url": "https://huggingface.co/datasets/AIGeeksGroup/GameCode4D", "supports": "Public data card lists 20 prompt variants and matching target filenames; inspected repository lists 16 Blender artifacts and no license declaration." }, { "url": "https://github.com/AIGeeksGroup/Code2Games#readme", "supports": "Official code documents external dependencies and generation of gaming_world.blend/gameplay.mp4, not a complete public Unreal evaluation runner." }, { "url": "https://arxiv.org/abs/2610.05033", "supports": "Paper introduces the original ten-prompt GameCode4D evaluation and Unreal adaptation research." } ] }, { "id": "bash-fighter", "name": "Bash Fighter", "creator": "Bash Entertainment", "content_type": "game", "implementation": "ai-assisted", "description_en": "A browser platform fighter with shared arenas, online matches and bots that fill empty player slots.", "description_zh": "提供共享竞技场、在线对战和机器人补位的浏览器平台格斗游戏。", "primary_url": "https://bashfighter.itch.io/bash-fighter", "repo_url": "https://github.com/Bash-Entertainment/bash-fighter", "demo_url": "https://bashfighter.itch.io/bash-fighter", "provenance_source": "https://github.com/Bash-Entertainment/bash-fighter#readme", "enables_en": "Join arena matches or inspect and self-host the published client, simulation and match-server code.", "enables_zh": "参与竞技场对战，或查看并自行部署公开的客户端、模拟核心和对战服务器代码。", "limits_en": "The creator describes an early build with rough edges, and online play depends on a match server. The itch.io page was accessible; the separate bashfighter.com host returned 403 to this research client.", "limits_zh": "作者将其描述为仍有不足的早期版本，在线对战依赖服务器。itch.io 页面可访问，独立站点 bashfighter.com 对本轮研究客户端返回 403。", "provenance_note_en": "The maintainer explicitly says an AI agent performs day-to-day development and wrote most repository code on behalf of a human owner; this is an attributed project statement.", "provenance_note_zh": "维护者明确说明，AI Agent 代表人类所有者承担日常开发并编写了仓库大部分代码；此处保留为项目方声明。", "license_en": "AGPL-3.0; the repository also requires a contributor agreement that permits additional commercial licensing.", "license_zh": "AGPL-3.0；仓库另要求贡献者签署允许额外商业授权的协议。", "license_url": "https://github.com/Bash-Entertainment/bash-fighter/blob/main/LICENSE", "evidence": [ { "url": "https://github.com/Bash-Entertainment/bash-fighter#readme", "supports": "Project's explicit disclosure attributes most code and ongoing development to an AI agent; it provides source layout and local runtime instructions." }, { "url": "https://bashfighter.itch.io/bash-fighter", "supports": "Creator-controlled HTML5 listing repeats the development disclosure, links the repository and provides the browser artifact." }, { "url": "https://github.com/Bash-Entertainment/bash-fighter#license", "supports": "Maintainer documents AGPL-3.0 and contributor-agreement terms." } ], "platforms": [ "web" ], "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "download_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "threejs-quake", "name": "Three.js Quake", "creator": "mrdoob", "content_type": "game", "implementation": "ai-assisted", "description_en": "A Three.js browser port of Quake, with the original game's shareware episode and a public source repository.", "description_zh": "将 Quake 移植到 Three.js 浏览器环境的项目，包含原作共享版章节及公开源码仓库。", "primary_url": "https://github.com/mrdoob/three-quake", "repo_url": "https://github.com/mrdoob/three-quake", "demo_url": "https://mrdoob.github.io/three-quake/", "provenance_source": "https://x.com/mrdoob/status/2015076521531355583", "enables_en": "Try the browser port or study how a classic game engine was adapted to a JavaScript rendering stack.", "enables_zh": "体验浏览器移植版，或研究经典游戏引擎如何适配 JavaScript 渲染环境。", "limits_en": "The AI-development claim concerns the port; id Software created the original game. The repository includes Episode 1 shareware data and requires a registered game's data for full-game content.", "limits_zh": "AI 开发声明针对移植工作，原作由 id Software 创作。仓库包含第一章共享版数据，完整游戏内容需使用正版注册版数据。", "provenance_note_en": "The maintainer's linked development post asks Claude to port Quake to Three.js and shows the resulting implementation; the README credits the port to mrdoob with Claude.", "provenance_note_zh": "维护者链接的开发帖记录了请 Claude 将 Quake 移植到 Three.js 的过程和实现结果，README 也将移植署名为 mrdoob 与 Claude。", "license_en": "GPL-2.0 for code; the original game's shareware/registered asset terms remain separate.", "license_zh": "代码采用 GPL-2.0，原游戏共享版及注册版素材的条款另行适用。", "license_url": "https://github.com/mrdoob/three-quake/blob/main/LICENSE", "evidence": [ { "url": "https://x.com/mrdoob/status/2015076521531355583", "supports": "First-party development post documents asking Claude to implement the Three.js port." }, { "url": "https://publish.twitter.com/oembed?url=https%3A%2F%2Fx.com%2Fmrdoob%2Fstatus%2F2015076521531355583", "supports": "X's official public oEmbed response returned the maintainer's original post and date." }, { "url": "https://github.com/mrdoob/three-quake#readme", "supports": "README cross-links the exact dev log and live artifact, credits id Software, and distinguishes code licensing from game data." }, { "url": "https://mrdoob.github.io/three-quake/", "supports": "Public browser artifact returned HTTP 200; no game session was started." } ], "platforms": [ "web" ], "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "download_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "space-defenders", "name": "Space Defenders", "creator": "Jason Leow", "content_type": "game", "implementation": "ai-assisted", "description_en": "A single-player browser arcade game in which a ship moves horizontally and shoots descending aliens.", "description_zh": "玩家控制飞船左右移动、射击下落外星敌人的单人浏览器街机游戏。", "primary_url": "https://jasonleow.com/space-defenders/", "repo_url": null, "demo_url": "https://jasonleow.com/space-defenders/", "provenance_source": "https://www.linkedin.com/posts/jasonleowsg_did-claude-sonnet-37-just-one-shot-a-space-activity-7300882292527222785-xYub", "enables_en": "Play a compact arcade game and inspect the creator's published prompt for the initial browser implementation.", "enables_zh": "体验小型街机游戏，并查看作者公开的初始浏览器实现提示词。", "limits_en": "Keyboard controls are documented. The creator's generation claim covers the initial single-file client; later score-service code was not traced, and no reuse license was identified.", "limits_zh": "页面提供键盘操作说明。作者的生成声明针对初始单文件客户端，后续计分服务代码来源未追溯，也未找到复用许可。", "provenance_note_en": "Leow reports that Claude Sonnet 3.7 generated the initial HTML/CSS/JavaScript game from one prompt, publishes that prompt and links the exact playable page; the broader current project is labeled AI-assisted conservatively.", "provenance_note_zh": "Leow 表示 Claude Sonnet 3.7 通过一个提示生成了初始 HTML/CSS/JavaScript 游戏，并公开提示及准确演示链接；对当前整体项目采用较保守的 AI 辅助开发标签。", "license_en": "No open-source code license identified in the reviewed maker and project pages.", "license_zh": "已核查的作者及项目页面中未找到开源代码许可。", "license_url": null, "evidence": [ { "url": "https://www.linkedin.com/posts/jasonleowsg_did-claude-sonnet-37-just-one-shot-a-space-activity-7300882292527222785-xYub", "supports": "Maker explicitly reports model generation of the initial single-file game, provides the prompt and links jasonleow.com/space-defenders." }, { "url": "https://jasonleow.com/space-defenders/", "supports": "Creator-hosted game page identifies Jason Leow and documents movement, shooting and pause controls; no gameplay was performed." } ], "platforms": [ "web" ], "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "download_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "firewood-splitting-simulator", "name": "Firewood Splitting Simulator", "creator": "shapiro500", "content_type": "game", "implementation": "ai-assisted", "description_en": "A browser-based 3D wood-splitting toy built around scanned wood, an axe and a stump, with interactive splitting and a rotating view.", "description_zh": "围绕木材、斧头及树桩扫描素材构建的三维浏览器劈柴互动作品，支持劈切和视角旋转。", "primary_url": "https://screen.toys/firewood/", "repo_url": null, "demo_url": "https://screen.toys/firewood/", "provenance_source": "https://www.reddit.com/r/vibecoding/comments/1uckug1/heres_how_i_made_the_firewood_splitting_simulator/", "enables_en": "Explore a tactile browser interaction assembled from real-world scans and AI-assisted implementation.", "enables_zh": "体验将真实扫描素材与 AI 辅助实现结合的浏览器交互作品。", "limits_en": "This entry covers the browser version. The creator supplied and edited physical scans, animation and sound; no public source repository or code-reuse license was identified.", "limits_zh": "本条目仅覆盖浏览器版本。实体扫描、动画及声音由作者采集和编辑，未找到公开源码仓库或代码复用许可。", "provenance_note_en": "The creator details using Claude in Antigravity to implement the game from a long specification and repeated refinements, while personally producing scans, edited assets and recorded sound.", "provenance_note_zh": "作者详细说明，在 Antigravity 中使用 Claude 根据长篇需求和多轮修改实现游戏，同时亲自制作扫描素材、编辑资产并录制声音。", "license_en": "No open-source code or asset-reuse license identified in the reviewed creator material.", "license_zh": "已核查的作者资料中未找到开源代码或素材复用许可。", "license_url": null, "evidence": [ { "url": "https://www.reddit.com/r/vibecoding/comments/1uckug1/heres_how_i_made_the_firewood_splitting_simulator/", "supports": "First-person behind-the-scenes post describes the Claude implementation process, human asset pipeline and exact browser URL." }, { "url": "https://www.reddit.com/r/vibecoding/comments/1u2yuus/vibecoded_a_firewood_splitting_simulator_using_my/", "supports": "Earlier maker post describes specifying game rules and iterating on generated versions alongside original scans and animation." }, { "url": "https://screen.toys/firewood/", "supports": "Public creator-hosted artifact documents drag-to-rotate and click-to-split interaction; no gameplay occurred." } ], "platforms": [ "web" ], "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "download_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "legends-of-future-past", "name": "Legends of Future Past — 2026 reconstruction", "creator": "Jon Radoff", "content_type": "game", "implementation": "ai-assisted", "description_en": "A browser-accessible reconstruction of a 1990s multiplayer text adventure, running preserved world scripts on a newly implemented engine.", "description_zh": "通过浏览器访问的 1990 年代多人文字冒险重建版，使用新实现的引擎运行保留下来的世界脚本。", "primary_url": "https://github.com/jonradoff/lofp", "repo_url": "https://github.com/jonradoff/lofp", "demo_url": "https://lofp.metavert.io/", "provenance_source": "https://github.com/jonradoff/lofp#how-we-brought-it-back", "enables_en": "Explore the restored text world or inspect how legacy scripts, documentation and session logs informed a new server implementation.", "enables_zh": "探索重建的文字世界，或查看旧脚本、文档及会话记录如何用于实现新的服务器。", "limits_en": "The original world content is human-authored. Self-hosting requires the documented Go/Node/MongoDB stack and login configuration; this review did not authenticate or verify fidelity to the original.", "limits_zh": "原始世界内容由人工创作。自行部署需要文档中的 Go/Node/MongoDB 环境和登录配置；本轮未登录，也未验证与原作的一致程度。", "provenance_note_en": "Original creator Jon Radoff explains how he used Claude Code to reconstruct the script parser, mechanics and server from preserved materials; this claim concerns the 2026 reconstruction.", "provenance_note_zh": "原作者 Jon Radoff 说明如何借助 Claude Code，根据保留的资料重建脚本解析器、机制及服务器；该声明仅针对 2026 年的重建工作。", "license_en": "MIT for the released 2026 repository; the README preserves credits and copyright attribution for the original game and world.", "license_zh": "发布的 2026 年仓库采用 MIT；README 保留原游戏及世界内容的署名与版权信息。", "license_url": "https://github.com/jonradoff/lofp/blob/main/LICENSE", "evidence": [ { "url": "https://github.com/jonradoff/lofp#how-we-brought-it-back", "supports": "Original creator provides a detailed, component-specific account of Claude-assisted reconstruction and distinguishes preserved human game scripts from the new engine." }, { "url": "https://github.com/jonradoff/lofp#running-your-own-server", "supports": "Repository documents the executable server/client stack and authentication setup." }, { "url": "https://github.com/jonradoff/lofp/blob/main/LICENSE", "supports": "Published repository provides an MIT license." }, { "url": "https://lofp.metavert.io/", "supports": "The creator-linked browser entry point returned HTTP 200; no login or game session was attempted." } ], "platforms": [ "web" ], "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "download_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "neon-zenith", "name": "Neon Zenith", "creator": "Bill Chan", "content_type": "game", "implementation": "ai-assisted", "description_en": "A browser hover-car exploration game set in a dense neon city, with street-level driving, flight and time trials.", "description_zh": "在密集霓虹城市中探索的浏览器悬浮车游戏，包含街道驾驶、飞行及计时挑战。", "primary_url": "https://github.com/billpwchan/neon-zenith", "repo_url": "https://github.com/billpwchan/neon-zenith", "demo_url": "https://zenith.billpwchan.art/", "provenance_source": "https://github.com/billpwchan/neon-zenith#how-it-was-made", "enables_en": "Explore the published game source, shaders and asset pipeline, or run the documented browser build.", "enables_zh": "查看游戏源码、着色器和素材处理流程，或运行文档中的浏览器构建版本。", "limits_en": "The project uses a substantial asset bundle and WebGPU/WebGL2 rendering. Its live host returned 403 to this research client; source and local build instructions are public. Third-party assets have separate licenses.", "limits_zh": "项目包含较大的素材包，使用 WebGPU/WebGL2 渲染。演示站点对本轮研究客户端返回 403，但源码及本地构建说明公开；第三方素材采用独立许可。", "provenance_note_en": "The maintainer states that Claude Code wrote the application code, TSL shaders and Blender pipelines under his direction and review; reused models, textures and fonts are separately credited.", "provenance_note_zh": "维护者明确说明 Claude Code 在其指导与审查下编写应用代码、TSL 着色器和 Blender 流程；复用的模型、纹理和字体另有署名。", "license_en": "MIT covers source code; credited models use CC BY 4.0, specified textures use CC0, and fonts retain SIL OFL licenses.", "license_zh": "源码采用 MIT；已署名模型使用 CC BY 4.0，指定纹理使用 CC0，字体保留 SIL OFL 许可。", "license_url": "https://github.com/billpwchan/neon-zenith/blob/main/LICENSE", "evidence": [ { "url": "https://github.com/billpwchan/neon-zenith#how-it-was-made", "supports": "Maintainer explicitly identifies AI-authored code, shaders and Blender pipelines and his own direction/review role." }, { "url": "https://github.com/billpwchan/neon-zenith#run-it-locally", "supports": "Repository contains the application, processed assets and documented Node/Vite build path." }, { "url": "https://github.com/billpwchan/neon-zenith/blob/main/LICENSE", "supports": "License expressly limits MIT coverage to source code." }, { "url": "https://github.com/billpwchan/neon-zenith/blob/main/CREDITS.md", "supports": "Credits list separate third-party model, texture and font licenses." }, { "url": "https://zenith.billpwchan.art/", "supports": "Author-linked demo URL returned HTTP 403 in this research environment; browser usability remains for separate observation." } ], "platforms": [ "web" ], "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "download_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "pyramid-wars-demo", "name": "Pyramid Wars — browser demo", "creator": "Durian Arcade", "content_type": "game", "implementation": "ai-assisted", "description_en": "A browser real-time strategy demo combining base construction, unit control, abilities and collectible army upgrades.", "description_zh": "结合基地建设、单位控制、技能及军队升级收集的浏览器即时战略演示版。", "primary_url": "https://durian-arcade.itch.io/pyramid-wars", "repo_url": null, "demo_url": "https://durian-arcade.itch.io/pyramid-wars", "provenance_source": "https://durian-arcade.itch.io/pyramid-wars", "enables_en": "Try the creator's browser strategy prototype and inspect the published disclosure of AI involvement in code and media.", "enables_zh": "体验作者的浏览器策略原型，并查看其对代码和媒体中 AI 参与情况的公开说明。", "limits_en": "The itch.io artifact is a demo/prototype. The separate full Steam game was still listed as upcoming during review; no public source repository or open-source reuse license was identified.", "limits_zh": "itch.io 作品为演示/原型版本。核查时，独立的 Steam 完整版仍列为即将推出；未找到公开源码仓库或开源复用许可。", "provenance_note_en": "The solo developer identifies Claude as the tool used to implement the HTML/JavaScript game and create its coded sounds and graphics; the publisher's Steam disclosure also explicitly lists code among AI-assisted components.", "provenance_note_zh": "独立开发者说明使用 Claude 实现 HTML/JavaScript 游戏及代码中的声音、图形；发行方的 Steam 声明也明确将代码列入 AI 辅助组件。", "license_en": "Creator-hosted demo; no open-source code license identified.", "license_zh": "作者托管的演示版，未找到开源代码许可。", "license_url": null, "evidence": [ { "url": "https://durian-arcade.itch.io/pyramid-wars", "supports": "Creator describes the Claude-based implementation, names the HTML/JavaScript codebase and provides the playable HTML5 demo." }, { "url": "https://store.steampowered.com/app/4996430/Pyramid_Wars/", "supports": "Publisher's AI disclosure explicitly covers code and media; listing distinguishes the upcoming full release from available demos." } ], "platforms": [ "web" ], "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "download_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "formula-minus-one", "name": "Formula Minus One — browser beta", "creator": "Moisés Cabello", "content_type": "game", "implementation": "ai-assisted", "description_en": "A futuristic browser racing game with hovercraft, boost and air-brake controls, time trials and selectable race modes.", "description_zh": "提供悬浮赛车、加速和空气刹车控制、计时挑战及多种比赛模式的未来风格浏览器竞速游戏。", "primary_url": "https://fm1.app/", "repo_url": null, "demo_url": "https://fm1.app/", "provenance_source": "https://www.reddit.com/r/ClaudeCode/comments/1x4ajpq/formula_minus_one_open_beta_6_months_of_work_huge/", "enables_en": "Try the open browser beta and examine an independent creator's workflow combining AI-generated code with human-led game and asset design.", "enables_zh": "体验公开浏览器 Beta，并了解独立作者如何结合 AI 生成代码和人工主导的游戏及素材设计。", "limits_en": "This is the browser beta, with landscape orientation requested on phones. The creator lists a WebGPU-based stack; compatibility and performance were not tested. No open-source code license was identified.", "limits_zh": "本条目为浏览器 Beta，手机页面要求横屏。作者注明使用 WebGPU 技术栈；本轮未测试兼容性或性能，也未找到开源代码许可。", "provenance_note_en": "The maker's build account attributes code and tooling to OpenAI/Anthropic and open-source models, while describing human design work, sketches, Blender cleanup and music sketches.", "provenance_note_zh": "作者开发记录将代码及工具实现归于 OpenAI、Anthropic 和开源模型，并说明了人工设计、草图、Blender 修整及音乐草稿等工作。", "license_en": "Creator-hosted browser beta; no open-source code license identified. The in-game credits retain separate third-party asset attributions.", "license_zh": "作者托管的浏览器 Beta，未找到开源代码许可；游戏署名保留第三方素材的独立归属。", "license_url": null, "evidence": [ { "url": "https://www.reddit.com/r/ClaudeCode/comments/1x4ajpq/formula_minus_one_open_beta_6_months_of_work_huge/", "supports": "First-person build account specifically attributes code/tooling to LLMs and distinguishes human and AI contributions to assets; it links the exact beta URL." }, { "url": "https://fm1.app/", "supports": "Public beta identifies Moisés Cabello, documents controls and modes, and includes asset credits; the page returned HTTP 200 without gameplay." }, { "url": "https://fm1.moises.cloud/", "supports": "The former creator URL explicitly redirects visitors to fm1.app, supporting continuity of the project." } ], "platforms": [ "web" ], "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "download_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "gymnasium", "name": "Gymnasium", "description_en": "A common interface and reference environments for single-agent reinforcement learning, continuing the maintenance lineage of OpenAI Gym.", "description_zh": "用于单 Agent 强化学习的统一接口与参考环境集合，延续 OpenAI Gym 的维护体系。", "content_type": "game", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://gymnasium.farama.org/", "repo_url": "https://github.com/Farama-Foundation/Gymnasium", "demo_url": null, "paper_url": "https://arxiv.org/abs/2407.17032", "limits_en": "The base package does not install every environment family. Current main requires Python 3.11+; extras, simulators and game assets have separate requirements. Pin environment version suffixes for comparisons. Framework code is MIT; external environments and ROMs may have different terms.", "limits_zh": "基础包不会安装所有环境系列。当前主分支要求 Python 3.11+，扩展、模拟器与游戏素材另有依赖。比较实验应固定环境版本后缀。框架代码采用 MIT，外部环境和 ROM 可能适用不同条款。", "enables_en": "Train and compare policies through consistent reset/step, observation, reward and termination interfaces, or build custom environments.", "enables_zh": "通过统一的重置、动作、观察、奖励和终止接口训练及比较策略，也可创建自定义环境。", "resource_kind": "environment-framework", "runtime": "Python gymnasium package with selected environment extras; this is an API/framework rather than one standalone game.", "evidence": [ { "url": "https://github.com/Farama-Foundation/Gymnasium#readme", "supports": "Defines the maintained Gym fork, reference environment families, installation extras and versioning policy." }, { "url": "https://github.com/Farama-Foundation/Gymnasium/blob/main/pyproject.toml", "supports": "Current main package metadata requires Python 3.11+." }, { "url": "https://github.com/Farama-Foundation/Gymnasium/blob/main/LICENSE", "supports": "Framework code is MIT licensed." }, { "url": "https://arxiv.org/abs/2407.17032", "supports": "Primary paper describes the environment API, reference tasks and reproducibility tools." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "No OS tag is inferred for this general environment API; selected environments have their own runtime requirements." }, { "id": "pettingzoo", "name": "PettingZoo", "description_en": "A multi-agent environment library with turn-by-turn Agent Environment Cycle and parallel-action APIs, covering cooperative and competitive games.", "description_zh": "提供逐 Agent 轮转的 Agent Environment Cycle 接口和并行动作接口的多 Agent 环境库，覆盖合作与竞争游戏。", "content_type": "game", "platforms": [ "linux", "macos" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://pettingzoo.farama.org/", "repo_url": "https://github.com/Farama-Foundation/PettingZoo", "demo_url": null, "paper_url": "https://arxiv.org/abs/2009.14471", "limits_en": "Each environment family needs its own extras. Current main requires Python 3.11–3.14; Windows is not officially supported in the README. MAgent and MPE now live in separate MAgent2/MPE2 projects. Core code is MIT; emulator/game dependencies may carry additional terms.", "limits_zh": "各环境系列需要单独的扩展依赖。当前主分支要求 Python 3.11–3.14，README 未正式支持 Windows。MAgent 和 MPE 已迁至独立的 MAgent2/MPE2 项目。核心代码采用 MIT，模拟器与游戏依赖可能另有条款。", "enables_en": "Train agents that cooperate or compete using sequential or simultaneous action semantics appropriate to the game.", "enables_zh": "根据游戏选择顺序或同时行动语义，训练合作或竞争的 Agent。", "resource_kind": "environment-framework", "runtime": "Python pettingzoo package with per-family extras; AEC API and a Parallel API for compatible environments.", "evidence": [ { "url": "https://github.com/Farama-Foundation/PettingZoo#readme", "supports": "Documents APIs, environment families, installation and Linux/macOS support." }, { "url": "https://github.com/Farama-Foundation/PettingZoo/blob/main/pyproject.toml", "supports": "Current main declares Python >=3.11,<3.15." }, { "url": "https://pettingzoo.farama.org/environments/mpe/", "supports": "Official migration guide explains the separate MPE2 package." }, { "url": "https://github.com/Farama-Foundation/PettingZoo/blob/main/LICENSE", "supports": "Core repository is MIT licensed." }, { "url": "https://arxiv.org/abs/2009.14471", "supports": "Original paper introduces PettingZoo and the AEC model." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "vizdoom", "name": "ViZDoom", "description_en": "A configurable first-person game environment exposing Doom-engine observations and controls for visual reinforcement learning and agent evaluation.", "description_zh": "可配置的第一人称游戏环境，为视觉强化学习和 Agent 评测提供 Doom 引擎的观察与控制接口。", "content_type": "game", "platforms": [ "linux", "macos", "windows" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://vizdoom.farama.org/", "repo_url": "https://github.com/Farama-Foundation/ViZDoom", "demo_url": null, "paper_url": "https://arxiv.org/abs/1605.02097", "limits_en": "Freedoom assets are supplied; original Doom/Doom II WADs are not bundled. ViZDoom-original code is MIT, while engine components have separate terms. Current wheels require Python 3.10+; macOS 1.3+ wheels target Apple Silicon, and Windows is documented as less tested.", "limits_zh": "项目提供 Freedoom 素材，不附带原版 Doom/Doom II WAD。ViZDoom 自身代码采用 MIT，引擎组件另有条款。当前 wheel 要求 Python 3.10+；1.3+ 的 macOS wheel 面向 Apple Silicon，文档说明 Windows 的测试程度较低。", "enables_en": "Create navigation, control or multiplayer scenarios and train agents from screen observations with configurable rewards and game state.", "enables_zh": "创建导航、控制或多人场景，利用屏幕观察及可配置奖励、游戏状态训练 Agent。", "resource_kind": "environment-and-benchmark", "runtime": "Python/C++ API with Gymnasium wrappers; pip installation where compatible wheels exist, otherwise a native source build.", "evidence": [ { "url": "https://github.com/Farama-Foundation/ViZDoom#readme", "supports": "Documents scenarios, APIs, platform/wheel constraints, Freedoom defaults and mixed engine licensing." }, { "url": "https://arxiv.org/abs/1605.02097", "supports": "Original platform paper describes first-person visual RL and custom scenarios." }, { "url": "https://arxiv.org/abs/1809.03470", "supports": "Competition paper describes visual multiplayer evaluation and the platform." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "miniworld", "name": "MiniWorld", "description_en": "A minimal 3D indoor world simulator for navigation and goal-directed learning, with configurable rooms, objects and visual observations.", "description_zh": "用于导航与目标导向学习的简化三维室内模拟器，支持配置房间、物体和视觉观察。", "content_type": "game", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://miniworld.farama.org/", "repo_url": "https://github.com/Farama-Foundation/Miniworld", "demo_url": null, "paper_url": "https://arxiv.org/abs/2306.13831", "limits_en": "Deprecated on 2025-08-11 with no further updates/support planned. Source remains inspectable, but setup is historical: setup.py requires Python 3.8+ and pyglet<2. The license file is Apache 2.0 while README still says MIT. Graphics and physics are intentionally simple.", "limits_zh": "项目于 2025-08-11 宣布弃用，不再计划更新或提供支持。源码仍可查阅，但配置属于历史方案：setup.py 要求 Python 3.8+、pyglet<2。许可文件为 Apache 2.0，README 仍称 MIT。图形和物理模型刻意简化。", "enables_en": "Inspect or reproduce historical 3D navigation experiments and modify compact indoor layouts for controlled research tasks.", "enables_zh": "研究或复现历史三维导航实验，并修改小型室内布局以构建可控研究任务。", "resource_kind": "environment-framework", "runtime": "Python/Gymnasium with pyglet/OpenGL; published source/pip recipe, manual-control script and optional EGL/Xvfb offscreen rendering.", "evidence": [ { "url": "https://github.com/Farama-Foundation/Miniworld/blob/master/README.md", "supports": "Describes 3D worlds and rendering; dates deprecation to 2025-08-11 and retains an MIT statement." }, { "url": "https://github.com/Farama-Foundation/Miniworld/blob/master/setup.py", "supports": "Declares Python >=3.8, pyglet>=1.5.27,<2.0 and an Apache package license." }, { "url": "https://github.com/Farama-Foundation/Miniworld/blob/master/license", "supports": "The actual lowercase license file contains Apache License 2.0." }, { "url": "https://arxiv.org/abs/2306.13831", "supports": "Shared MiniGrid/MiniWorld paper describes configurable 2D/3D tasks." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "No current supported OS matrix is established; no OS tag is inferred from Python/OpenGL." }, { "id": "procgen2", "name": "Procgen2", "description_en": "A community rewrite of Procgen with C++ game implementations and a CEnv interface for using compiled worlds from Python.", "description_zh": "Procgen 的社区重写项目，包含 C++ 游戏实现，以及从 Python 调用编译后世界的 CEnv 接口。", "content_type": "game", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/Farama-Foundation/Procgen2", "repo_url": "https://github.com/Farama-Foundation/Procgen2", "demo_url": null, "paper_url": null, "limits_en": "A separate development implementation, not proven benchmark-equivalent to original Procgen. Source builds require CMake and SDL3/SDL3_image; examples use locally built libraries. A supported release/platform matrix and parity across the original game suite were not established. Code is MIT; asset terms need separate review.", "limits_zh": "这是独立的开发实现，尚不能认定与原版 Procgen 的基准行为等价。源码构建需要 CMake、SDL3/SDL3_image，示例使用本地编译库。本次未确认正式版本的平台矩阵或对原游戏集的完整等价性。代码采用 MIT，素材条款需单独核查。", "enables_en": "Inspect and extend the rewritten games, or connect compiled environments to Python through reset, step and rendering interfaces.", "enables_zh": "研究和扩展重写的游戏，或通过重置、动作和渲染接口将编译环境连接到 Python。", "resource_kind": "environment-framework", "runtime": "C++/CMake source build, SDL3/SDL3_image, CEnv shared-library interface and Python/Gymnasium example viewers.", "evidence": [ { "url": "https://github.com/Farama-Foundation/Procgen2#readme", "supports": "Explicitly describes a rewrite and distinguishes its earlier maintained-fork plan." }, { "url": "https://github.com/Farama-Foundation/Procgen2/blob/main/DEV_GUIDE.md", "supports": "Documents compiling CoinRun with CMake/SDL3 and extending game implementation." }, { "url": "https://github.com/Farama-Foundation/Procgen2/blob/main/CMakeLists.txt", "supports": "Build targets include CoinRun, Jumper, Chaser, CaveFlyer, Bossfight, Maze and Climber." }, { "url": "https://github.com/Farama-Foundation/Procgen2/blob/main/cenv/USAGE_GUIDE.md", "supports": "Documents compiled shared-library APIs and Python interaction." }, { "url": "https://github.com/Farama-Foundation/Procgen2/blob/main/interactive_viewer.py", "supports": "Viewer source loads locally compiled CaveFlyer; it was not executed." }, { "url": "https://github.com/Farama-Foundation/Procgen2/blob/main/LICENSE", "supports": "Repository code is MIT licensed." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "The ABI guide discusses Linux/macOS/Windows library formats; that alone is not a verified supported-platform matrix." }, { "id": "stable-retro", "name": "Stable-Retro", "description_en": "A maintained Gym Retro fork connecting classic-game emulators to Gymnasium, with integrations defining rewards, state variables and episode endings.", "description_zh": "持续维护的 Gym Retro 分支，将经典游戏模拟器接入 Gymnasium，并通过集成配置定义奖励、状态变量和回合终止条件。", "content_type": "game", "platforms": [ "linux", "macos" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://stable-retro.farama.org/", "repo_url": "https://github.com/Farama-Foundation/Stable-Retro", "demo_url": null, "paper_url": null, "limits_en": "Most integrations need separately supplied matching ROMs or BIOS files; only limited homebrew/test examples are included. Python 3.11–3.14 is documented; Windows uses WSL2 and core support varies. Wrapper code is MIT, but emulator cores include GPL and non-commercial terms. This is not a blanket game redistribution license.", "limits_zh": "多数集成需要另行提供匹配的 ROM 或 BIOS，仅附带有限的自制或测试示例。文档列出 Python 3.11–3.14；Windows 使用 WSL2，不同核心支持范围不同。封装代码采用 MIT，但核心包含 GPL 和非商业条款，不构成统一的游戏再分发许可。", "enables_en": "Train agents on emulator-backed games and create integrations translating game memory into rewards and success conditions.", "enables_zh": "在模拟器游戏上训练 Agent，并创建将游戏内存状态转为奖励及成功条件的集成。", "resource_kind": "environment-framework", "runtime": "Python stable-retro package, emulator cores and per-game integration files; native dependencies vary by core.", "evidence": [ { "url": "https://github.com/Farama-Foundation/Stable-Retro#readme", "supports": "Documents Gym Retro lineage, Python/platform requirements, Gymnasium integration and external ROM requirements." }, { "url": "https://github.com/Farama-Foundation/Stable-Retro/blob/main/docs/included_roms.md", "supports": "Lists limited included homebrew/test ROMs; no game files were downloaded." }, { "url": "https://github.com/Farama-Foundation/Stable-Retro/blob/main/LICENSE", "supports": "Wrapper repository publishes MIT terms." }, { "url": "https://github.com/Farama-Foundation/Stable-Retro/blob/main/LICENSES.md", "supports": "Lists separate emulator-core licenses, including GPLv2 and non-commercial terms." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "Windows is documented through WSL2; Linux/macOS tags do not imply a verified native Windows build." }, { "id": "magent2", "name": "MAgent2", "description_en": "A grid-world engine and reference scenarios for many interacting agents, including battle and pursuit settings exposed through PettingZoo APIs.", "description_zh": "面向大量交互 Agent 的网格世界引擎与参考场景，包含战斗、追逐等任务，并提供 PettingZoo 接口。", "content_type": "game", "platforms": [ "linux", "macos" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://magent2.farama.org/", "repo_url": "https://github.com/Farama-Foundation/MAgent2", "demo_url": null, "paper_url": "https://arxiv.org/abs/1712.00600", "limits_en": "A maintained fork of MAgent whose environments were split out of PettingZoo. Current main requires Python 3.11+, despite the older 3.8+ README statement. Pin versions and scenarios; the original paper does not validate current-fork results. Repository code is MIT.", "limits_zh": "这是 MAgent 的维护分支，相关环境已从 PettingZoo 拆出。当前主分支要求 Python 3.11+，与 README 较旧的 3.8+ 说明不同。应固定版本与场景；原论文不能证明当前分支的实验结果。仓库代码采用 MIT。", "enables_en": "Study collective behavior, coordination and competition among many agents in configurable grid-world scenarios.", "enables_zh": "在可配置的网格世界场景中研究大量 Agent 的集体行为、协调和竞争。", "resource_kind": "environment-framework", "runtime": "Python magent2 package with native engine, NumPy, pygame-ce and PettingZoo; package metadata specifies the current Python floor.", "evidence": [ { "url": "https://github.com/Farama-Foundation/MAgent2#readme", "supports": "Describes grid-world scenarios, MAgent lineage, PettingZoo split and Linux/macOS support." }, { "url": "https://github.com/Farama-Foundation/MAgent2/blob/main/pyproject.toml", "supports": "Current main requires Python >=3.11, differing from the older README." }, { "url": "https://github.com/Farama-Foundation/MAgent2/blob/main/LICENSE", "supports": "Repository code is MIT licensed." }, { "url": "https://arxiv.org/abs/1712.00600", "supports": "Original MAgent paper supplies research lineage, not a new paper for MAgent2." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "mpe2", "name": "MPE2", "description_en": "Particle-world environments for communication, cooperation and competition, continuing the MPE environments formerly distributed with PettingZoo.", "description_zh": "研究交流、合作和竞争的粒子世界环境，延续此前随 PettingZoo 分发的 MPE 环境。", "content_type": "game", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://mpe2.farama.org/", "repo_url": "https://github.com/Farama-Foundation/MPE2", "demo_url": null, "paper_url": "https://arxiv.org/abs/1703.04908", "limits_en": "Use the separate mpe2 package; pettingzoo.mpe now raises an import error. Current main requires Python 3.11+. Discrete actions are the default, unlike original OpenAI MPE, and newer scenarios extend the old collection. Code is MIT; historical papers describe the original environments.", "limits_zh": "应使用独立的 mpe2 包，pettingzoo.mpe 现会触发导入错误。当前主分支要求 Python 3.11+。默认采用离散动作，与原 OpenAI MPE 不同，并新增了场景。代码采用 MIT；历史论文描述的是原始环境。", "enables_en": "Evaluate agents that move and exchange messages while sharing goals or competing over landmarks and objects.", "enables_zh": "评估在共享目标或争夺地标、物体时进行移动和消息交流的 Agent。", "resource_kind": "environment-framework", "runtime": "Python mpe2 with PettingZoo AEC/Parallel APIs, pygame-ce and SciPy; installed separately from PettingZoo.", "evidence": [ { "url": "https://github.com/Farama-Foundation/MPE2#readme", "supports": "Defines particle-world behavior, OpenAI lineage, discrete default and added scenarios." }, { "url": "https://pettingzoo.farama.org/environments/mpe/", "supports": "Official migration guide confirms the standalone package and preserved AEC/Parallel APIs." }, { "url": "https://github.com/Farama-Foundation/MPE2/blob/main/pyproject.toml", "supports": "Current main requires Python >=3.11 and lists dependencies." }, { "url": "https://github.com/Farama-Foundation/MPE2/blob/main/LICENSE", "supports": "Repository code is MIT licensed." }, { "url": "https://arxiv.org/abs/1703.04908", "supports": "Original environment study cited by MPE2." }, { "url": "https://arxiv.org/abs/1706.02275", "supports": "Study accompanying the original public MPE release." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "No current OS support matrix is stated in the reviewed sources; Python/pygame alone does not establish platform support." }, { "id": "gvgai", "name": "GVGAI", "description_en": "A Video Game Description Language framework for evaluating planners and generating game levels or rules across multiple games.", "description_zh": "基于 Video Game Description Language 的框架，可跨多款游戏评估规划器，并生成关卡或游戏规则。", "content_type": "game", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/GAIGResearch/GVGAI", "repo_url": "https://github.com/GAIGResearch/GVGAI", "demo_url": null, "paper_url": "https://arxiv.org/abs/1802.10363", "limits_en": "The main repository covers planning and procedural-content tracks; the learning-track implementation is linked separately. Setup follows older Java/IDE instructions and modern compatibility was not tested. Code is GPL-3.0-or-later. Report track settings and forward-model access when comparing results.", "limits_zh": "主仓库覆盖规划与程序化内容生成赛道，学习赛道实现另有链接。配置沿用较旧的 Java/IDE 说明，未测试现代环境兼容性。代码采用 GPL-3.0-or-later。比较结果时需说明赛道设置及前向模型访问条件。", "enables_en": "Compare planners across VGDL games, replay action traces, and evaluate generated levels or rules.", "enables_zh": "跨 VGDL 游戏比较规划器、回放动作轨迹，并评估生成的关卡或规则。", "resource_kind": "environment-framework", "runtime": "Java framework with track-specific main classes, sample controllers and game/level descriptions; no modern Python/Gymnasium compatibility is implied.", "evidence": [ { "url": "https://github.com/GAIGResearch/GVGAI#readme", "supports": "Identifies planning/PCG scope and links the learning track to a separate repository." }, { "url": "https://github.com/GAIGResearch/GVGAI/blob/master/README_CODE_STRUCTURE.txt", "supports": "Lists Java entry points for human/controller play, replay, multiplayer, level and rule generation." }, { "url": "https://github.com/GAIGResearch/GVGAI/blob/master/src/tracks/singlePlayer/Test.java", "supports": "Inspectable example loads game/level descriptions and starts human/controller play; not executed." }, { "url": "https://github.com/GAIGResearch/GVGAI/blob/master/LICENSE.txt", "supports": "Framework license states GPL version 3 or later." }, { "url": "https://arxiv.org/abs/1802.10363", "supports": "Primary survey explains GVGAI, VGDL and the different evaluation tracks." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "No operating-system tags are inferred from Java." }, { "id": "a9-voice", "name": "A9 Voice", "creator": "Atlas9 LLC / Jwishon", "description_en": "A Windows dictation and meeting-notes application with hotkey voice input, transcription and locally stored note folders.", "description_zh": "提供快捷键语音输入、转写和本机笔记文件夹的 Windows 听写及会议笔记应用。", "platforms": [ "windows" ], "implementation": "ai-assisted", "primary_url": "https://a9voice.com/", "repo_url": null, "distribution_repository_url": "https://github.com/atlas9-ai/a9voice-releases", "download_url": "https://github.com/atlas9-ai/a9voice-releases/releases/tag/v1.0.9", "provenance_source": "https://www.reddit.com/r/windowsapps/comments/1x36rwl/i_vibecoded_a_windows_dictation_meetingnotes_app/", "enables_en": "Dictate into other applications and organize meeting transcripts and notes.", "enables_zh": "向其他应用输入听写文本，并整理会议转写和笔记。", "limits_en": "Requires 64-bit Windows 10/11. Dictation is free; Notes has a trial then a paid license. Selected cloud providers receive the configured audio/text, and license checks require network access.", "limits_zh": "需要 64 位 Windows 10/11。听写免费，Notes 试用后需付费许可。选择云服务时，相应音频或文本会发送给提供方，许可检查也需要联网。", "provenance_note_en": "The maker says Claude wrote nearly all application code and describes agents implementing feature specifications on branches, with human product decisions, testing and review.", "provenance_note_zh": "作者说明 Claude 编写了应用大部分代码，并描述 Agent 根据功能规格在分支中实现，由人类负责产品决策、测试和审查。", "license_en": "Proprietary. The public GitHub repository distributes releases and documentation, not application source.", "license_zh": "专有软件。公开 GitHub 仓库用于分发版本和文档，不提供应用源码。", "license_url": "https://github.com/atlas9-ai/a9voice-releases/blob/main/LICENSE", "evidence": [ { "url": "https://www.reddit.com/r/windowsapps/comments/1x36rwl/i_vibecoded_a_windows_dictation_meetingnotes_app/", "supports": "Maker Jwishon gives a detailed Claude/agent implementation account and links the exact official website and distribution repository." }, { "url": "https://github.com/atlas9-ai/a9voice-releases#readme", "supports": "Publisher documents native Windows 10/11 x64 distribution and app functions." }, { "url": "https://github.com/atlas9-ai/a9voice-releases/releases/tag/v1.0.9", "supports": "Publisher release dated 2026-10-16 includes Windows x64 EXE installers; none was downloaded or run." }, { "url": "https://github.com/atlas9-ai/a9voice-releases/blob/main/LICENSE", "supports": "Publisher's proprietary license distinguishes free dictation, trial/paid Notes, cloud-provider data handling and license checks." } ], "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "better-stickies", "name": "Better Stickies", "creator": "HimaSphere", "description_en": "A desktop sticky-note application for rich text, file shortcuts, pasted images, code blocks, reminders and tabbed notes.", "description_zh": "支持富文本、文件快捷方式、粘贴图片、代码块、提醒和分页便签的桌面便签应用。", "platforms": [ "windows", "linux" ], "implementation": "ai-assisted", "primary_url": "https://betterstickies.com/", "repo_url": null, "distribution_repository_url": "https://github.com/HimaSphere/BetterStickies", "download_url": "https://github.com/HimaSphere/BetterStickies/releases/tag/v1.2.1", "provenance_source": "https://www.reddit.com/r/ClaudeAI/comments/1weari5/comment/p9c52cm/", "enables_en": "Keep mixed-format notes and reminders on the desktop without moving them into a browser.", "enables_zh": "直接在桌面保存混合格式的便签及提醒。", "limits_en": "The free edition limits note slots; Premium removes that limit. Windows EXE and Linux packages are published, while macOS remains listed as in development in the reviewed README.", "limits_zh": "免费版限制便签槽位，Premium 可解除限制。已发布 Windows EXE 和 Linux 安装包，核查时 README 仍将 macOS 列为开发中。", "provenance_note_en": "The maintainer's artifact-specific post says the app was built entirely with Claude Code and describes its PySide6 implementation and ongoing feature work; exact code authorship is not independently audited.", "provenance_note_zh": "维护者针对该应用的帖子说明全程使用 Claude Code 构建，并介绍 PySide6 实现及后续功能开发；具体代码归属未独立审计。", "license_en": "Proprietary, with free and paid editions; the GitHub repository explicitly states it contains releases only, not source code.", "license_zh": "专有软件，分免费版和付费版；GitHub 仓库明确说明只提供发布文件，不提供源码。", "license_url": "https://github.com/HimaSphere/BetterStickies#readme", "evidence": [ { "url": "https://www.reddit.com/r/ClaudeAI/comments/1weari5/comment/p9c52cm/", "supports": "Comment by the maintainer HimaSphere explicitly attributes the app's development to Claude Code and identifies its functions and desktop technology." }, { "url": "https://github.com/HimaSphere/BetterStickies#readme", "supports": "Publisher documents Windows/Linux support, macOS status, free/Premium distinction and proprietary distribution-only repository." }, { "url": "https://github.com/HimaSphere/BetterStickies/releases/tag/v1.2.1", "supports": "Release contains Windows EXE and Linux AppImage/tar packages; no package was executed." } ], "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "screen-printing-film-analyzer", "name": "Screen Printing Film Analyzer", "creator": "Gerard (desordenades) / Trama Serigrafia", "description_en": "A Python command-line utility that examines printing-film PDFs and exports annotated TIFF images and a text report.", "description_zh": "检查丝网印刷制版 PDF，并导出标注 TIFF 图片及文本报告的 Python 命令行工具。", "platforms": [ "windows", "macos", "linux" ], "implementation": "ai-assisted", "primary_url": "https://github.com/desordenades/Positive_Analyzer_for_Screen_Printing", "repo_url": "https://github.com/desordenades/Positive_Analyzer_for_Screen_Printing", "download_url": null, "provenance_source": "https://github.com/desordenades/Positive_Analyzer_for_Screen_Printing#readme", "enables_en": "Review thin line/dot features in PDF printing films and inspect the workshop's automation script.", "enables_zh": "检查印刷制版 PDF 中过细的线条或网点，并查看工作室的自动化脚本。", "limits_en": "Requires Python dependencies and workshop-specific calibration. The actual file is Screen_Printing_Positive_Analyzer.py, while the README names a different script. SMTP reporting is optional; processing was not executed here.", "limits_zh": "需要 Python 依赖及针对工作室的参数校准。实际文件为 Screen_Printing_Positive_Analyzer.py，README 使用了不同文件名。SMTP 报告可选，本轮未执行处理。", "provenance_note_en": "The maintainer explicitly says the script was generated with Gemini assistance to solve a daily workshop problem, then adjusted using real manual-printing designs.", "provenance_note_zh": "维护者明确说明脚本借助 Gemini 生成，用于解决工作室日常问题，并使用实际手工印刷设计调整。", "runtime_note_en": "Runs as a Python script. The maintainer documents Windows CMD and macOS/Linux environment setup; this is not a browser application or a packaged Windows EXE.", "runtime_note_zh": "通过 Python 解释器运行。维护者记录了 Windows CMD 和 macOS/Linux 环境设置；它不是浏览器应用，也不是打包的 Windows EXE。", "license_en": "GPL-3.0 for the repository.", "license_zh": "仓库采用 GPL-3.0。", "license_url": "https://github.com/desordenades/Positive_Analyzer_for_Screen_Printing/blob/main/LICENSE", "evidence": [ { "url": "https://github.com/desordenades/Positive_Analyzer_for_Screen_Printing#readme", "supports": "Maintainer explicitly attributes script generation to Gemini assistance and documents PDF/TIFF workflow and per-platform environment configuration." }, { "url": "https://github.com/desordenades/Positive_Analyzer_for_Screen_Printing/blob/main/Screen_Printing_Positive_Analyzer.py", "supports": "Actual public Python artifact exists under this filename; static parsing succeeded without execution." }, { "url": "https://github.com/desordenades/Positive_Analyzer_for_Screen_Printing/blob/main/LICENSE", "supports": "Repository provides a GPL version 3 license." } ], "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "poirot", "name": "Poirot", "creator": "a7t-ai maintainers", "description_en": "A native macOS companion for browsing Claude Code sessions, inspecting tool activity and reviewing conversation history.", "description_zh": "用于浏览 Claude Code 会话、检查工具操作和回看对话历史的原生 macOS 配套应用。", "platforms": [ "macos" ], "implementation": "ai-assisted", "primary_url": "https://github.com/a7t-ai/poirot", "repo_url": "https://github.com/a7t-ai/poirot", "download_url": "https://github.com/a7t-ai/poirot/releases/tag/v26.08.5", "provenance_source": "https://github.com/a7t-ai/poirot#the-story", "enables_en": "Inspect locally stored agent sessions and their file changes from a desktop interface.", "enables_zh": "通过桌面界面查看本地保存的 Agent 会话及文件修改。", "limits_en": "Requires macOS 15+. Session views depend on existing Claude Code data; rate-limit features rely on its stored authentication and provider interfaces.", "limits_zh": "需要 macOS 15 及以上版本。会话视图依赖已有 Claude Code 数据，用量限制功能依赖其保存的认证信息和提供方接口。", "provenance_note_en": "The maintainers specifically identify architecture, parser, UI and tests as components built with Claude Code as a copilot, rather than merely stating that the app integrates an AI tool.", "provenance_note_zh": "维护者具体说明架构、解析器、界面和测试均与 Claude Code 协作构建，不只是描述应用接入了 AI 工具。", "license_en": "MIT for the published application source.", "license_zh": "公开应用源码采用 MIT。", "license_url": "https://github.com/a7t-ai/poirot/blob/main/LICENSE", "evidence": [ { "url": "https://github.com/a7t-ai/poirot#the-story", "supports": "First-party statement identifies substantial Claude Code participation across architecture, parsing, UI and tests." }, { "url": "https://github.com/a7t-ai/poirot#readme", "supports": "Repository documents native SwiftUI implementation, macOS requirements and local transcript dependencies." }, { "url": "https://github.com/a7t-ai/poirot/releases/tag/v26.08.5", "supports": "Published release contains Poirot-v26.08.5.dmg; no application was installed or run." }, { "url": "https://github.com/a7t-ai/poirot/blob/main/LICENSE", "supports": "Application source has an MIT license." } ], "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "nacre", "name": "Nacre", "creator": "RYO ITABASHI", "description_en": "An Android Japanese keyboard with split layouts, kana-kanji conversion, developer shortcuts and optional voice-input processing.", "description_zh": "提供分体布局、假名汉字转换、开发者快捷键及可选语音处理的 Android 日语输入法。", "platforms": [ "android" ], "implementation": "ai-built", "primary_url": "https://github.com/RYOITABASHI/Nacre", "repo_url": "https://github.com/RYOITABASHI/Nacre", "download_url": "https://github.com/RYOITABASHI/Nacre/releases/tag/build-365", "provenance_source": "https://github.com/RYOITABASHI/Shelly#the-story", "enables_en": "Type Japanese on an Android device with a keyboard designed for compact and foldable layouts.", "enables_zh": "使用面向紧凑屏幕与折叠屏布局设计的键盘，在 Android 设备上输入日语。", "limits_en": "Distributed as a developer APK requiring input-method setup. The build declares Android API 26 minimum, while optional speech/language models require separate downloads and suitable device resources.", "limits_zh": "以开发版 APK 分发，需要进行输入法设置。构建声明最低 Android API 26，可选语音及语言模型还需单独下载并满足设备资源要求。", "provenance_note_en": "In his own Shelly development account, the same maintainer names and links Nacre and explicitly says this Kotlin keyboard was created entirely through AI conversation; the claim concerns this distinct keyboard app.", "provenance_note_zh": "同一维护者在自己的 Shelly 开发说明中点名并链接 Nacre，明确表示这款 Kotlin 键盘完全通过与 AI 对话创建；声明针对独立的输入法应用。", "license_en": "Apache-2.0 for Nacre; third-party dictionaries and runtimes retain BSD, MIT and LGPL licenses documented in the repository.", "license_zh": "Nacre 采用 Apache-2.0；第三方词典及运行时保留仓库列明的 BSD、MIT 和 LGPL 许可。", "license_url": "https://github.com/RYOITABASHI/Nacre/blob/main/LICENSE", "evidence": [ { "url": "https://github.com/RYOITABASHI/Shelly#the-story", "supports": "Maintainer's first-party account explicitly identifies Nacre by name/link and attributes its creation to AI conversation." }, { "url": "https://github.com/RYOITABASHI/Nacre#readme", "supports": "Keyboard-specific repository documents functionality, model downloads and build/distribution requirements." }, { "url": "https://github.com/RYOITABASHI/Nacre/blob/main/app/build.gradle.kts", "supports": "Build configuration declares minSdk 26; this is a source declaration, not an independent device test." }, { "url": "https://github.com/RYOITABASHI/Nacre/releases/tag/build-365", "supports": "Release listing contains nacre-365.apk; the APK was not installed." }, { "url": "https://github.com/RYOITABASHI/Nacre/blob/main/THIRD_PARTY_LICENSES.md", "supports": "Repository separately records licenses for Mozc, whisper.cpp, llama.cpp and KenLM." } ], "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "shagunly", "name": "Shagunly", "creator": "Piyush Grover", "description_en": "A native iPhone and iPad ledger for recording gifts and cash exchanged at celebrations, organized by person and occasion.", "description_zh": "按人物和场合记录庆典礼物及礼金往来的原生 iPhone/iPad 礼簿应用。", "platforms": [ "ios" ], "implementation": "ai-built", "primary_url": "https://apps.apple.com/us/app/shagunly/id6783436742", "repo_url": null, "download_url": "https://apps.apple.com/us/app/shagunly/id6783436742", "provenance_source": "https://piyushgrover.com/shagunly.html", "enables_en": "Record gifts given and received and revisit the history of a relationship or event.", "enables_zh": "记录赠出和收到的礼物，并回看特定关系或活动的往来历史。", "limits_en": "Requires iOS/iPadOS 17+ and account sign-in for its hosted backend. The creator still describes Android as in development; no Android or native macOS support is claimed here.", "limits_zh": "需要 iOS/iPadOS 17 及以上版本，并登录账户使用托管后端。作者仍将 Android 列为开发中，本条目不扩展为 Android 或原生 macOS 支持。", "provenance_note_en": "The creator's case study says he designed the product in Figma and directed Claude to implement every screen, flow and backend interaction without hand-writing code.", "provenance_note_zh": "作者案例说明自己在 Figma 中设计产品，再指导 Claude 实现各个界面、流程和后端交互，没有手工编写代码。", "license_en": "Public App Store distribution; no public application source or open-source license was identified.", "license_zh": "通过 App Store 公开分发，未找到公开应用源码或开源许可。", "license_url": null, "evidence": [ { "url": "https://piyushgrover.com/shagunly.html", "supports": "Creator's detailed case study separates human Figma/product design from Claude-generated implementation and directly links the app." }, { "url": "https://apps.apple.com/us/app/shagunly/id6783436742", "supports": "Public store listing identifies Piyush Grover, iPhone/iPad availability and current app features." }, { "url": "https://itunes.apple.com/lookup?id=6783436742&country=us", "supports": "Apple lookup confirms exact app identity, seller and iOS 17 minimum." } ], "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "travel-times-milwaukee", "name": "Travel Times Milwaukee", "creator": "Vinny Carpenter", "description_en": "An iPhone and iPad utility for Milwaukee freeway travel times, incidents, favorite routes and commute alerts.", "description_zh": "展示密尔沃基高速路行程时间、事故信息、收藏路线及通勤提醒的 iPhone/iPad 工具。", "platforms": [ "ios" ], "implementation": "ai-assisted", "primary_url": "https://traveltimesapp.com/", "repo_url": null, "download_url": "https://apps.apple.com/us/app/travel-times-milwaukee/id932688030", "provenance_source": "https://vinny.dev/blog/2026-04-12-one-week-with-ai-copilot/", "enables_en": "Review regional traffic information and organize frequently used routes in a native mobile interface.", "enables_zh": "在原生移动界面中查看地区交通信息，并整理常用路线。", "limits_en": "The current store metadata requires iOS/iPadOS 26+. Information depends on Wisconsin traffic feeds; data accuracy/freshness was not independently checked. The separately described Android port has no verified public artifact in this review.", "limits_zh": "当前商店元数据要求 iOS/iPadOS 26 及以上版本。信息依赖威斯康星州交通数据，本轮未独立核验准确性和时效；另述 Android 移植尚未核实公开产物。", "provenance_note_en": "The original 2014 app was human-built. Carpenter's 2026 account details substantial Claude-generated feature and refactoring work, with line-by-line human review and some manual fixes; the label applies to that documented evolution.", "provenance_note_zh": "2014 年原始应用由人工构建。Carpenter 的 2026 年记录详细描述大量 Claude 生成的功能和重构、逐行人工审查及部分手动修复；标签针对这段有记录的演进。", "license_en": "Public App Store application; no public source repository or open-source license was identified in reviewed materials.", "license_zh": "公开 App Store 应用，已核查资料中未找到公共源码仓库或开源许可。", "license_url": null, "evidence": [ { "url": "https://vinny.dev/blog/2026-04-12-one-week-with-ai-copilot/", "supports": "Creator gives artifact-specific accounts of AI implementation, existing human code, manual review and corrections during the iOS rebuild." }, { "url": "https://apps.apple.com/us/app/travel-times-milwaukee/id932688030", "supports": "Store lists the same creator and released iPhone/iPad app; version history corroborates the documented 2026 feature additions." }, { "url": "https://itunes.apple.com/lookup?id=932688030&country=us", "supports": "Apple metadata confirms current app title, Vinny Carpenter as seller and iOS 26 minimum." }, { "url": "https://traveltimesapp.com/", "supports": "Creator's official website links the exact store artifact and identifies the regional data source." } ], "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "paper_url": null, "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "craftax", "name": "Craftax", "description_en": "A JAX survival-and-crafting world that extends Crafter with roguelike mechanics; Craftax-Classic is a separate simpler mode in the same project.", "description_zh": "用 JAX 实现的生存与制作世界，在 Crafter 基础上扩展了类 roguelike 机制；同一项目另提供较简化的 Craftax-Classic 模式。", "content_type": "game", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://craftaxenv.github.io/", "repo_url": "https://github.com/MichaelTMatthews/Craftax", "demo_url": null, "paper_url": "https://arxiv.org/abs/2402.16801", "limits_en": "Craftax and Craftax-Classic have different mechanics and should not share scores. Version fixes changed plant growth and Classic death rewards. Python 3.9+ is required; accelerator JAX must be installed separately and first use compiles functions. Code is MIT; no gameplay was tested.", "limits_zh": "Craftax 与 Craftax-Classic 的机制不同，不能混用成绩。版本修复改变了植物生长和 Classic 死亡奖励。需要 Python 3.9+，加速器版 JAX 需另行安装，首次使用会编译函数。代码采用 MIT，未实际测试游戏。", "enables_en": "Study exploration, long-term planning and progression in a procedurally generated world, using explicit environment and reset settings.", "enables_zh": "在程序化生成的世界中研究探索、长期规划和成长过程，并明确记录环境及重置设置。", "resource_kind": "environment-and-benchmark", "runtime": "PyPI craftax 1.6.1; Python/JAX with a gymnax-style API and optional Pygame human-play commands.", "evidence": [ { "url": "https://github.com/MichaelTMatthews/Craftax#readme", "supports": "Documents Crafter lineage, expanded/Classic modes, setup, JAX compilation, reset pitfalls and version errata." }, { "url": "https://github.com/MichaelTMatthews/Craftax/blob/main/pyproject.toml", "supports": "Declares version 1.6.1 and Python >=3.9." }, { "url": "https://github.com/MichaelTMatthews/Craftax/blob/main/LICENSE", "supports": "Project code is MIT licensed." }, { "url": "https://arxiv.org/abs/2402.16801", "supports": "Original paper distinguishes Craftax-Classic from the expanded Craftax benchmark." }, { "url": "https://pypi.org/project/craftax/1.6.1/", "supports": "PyPI metadata observed at review lists craftax 1.6.1, Python >=3.9; distributions were not downloaded." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "CPU/GPU/TPU are execution backends, not documented operating-system targets." }, { "id": "jumanji", "name": "Jumanji", "description_en": "A JAX suite of interactive logic games, packing, routing and other decision-making environments with configurable problem instances.", "description_zh": "用 JAX 实现的交互式逻辑游戏、装箱、路径规划等决策环境套件，可配置问题实例。", "content_type": "game", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://instadeepai.github.io/jumanji/", "repo_url": "https://github.com/instadeepai/jumanji", "demo_url": null, "paper_url": "https://arxiv.org/abs/2306.09884", "limits_en": "Requires Python 3.10+ and a suitable JAX backend. Rendering uses a Matplotlib GUI backend. Environment versions and instance generators affect comparisons; internal simulator state is not the same as the agent's observation. Repository code is Apache 2.0.", "limits_zh": "需要 Python 3.10+ 和合适的 JAX 后端，渲染依赖 Matplotlib GUI 后端。环境版本和实例生成器会影响比较；模拟器内部状态不同于 Agent 可见的观察。仓库代码采用 Apache 2.0。", "enables_en": "Train or compare policies on game-like and combinatorial tasks while varying initial-state distributions and problem size.", "enables_zh": "在游戏式与组合决策任务上训练或比较策略，并改变初始状态分布和问题规模。", "resource_kind": "environment-and-benchmark", "runtime": "PyPI jumanji 1.1.2; JAX state/timestep API with JIT/vectorization and optional rendering/training examples.", "evidence": [ { "url": "https://github.com/instadeepai/jumanji#readme", "supports": "Documents task categories, installation, state-versus-observation distinction, rendering and environment versioning." }, { "url": "https://github.com/instadeepai/jumanji/blob/main/pyproject.toml", "supports": "Current metadata requires Python >=3.10 and lists JAX dependencies." }, { "url": "https://github.com/instadeepai/jumanji/blob/main/LICENSE", "supports": "Repository code is Apache 2.0 licensed." }, { "url": "https://arxiv.org/abs/2306.09884", "supports": "Primary paper introduces the configurable environment suite." }, { "url": "https://pypi.org/project/jumanji/1.1.2/", "supports": "PyPI metadata observed at review lists jumanji 1.1.2, Python >=3.10; distributions were not downloaded." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "The reviewed project does not publish a single OS support matrix for every JAX/backend configuration." }, { "id": "melting-pot", "name": "Melting Pot", "description_en": "A multi-agent social-evaluation suite pairing interactive worlds with background agent populations to test generalization to unfamiliar partners.", "description_zh": "将交互世界与背景 Agent 群体组合的多 Agent 社会交互评测套件，用于测试面对陌生伙伴时的泛化能力。", "content_type": "game", "platforms": [ "linux", "macos" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/google-deepmind/meltingpot", "repo_url": "https://github.com/google-deepmind/meltingpot", "demo_url": null, "paper_url": "https://arxiv.org/abs/2211.13746", "limits_en": "Substrates used for training and evaluation scenarios with background populations are different units. Current package requires Python 3.11+, Lab2D and TensorFlow; missing Lab2D wheels require source builds. Pin substrate/scenario and population versions. Repository code is Apache 2.0.", "limits_zh": "训练使用的基础世界与包含背景群体的评测场景是不同单位。当前包要求 Python 3.11+、Lab2D 和 TensorFlow；缺少对应 Lab2D wheel 时需源码构建。应固定基础世界、场景和群体版本。仓库代码采用 Apache 2.0。", "enables_en": "Evaluate cooperation, competition and social adaptation beyond an agent's original training partners.", "enables_zh": "评估超出原训练伙伴范围的合作、竞争与社会适应能力。", "resource_kind": "environment-and-benchmark", "runtime": "PyPI dm-meltingpot 2.4.1; Python with dmlab2d, dm_env and model/evaluation dependencies; optional human-player scripts.", "evidence": [ { "url": "https://github.com/google-deepmind/meltingpot#readme", "supports": "Documents training substrates, social evaluation, background populations, installation and Lab2D dependency." }, { "url": "https://github.com/google-deepmind/meltingpot/blob/main/setup.py", "supports": "Requires Python >=3.11 and declares Linux/macOS and TensorFlow/dmlab2d dependencies." }, { "url": "https://github.com/google-deepmind/meltingpot/blob/main/LICENSE", "supports": "Repository code is Apache 2.0 licensed." }, { "url": "https://arxiv.org/abs/2107.06857", "supports": "Original paper defines social-generalization evaluation." }, { "url": "https://arxiv.org/abs/2211.13746", "supports": "Version 2.0 report explains substrates, scenarios, background populations and asymmetric roles." }, { "url": "https://pypi.org/project/dm-meltingpot/2.4.1/", "supports": "PyPI metadata observed at review lists dm-meltingpot 2.4.1, Python >=3.11; distributions were not downloaded." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "deepmind-lab", "name": "DeepMind Lab", "description_en": "A customizable first-person 3D game platform for visual navigation, memory and control tasks, built on the Quake III/ioquake3 engine lineage.", "description_zh": "可定制的第一人称三维游戏平台，用于视觉导航、记忆和控制任务，基于 Quake III/ioquake3 引擎体系。", "content_type": "game", "platforms": [ "linux" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/google-deepmind/lab", "repo_url": "https://github.com/google-deepmind/lab", "demo_url": null, "paper_url": "https://arxiv.org/abs/1612.03801", "limits_en": "Source-build instructions target Linux x86 and include historical Python/Bazel guidance; modern compatibility was not tested. Rendering needs SDL2 and an OpenGL/EGL/GLX or software backend. Licensing is mixed: engine/OpenArena components use GPLv2, main assets CC BY 4.0, with additional component notices.", "limits_zh": "源码构建说明面向 Linux x86，包含历史 Python/Bazel 配置，本次未测试现代兼容性。渲染需要 SDL2 及 OpenGL/EGL/GLX 或软件后端。许可混合：引擎和 OpenArena 组件采用 GPLv2，主要素材采用 CC BY 4.0，其他组件另有说明。", "enables_en": "Create and compare visually rich, partially observed tasks using a programmable level system and agent API.", "enables_zh": "通过可编程关卡系统和 Agent 接口，创建并比较视觉丰富、部分可观测的任务。", "resource_kind": "environment-framework", "runtime": "Source-only Bazel build with Python/dm_env bindings and Lua level logic; optional self-built pip package. Latest listed GitHub release is release-2020-12-07.", "evidence": [ { "url": "https://github.com/google-deepmind/lab#readme", "supports": "Documents Linux-oriented source builds, APIs, dependencies and human/random-agent examples." }, { "url": "https://github.com/google-deepmind/lab/blob/master/docs/users/build.md", "supports": "Provides author build guidance for the source distribution." }, { "url": "https://github.com/google-deepmind/lab/blob/master/LICENSE", "supports": "Separates GPLv2 engine/OpenArena, CC BY 4.0 assets and other component terms." }, { "url": "https://github.com/google-deepmind/lab/releases/tag/release-2020-12-07", "supports": "Latest non-prerelease GitHub listing observed at review; this is not a claim that current source is identical." }, { "url": "https://arxiv.org/abs/1612.03801", "supports": "Original platform paper describes first-person visual and partially observed worlds." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "Linux is the documented build target; other operating systems are not inferred from the underlying engine." }, { "id": "deepmind-lab2d", "name": "DeepMind Lab2D", "description_en": "A C++/Lua engine for building layered two-dimensional grid worlds with multiple agents, text maps and programmable behavior.", "description_zh": "用于构建分层二维网格世界的 C++/Lua 引擎，支持多个 Agent、文本地图和可编程行为。", "content_type": "game", "platforms": [ "linux", "macos" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/google-deepmind/lab2d", "repo_url": "https://github.com/google-deepmind/lab2d", "demo_url": null, "paper_url": "https://arxiv.org/abs/2011.07027", "limits_en": "Lab2D is an environment engine, distinct from the Melting Pot scenarios built on it and the 3D DeepMind Lab project. Wheels are provided for selected Linux/macOS configurations; other setups need a Bazel source build. Code is Apache 2.0, with external dependency terms.", "limits_zh": "Lab2D 是环境引擎，与基于它构建的 Melting Pot 场景及三维 DeepMind Lab 项目不同。部分 Linux/macOS 配置提供 wheel，其他配置需要 Bazel 源码构建。代码采用 Apache 2.0，外部依赖另有条款。", "enables_en": "Design custom multi-agent worlds and expose them through dm_env or the lower-level C interface.", "enables_zh": "设计自定义多 Agent 世界，并通过 dm_env 或底层 C 接口使用。", "resource_kind": "environment-framework", "runtime": "PyPI dmlab2d 1.0.2; native C++/Lua backend, Python dm_env interface and platform-specific wheels or Bazel build.", "evidence": [ { "url": "https://github.com/google-deepmind/lab2d#readme", "supports": "Defines grid-world authoring, multiple agents, APIs, wheels and native-build constraints." }, { "url": "https://github.com/google-deepmind/lab2d/blob/main/LICENSE", "supports": "Repository code is Apache 2.0 licensed." }, { "url": "https://arxiv.org/abs/2011.07027", "supports": "Original whitepaper describes researcher-led multi-agent environment design." }, { "url": "https://pypi.org/project/dmlab2d/1.0.2/", "supports": "PyPI metadata observed at review lists dmlab2d 1.0.2; distributions were not downloaded." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "bsuite", "name": "Behaviour Suite (bsuite)", "description_en": "A collection of controlled interactive RL experiments designed to diagnose exploration, memory and other agent capabilities with standardized logging.", "description_zh": "一组受控的交互式强化学习实验，通过标准化记录诊断探索、记忆等 Agent 能力。", "content_type": "game", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/google-deepmind/bsuite", "repo_url": "https://github.com/google-deepmind/bsuite", "demo_url": null, "paper_url": "https://arxiv.org/abs/1908.03568", "limits_en": "These are diagnostic tasks rather than a realistic open world. Evaluation depends on the experiment ID, configuration and prescribed episode count. README setup examples reference older Python versions and optional baseline stacks; current compatibility was not tested. Code is Apache 2.0.", "limits_zh": "这些是诊断任务，而非写实开放世界。评测依赖实验 ID、配置和规定的回合数。README 配置示例涉及较旧的 Python 版本及可选基线依赖，本次未测试当前兼容性。代码采用 Apache 2.0。", "enables_en": "Compare learning behavior across small controlled tasks and inspect where an agent succeeds or fails.", "enables_zh": "跨小型受控任务比较学习行为，检查 Agent 成功或失败的具体能力。", "resource_kind": "benchmark-suite", "runtime": "PyPI bsuite 0.3.6; dm_env interface, experiment sweeps and CSV/reporting tools; optional baseline dependencies are separate.", "evidence": [ { "url": "https://github.com/google-deepmind/bsuite#readme", "supports": "Documents experiment/configuration IDs, dm_env interaction, episode budgets, logging and optional baselines." }, { "url": "https://github.com/google-deepmind/bsuite/blob/main/setup.py", "supports": "Shows distinct environment versus baseline dependency groups." }, { "url": "https://github.com/google-deepmind/bsuite/blob/main/LICENSE", "supports": "Repository code is Apache 2.0 licensed." }, { "url": "https://arxiv.org/abs/1908.03568", "supports": "Primary paper defines the controlled capability-diagnostic suite." }, { "url": "https://pypi.org/project/bsuite/0.3.6/", "supports": "PyPI metadata observed at review lists bsuite 0.3.6; distributions were not downloaded." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "Package classifiers list OS families, but this review does not establish a tested support matrix for the optional stacks." }, { "id": "pufferlib", "name": "PufferLib / Ocean", "description_en": "An RL execution framework with an Ocean collection of native game and simulation implementations, alongside environment integration and training infrastructure.", "description_zh": "包含 Ocean 原生游戏与模拟实现集合的强化学习执行框架，并提供环境集成和训练基础设施。", "content_type": "game", "platforms": [ "linux", "macos", "web" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://puffer.ai/", "repo_url": "https://github.com/PufferAI/PufferLib", "demo_url": null, "paper_url": "https://arxiv.org/abs/2406.12905", "limits_en": "PyPI still lists 3.0.0 while the default source branch is 5.0 with C/CUDA builds; do not mix their setup recipes. macOS CPU and web paths cover selected environments, not all training modes. The 5.0-experiments release contains experiment/model data. MIT framework terms do not establish all third-party asset rights.", "limits_zh": "PyPI 仍列为 3.0.0，而默认源码分支为使用 C/CUDA 构建的 5.0，不能混用配置说明。macOS CPU 与 Web 路径仅覆盖部分环境，不代表所有训练模式。5.0-experiments 发布的是实验和模型数据。MIT 框架许可不代表所有第三方素材权利。", "enables_en": "Build or study native simulation tasks, run vectorized agent experiments and inspect the source of selected Ocean environments.", "enables_zh": "构建或研究原生模拟任务，运行向量化 Agent 实验，并查看选定 Ocean 环境的源码。", "resource_kind": "environment-framework", "runtime": "Older Python distribution: pufferlib 3.0.0. Current 5.0 source has native CUDA training, CPU play/evaluation and Emscripten web build modes.", "evidence": [ { "url": "https://github.com/PufferAI/PufferLib/tree/5.0/ocean", "supports": "Inspectable native environment source collection establishes substantive simulator artifacts beyond wrappers." }, { "url": "https://github.com/PufferAI/PufferLib/blob/5.0/build.sh", "supports": "Source documents native/CUDA, CPU and web modes and Linux/macOS-specific dependencies; not executed." }, { "url": "https://puffer.ai/blog/2.0/", "supports": "First-party account explains the introduction of independently implemented Ocean C environments." }, { "url": "https://github.com/PufferAI/PufferLib/releases/tag/5.0-experiments", "supports": "Release body identifies experiment logs and pretrained models, not a new Python package." }, { "url": "https://github.com/PufferAI/PufferLib/blob/5.0/LICENSE", "supports": "Framework repository is MIT licensed." }, { "url": "https://arxiv.org/abs/2406.12905", "supports": "Original paper describes the earlier interoperability/vectorization architecture, not current 5.0 compatibility." }, { "url": "https://pypi.org/project/pufferlib/3.0.0/", "supports": "PyPI metadata observed at review lists pufferlib 3.0.0, Python >=3.9; distributions were not downloaded." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "The source has Linux/macOS CPU and Emscripten browser paths; CUDA training availability and per-environment support are narrower." }, { "id": "gymnax", "name": "Gymnax", "description_en": "A functional JAX environment library with implemented classic-control, diagnostic and small-game tasks, exposing explicit state, randomness and parameters.", "description_zh": "采用函数式接口的 JAX 环境库，实现经典控制、诊断和小型游戏任务，显式传递状态、随机数和参数。", "content_type": "game", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/RobertTLange/gymnax", "repo_url": "https://github.com/RobertTLange/gymnax", "demo_url": null, "paper_url": null, "limits_en": "Many tasks reimplement Gym, bsuite or MinAtar and are not new benchmark families. PyPI 1.0.0 requires Python 3.10–3.13; the default env-dev branch contains older setup text. Pin code and task parameters and do not assume behavior matches upstream implementations. Code is Apache 2.0.", "limits_zh": "许多任务重新实现 Gym、bsuite 或 MinAtar，不应算作新的基准系列。PyPI 1.0.0 要求 Python 3.10–3.13，默认 env-dev 分支仍含较旧的配置文字。应固定代码和任务参数，不假定行为与上游实现完全相同。代码采用 Apache 2.0。", "enables_en": "Run batched JAX rollouts and vary environment parameters for controlled or meta-learning experiments.", "enables_zh": "批量运行 JAX 轨迹，并改变环境参数进行受控实验或元学习研究。", "resource_kind": "environment-framework", "runtime": "PyPI gymnax 1.0.0; JAX/jaxlib and related Python dependencies, with an explicit-state reset/step API.", "evidence": [ { "url": "https://github.com/RobertTLange/gymnax/blob/env-dev/README.md", "supports": "Documents JAX interface, implemented task families, installation and repository-only software citation." }, { "url": "https://github.com/RobertTLange/gymnax/blob/env-dev/gymnax/environments/bsuite/deep_sea.py", "supports": "Contains a substantive JAX state-transition implementation and attribution to original bsuite." }, { "url": "https://github.com/RobertTLange/gymnax/blob/env-dev/LICENSE", "supports": "Repository code is Apache 2.0 licensed." }, { "url": "https://pypi.org/project/gymnax/1.0.0/", "supports": "PyPI metadata observed at review lists gymnax 1.0.0, Python <3.14,>=3.10; distributions were not downloaded." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "No OS tag is inferred from JAX backend availability." }, { "id": "griddly", "name": "Griddly", "description_en": "A configurable grid-world game engine using a YAML description language to define mechanics, observations and single- or multi-agent tasks.", "description_zh": "使用 YAML 描述语言定义机制、观察与单 Agent 或多 Agent 任务的可配置网格游戏引擎。", "content_type": "game", "platforms": [ "linux", "macos", "windows", "web" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://griddly.readthedocs.io/", "repo_url": "https://github.com/Bam4d/Griddly", "demo_url": null, "paper_url": "https://arxiv.org/abs/2011.06363", "limits_en": "Pixel rendering and native builds have Vulkan/toolchain requirements. PyPI 1.6.7, older GitHub releases and the develop branch are different snapshots. Engine code is MIT, but official documentation says the Oryx artwork requires a purchased license for use in a product; do not assume the art is MIT.", "limits_zh": "像素渲染和原生构建有 Vulkan 及工具链要求。PyPI 1.6.7、较旧 GitHub 发布与 develop 分支是不同快照。引擎代码采用 MIT，但官方文档说明 Oryx 素材用于产品时需要购买许可，不能假定素材也采用 MIT。", "enables_en": "Author game mechanics and compare agents under different observation types, maps and interaction rules.", "enables_zh": "编写游戏机制，并在不同观察类型、地图和交互规则下比较 Agent。", "resource_kind": "environment-framework", "runtime": "PyPI griddly 1.6.7; C++ engine/Python interface, GDY YAML tasks, native builds and a documented WebAssembly build path.", "evidence": [ { "url": "https://github.com/Bam4d/Griddly/blob/develop/README.md", "supports": "Documents GDY, observation modes, native platform prerequisites and WebAssembly builds." }, { "url": "https://github.com/Bam4d/Griddly/blob/develop/LICENSE", "supports": "Engine repository code is MIT licensed." }, { "url": "https://griddly.readthedocs.io/en/latest/about/artwork.html", "supports": "Official artwork page distinguishes Oryx product-use licensing from engine code." }, { "url": "https://arxiv.org/abs/2011.06363", "supports": "Original paper describes configurable games, observers and the C++ engine." }, { "url": "https://pypi.org/project/griddly/1.6.7/", "supports": "PyPI metadata observed at review lists griddly 1.6.7; distributions were not downloaded." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "Web denotes the documented WebAssembly build target, not a verified hosted benchmark; native platforms follow author build instructions." }, { "id": "rlcard", "name": "RLCard", "description_en": "A card-game environment toolkit exposing observations, legal actions and trajectories for reinforcement learning and search under imperfect information.", "description_zh": "为不完全信息条件下的强化学习与搜索提供观察、合法动作和轨迹的卡牌游戏环境工具包。", "content_type": "game", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/datamllab/rlcard", "repo_url": "https://github.com/datamllab/rlcard", "demo_url": null, "paper_url": "https://arxiv.org/abs/1910.04376", "limits_en": "Game-specific rules and action abstractions affect comparability. The default package contains environments; learning algorithms and graphical demos have separate dependencies/projects. PyPI lists 1.2.0 while the latest GitHub release is older. Repository code is MIT; no model or game interaction was run.", "limits_zh": "不同游戏的规则和动作抽象会影响可比性。默认包包含环境，学习算法和图形演示另有依赖或项目。PyPI 列为 1.2.0，最新 GitHub 发布则较旧。仓库代码采用 MIT，未运行模型或游戏交互。", "enables_en": "Train or compare card-game policies with explicit legal-action handling and reproducible seeded episodes.", "enables_zh": "通过明确的合法动作处理和固定种子的回合，训练或比较卡牌游戏策略。", "resource_kind": "environment-framework", "runtime": "PyPI rlcard 1.2.0; Python/NumPy/termcolor environments, with optional PyTorch training extras and separate GUI examples.", "evidence": [ { "url": "https://github.com/datamllab/rlcard#readme", "supports": "Documents card environments, legal-action/state interfaces, installation and separate training/GUI components." }, { "url": "https://github.com/datamllab/rlcard/blob/master/setup.py", "supports": "Lists core environment dependencies and optional training dependencies." }, { "url": "https://github.com/datamllab/rlcard/blob/master/LICENSE.md", "supports": "Repository code is MIT licensed." }, { "url": "https://arxiv.org/abs/1910.04376", "supports": "Original toolkit paper explains imperfect-information card environments and interfaces." }, { "url": "https://pypi.org/project/rlcard/1.2.0/", "supports": "PyPI metadata observed at review lists rlcard 1.2.0; distributions were not downloaded." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "No native OS or web target is inferred from the Python API or separate GUI project." }, { "id": "xland-minigrid", "name": "XLand-MiniGrid", "description_en": "A JAX grid-world suite for meta-reinforcement learning where composable rules and goals change the dynamics an agent must discover.", "description_zh": "用于元强化学习的 JAX 网格世界套件，通过组合规则和目标改变 Agent 需要探索的环境动力学。", "content_type": "game", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/dunnolab/xland-minigrid", "repo_url": "https://github.com/dunnolab/xland-minigrid", "demo_url": null, "paper_url": "https://arxiv.org/abs/2312.12044", "limits_en": "This is a from-scratch meta-learning suite inspired by XLand/MiniGrid, not a release of DeepMind's original XLand. First benchmark loading downloads cached ruleset data. PyPI 0.9.3 requires Python 3.9+ and appropriate JAX setup. Included MiniGrid ports overlap existing tasks and have limited maintenance. Code is Apache 2.0.", "limits_zh": "这是受 XLand/MiniGrid 启发、从头实现的元学习套件，并非 DeepMind 原 XLand 的发布。首次加载基准会下载并缓存规则集数据。PyPI 0.9.3 要求 Python 3.9+ 和相应 JAX 配置。附带的 MiniGrid 移植任务与已有任务重合，维护有限。代码采用 Apache 2.0。", "enables_en": "Train and evaluate adaptation to changing rulesets, using explicit train/test task distributions and optional visual observations.", "enables_zh": "通过明确的训练、测试任务分布和可选视觉观察，训练与评估对变化规则集的适应能力。", "resource_kind": "environment-and-benchmark", "runtime": "PyPI xminigrid 0.9.3; functional JAX environment API, ruleset generator and separately loaded benchmark data.", "evidence": [ { "url": "https://github.com/dunnolab/xland-minigrid#readme", "supports": "Documents from-scratch implementation, composable rules/goals, benchmark downloads and limited support for included MiniGrid ports." }, { "url": "https://github.com/dunnolab/xland-minigrid/blob/main/pyproject.toml", "supports": "Declares Python >=3.9 and JAX-related requirements." }, { "url": "https://github.com/dunnolab/xland-minigrid/blob/main/LICENSE", "supports": "Repository code is Apache 2.0 licensed." }, { "url": "https://arxiv.org/abs/2312.12044", "supports": "Original paper introduces the meta-RL environment suite and ruleset benchmarks." }, { "url": "https://pypi.org/project/xminigrid/0.9.3/", "supports": "PyPI metadata observed at review lists xminigrid 0.9.3, Python >=3.9; distributions were not downloaded." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed", "platform_note": "CPU/GPU/TPU support is not used to infer operating-system tags." }, { "id": "gui-odyssey", "name": "GUI Odyssey", "description_en": "An offline Android dataset of cross-app navigation demonstrations, combining screenshots and actions with instructions and semantic reasoning annotations.", "description_zh": "面向 Android 跨应用导航的离线示范数据集，提供截图、动作、指令及语义推理标注。", "content_type": "app", "platforms": [ "android" ], "uses": [ "training", "evaluation" ], "primary_url": "https://github.com/OpenGVLab/GUI-Odyssey", "repo_url": "https://github.com/OpenGVLab/GUI-Odyssey", "demo_url": null, "paper_url": "https://arxiv.org/abs/2406.08451", "enables_en": "Train action predictors and compare cross-app generalization using the released app, device, task and random splits.", "enables_zh": "训练动作预测模型，并利用已发布的应用、设备、任务及随机划分比较跨应用泛化能力。", "limits_en": "Recorded trajectories, not an interactive emulator. The authors recommend the July 2025 GUIOdyssey release over the original Hub mirror; specify the data version and split. The current dataset is public and labeled CC BY 4.0. A separate repository code license was not located.", "limits_zh": "提供记录的轨迹，不是交互式模拟器。作者建议使用 2025 年 7 月更新的 GUIOdyssey 数据，而非最初的 Hub 镜像；比较时需注明版本和划分。当前数据公开并标注 CC BY 4.0；未找到单独的仓库代码许可。", "resource_kind": "dataset-and-benchmark", "runtime": "Offline screenshot/action data with processing, fine-tuning and evaluation instructions; no device image is included in this listing.", "evidence": [ { "url": "https://github.com/OpenGVLab/GUI-Odyssey#readme", "supports": "Documents cross-app Android demonstrations, split choices and the recommended July 2025 dataset release." }, { "url": "https://huggingface.co/datasets/hflqf88888/GUIOdyssey", "supports": "Current author-linked public dataset, annotations and CC BY 4.0 declaration." }, { "url": "https://github.com/OpenGVLab/GUI-Odyssey/blob/master/Quickstart.md", "supports": "Describes data preparation, model fine-tuning and offline evaluation." } ], "implementation": "unknown", "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "gui-world", "name": "GUI-World", "description_en": "A GUI video dataset and offline benchmark for understanding interface content, changes and operation sequences across desktop, mobile, web and XR recordings.", "description_zh": "覆盖桌面、移动端、网页及 XR 录屏的 GUI 视频数据集与离线基准，评估界面内容、变化及操作序列理解。", "content_type": "app", "platforms": [ "web", "macos", "windows", "ios", "android" ], "uses": [ "training", "evaluation" ], "primary_url": "https://github.com/Dongping-Chen/GUI-World", "repo_url": "https://github.com/Dongping-Chen/GUI-World", "demo_url": null, "paper_url": "https://arxiv.org/abs/2406.10819", "enables_en": "Train on annotated GUI videos and evaluate temporal understanding through the released question-answering splits and inference/judging code.", "enables_zh": "使用带标注的 GUI 视频训练模型，并通过已发布的问答划分及推理、判分代码评估时序理解。", "limits_en": "Video question answering does not execute GUI actions or verify workflow completion. The dataset is public under CC BY 4.0; evaluation needs a chosen model and, for judged responses, a configured judge. GUI-Vid is the associated model, not a separate environment.", "limits_zh": "视频问答不会执行 GUI 动作，也不验证工作流完成情况。数据按 CC BY 4.0 公开；评测需配置被测模型，依赖模型判分的回答还需配置评审模型。GUI-Vid 是关联模型，不是另一个环境。", "resource_kind": "dataset-and-benchmark", "platform_note": "Platform tags describe source recordings, not native installation support. The source also covers XR, which has no platform tag in this catalog.", "evidence": [ { "url": "https://github.com/Dongping-Chen/GUI-World/blob/master/Readme.md", "supports": "Defines the dataset and evaluation code and identifies GUI-Vid as a fine-tuned video model." }, { "url": "https://huggingface.co/datasets/ONE-Lab/GUI-World", "supports": "Public video/annotation artifact with training and benchmark files; the card text declares CC BY 4.0." }, { "url": "https://github.com/Dongping-Chen/GUI-World/blob/master/evaluation/README.md", "supports": "Documents inference, supported providers, answer judging and offline evaluation outputs." }, { "url": "https://gui-world.github.io/", "supports": "Identifies web, macOS, Windows, Android, iOS and XR recording scenarios." } ], "implementation": "unknown", "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "androidcontrol", "name": "AndroidControl", "description_en": "Human Android task demonstrations with both overall goals and step instructions, plus screenshots, accessibility trees and structured actions.", "description_zh": "提供 Android 人类任务示范，包含整体目标、逐步指令、截图、无障碍树及结构化动作。", "content_type": "app", "platforms": [ "android" ], "uses": [ "training", "evaluation" ], "primary_url": "https://github.com/google-research/google-research/tree/master/android_control", "repo_url": "https://github.com/google-research/google-research/tree/master/android_control", "demo_url": null, "paper_url": "https://arxiv.org/abs/2406.03679", "enables_en": "Study how training-data scale and instruction granularity affect action prediction and generalization to unfamiliar apps and tasks.", "enables_zh": "研究训练数据规模及指令粒度如何影响动作预测，以及对未见应用和任务的泛化。", "limits_en": "Public GZIP TFRecords and split files provide offline demonstrations, not a runnable device environment. Accessibility-tree parsing uses android_env protobuf definitions. The release README carries an Apache-2.0 notice for that file; a separate license statement for the hosted dataset was not located.", "limits_zh": "公开的 GZIP TFRecord 及划分文件提供离线示范，不是可运行的设备环境。解析无障碍树需使用 android_env 的 protobuf 定义。发布说明对该文件注明 Apache-2.0；未找到托管数据集的单独许可声明。", "resource_kind": "dataset-and-benchmark", "runtime": "TensorFlow TFRecord reader plus Android accessibility protobuf definitions; public bucket objects were inspected without downloading the full dataset.", "evidence": [ { "url": "https://github.com/google-research/google-research/blob/master/android_control/README.md", "supports": "Specifies dataset fields, action space, parsing requirements, data location and the file-level Apache-2.0 notice." }, { "url": "https://storage.googleapis.com/storage/v1/b/gresearch/o?prefix=android_control%2F&maxResults=50", "supports": "Public bucket metadata lists the TFRecord shards and released split files without authentication." }, { "url": "https://research.google/pubs/on-the-effects-of-data-scale-on-ui-control-agents/", "supports": "Primary author publication page links the dataset's training-scale and generalization study." } ], "implementation": "unknown", "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "guiact", "name": "GUIAct (GUICourse)", "description_en": "The released navigation component of GUICourse: screenshot/action data for single-step and multi-step website tasks and Android interactions.", "description_zh": "GUICourse 已发布的导航数据部分，包含单步、多步网页任务及 Android 交互的截图与动作记录。", "content_type": "app", "platforms": [ "web", "android" ], "uses": [ "training", "evaluation" ], "primary_url": "https://github.com/RUCBM/GUICourse", "repo_url": "https://github.com/RUCBM/GUICourse", "demo_url": null, "paper_url": "https://arxiv.org/abs/2406.11317", "enables_en": "Train visual GUI action predictors and evaluate them on the published smartphone, web-single and web-multi train/test splits.", "enables_zh": "训练视觉 GUI 动作预测模型，并使用已发布的 smartphone、web-single 和 web-multi 训练／测试划分评估。", "limits_en": "Static JSON/Parquet records with processing and evaluation code; no packaged live environment. The repository declares code MIT and datasets CC BY 4.0, while GUIAct's public Hub card says Apache-2.0. Data licensing is inconsistent. The separate GUIEnv-global pretraining release remains unchecked in the project checklist.", "limits_zh": "提供静态 JSON／Parquet 记录及处理、评测代码，未封装实时环境。仓库声明代码采用 MIT、数据采用 CC BY 4.0，但 GUIAct 的公开 Hub 卡片标注 Apache-2.0，数据许可存在冲突。项目清单中独立的 GUIEnv-global 预训练数据仍未标记为已发布。", "resource_kind": "dataset-and-benchmark", "evidence": [ { "url": "https://github.com/RUCBM/GUICourse#readme", "supports": "Defines GUIAct's web/Android scope, release checklist, preprocessing/evaluation code and code/data license declarations." }, { "url": "https://huggingface.co/datasets/yiye2023/GUIAct", "supports": "Public smartphone, web-single and web-multi train/test JSON/Parquet files; dataset card declares Apache-2.0, conflicting with the repository." }, { "url": "https://arxiv.org/abs/2406.11317", "supports": "Original GUICourse paper distinguishes GUIAct navigation from the GUIEnv and GUIChat components." } ], "implementation": "unknown", "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "aitz", "name": "Android in the Zoo (AITZ)", "description_en": "An Android-in-the-Wild-derived dataset that adds screen descriptions and action-reasoning annotations to recorded screenshot/action sequences.", "description_zh": "基于 Android in the Wild 子集构建的数据集，为记录的截图与动作序列补充屏幕描述及动作推理标注。", "content_type": "app", "platforms": [ "android" ], "uses": [ "training", "evaluation" ], "primary_url": "https://github.com/SuperAutomateAgent/CoAT-Android", "repo_url": "https://github.com/SuperAutomateAgent/CoAT-Android", "demo_url": null, "paper_url": "https://arxiv.org/abs/2403.02713", "enables_en": "Train or evaluate mobile action prediction with the released Chain-of-Action-Thought annotations and evaluation scripts.", "enables_zh": "利用已发布的 Chain-of-Action-Thought 标注及评测脚本，训练或评估移动端动作预测。", "limits_en": "An enriched subset of AITW, not independent live-device tasks or an emulator. Screenshots, JSON samples and evaluation code are inspectable in the repository; the full archive is linked through Google Drive and Baidu and was not downloaded. The README displays an Apache-2.0 badge, but no standalone license or precise dataset licensing scope was located.", "limits_zh": "这是 AITW 的增强标注子集，不是独立的实时设备任务或模拟器。仓库可查看截图、JSON 样本及评测代码；完整档案通过 Google Drive 和百度网盘提供，本次未下载。README 展示 Apache-2.0 许可徽章，但未找到独立许可文件或明确的数据许可适用范围。", "resource_kind": "dataset-and-benchmark", "evidence": [ { "url": "https://github.com/SuperAutomateAgent/CoAT-Android#readme", "supports": "Documents semantic annotation fields, download locations and released action-prediction evaluation." }, { "url": "https://github.com/SuperAutomateAgent/CoAT-Android/tree/main/data-example", "supports": "Public screenshot and JSON examples make the annotation artifact inspectable without accessing the full external archive." }, { "url": "https://github.com/SuperAutomateAgent/CoAT-Android/blob/main/coat/evaluate.py", "supports": "Released offline action-prediction evaluation implementation." }, { "url": "https://arxiv.org/abs/2403.02713", "supports": "Primary paper explains AITW-derived data construction and Chain-of-Action-Thought annotations." } ], "implementation": "unknown", "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "amex", "name": "AMEX", "description_en": "An Android GUI dataset combining element grounding, screen and element functionality descriptions, and instruction-following action chains.", "description_zh": "结合元素定位、屏幕及元素功能描述、指令驱动动作链的 Android GUI 数据集。", "content_type": "app", "platforms": [ "android" ], "uses": [ "training", "evaluation" ], "primary_url": "https://huggingface.co/datasets/Yuxiang007/AMEX", "repo_url": null, "demo_url": null, "paper_url": "https://arxiv.org/abs/2407.17490", "enables_en": "Train mobile interface perception and action prediction from three complementary annotation levels, with public samples for inspecting their formats.", "enables_zh": "利用三个互补标注层级训练移动界面感知和动作预测，并通过公开样本查看数据格式。", "limits_en": "Offline screenshot and annotation archives, not a device runtime. The public dataset is CC BY 4.0; full screenshots use multipart ZIP archives. Its action-chain format differs from AndroidControl, so action records cannot be treated as directly interchangeable. No separate runnable benchmark harness is claimed here.", "limits_zh": "提供离线截图及标注档案，不包含设备运行环境。公开数据采用 CC BY 4.0；完整截图使用分卷 ZIP。动作链格式与 AndroidControl 不同，不能直接互换。本条目不将其描述为独立可运行的评测框架。", "resource_kind": "dataset-and-benchmark", "evidence": [ { "url": "https://huggingface.co/datasets/Yuxiang007/AMEX", "supports": "Author-maintained public release declares CC BY 4.0, describes annotation levels, archive assembly and differing AndroidControl action spaces." }, { "url": "https://huggingface.co/datasets/Yuxiang007/AMEX/tree/main", "supports": "Public sample annotations/screenshots and full element, instruction and multipart screenshot archives are present." }, { "url": "https://arxiv.org/abs/2407.17490", "supports": "Primary paper describes the mobile dataset's three annotation levels." } ], "implementation": "unknown", "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "osworld-human", "name": "OSWorld-Human", "description_en": "Human reference trajectories and efficiency scoring for OSWorld tasks, measuring how many agent actions a successful workflow requires.", "description_zh": "为 OSWorld 任务提供人类参考轨迹及效率评分，用于衡量 Agent 成功完成工作流所需的动作数量。", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "primary_url": "https://github.com/WukLab/osworld-human", "repo_url": "https://github.com/WukLab/osworld-human", "demo_url": null, "paper_url": "https://arxiv.org/abs/2506.16042", "enables_en": "Compare agent action efficiency against annotated human trajectories using the released weighted-efficiency scorer.", "enables_zh": "利用已发布的加权效率评分程序，将 Agent 的动作效率与人工标注轨迹比较。", "limits_en": "The authors explicitly say not to use these data for training because the trajectories expose OSWorld evaluation solutions. This adds reference data and a scorer, not new desktop images. Report the OSWorld task version, maximum steps and single-action versus grouped-action convention. No explicit repository license was located.", "limits_zh": "作者明确要求不要用于训练，因为轨迹包含 OSWorld 评测任务答案。本项目补充参考数据及评分程序，不提供新的桌面镜像。结果需注明 OSWorld 任务版本、最大步数及单动作／分组动作口径。未找到明确的仓库许可。", "resource_kind": "dataset-and-benchmark", "platform_note": "No OS tags are inferred from the upstream OSWorld environment; this release contains reference traces and scoring code.", "evidence": [ { "url": "https://github.com/WukLab/osworld-human#readme", "supports": "Defines human reference traces, evaluation-only restriction and single/grouped-action efficiency scores." }, { "url": "https://github.com/WukLab/osworld-human/blob/main/score.py", "supports": "Public scoring implementation consumes agent results and human reference information." }, { "url": "https://arxiv.org/abs/2506.16042", "supports": "Primary study distinguishes latency analysis from reference-trajectory efficiency benchmarking." } ], "implementation": "unknown", "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "ui-vision", "name": "UI-Vision", "description_en": "An offline desktop GUI benchmark with publicly released element-grounding and layout-grounding screenshots, annotations and evaluation code.", "description_zh": "离线桌面 GUI 基准，已公开元素定位、布局定位所需的截图、标注及评测代码。", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "primary_url": "https://github.com/uivision/UI-Vision", "repo_url": "https://github.com/uivision/UI-Vision", "demo_url": null, "paper_url": "https://arxiv.org/abs/2503.15661", "enables_en": "Evaluate target localization and spatial/layout understanding on screenshots from desktop software.", "enables_zh": "在桌面软件截图上评估目标定位及空间、布局理解能力。", "limits_en": "The public release covers grounding; the paper's action-prediction component still has a 'Coming soon' placeholder. This is static screenshot evaluation, not a desktop execution environment. The public dataset card declares MIT and evaluation use; native OS support is not established by the artifact.", "limits_zh": "公开版本覆盖定位任务；论文中的动作预测部分仍是“Coming soon”占位说明。这是静态截图评测，不是桌面执行环境。公开数据卡声明 MIT 及评测用途；不能据此认定其支持某个原生操作系统。", "resource_kind": "dataset-and-benchmark", "platform_note": "No OS tags are inferred from desktop screenshots or application names.", "evidence": [ { "url": "https://github.com/uivision/UI-Vision#readme", "supports": "Release announcement specifies element and layout grounding data and evaluation code." }, { "url": "https://huggingface.co/datasets/ServiceNow/ui-vision", "supports": "Public MIT-labeled dataset contains element/layout annotations and images; intended usage is evaluation and analysis." }, { "url": "https://github.com/uivision/UI-Vision/blob/main/eval/action_prediction/README.md", "supports": "Action-prediction evaluator remains a Coming soon placeholder." }, { "url": "https://github.com/uivision/UI-Vision/tree/main/eval/grounding", "supports": "Inspectable code for the released grounding evaluations." } ], "implementation": "unknown", "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "llamatouch", "name": "LlamaTouch", "description_en": "An Android task-execution and evaluation testbed that checks whether agent traces reach annotated essential application states.", "description_zh": "Android 任务执行与评测平台，通过检查 Agent 轨迹是否到达人工标注的关键应用状态来评估任务完成。", "content_type": "app", "platforms": [ "android" ], "uses": [ "evaluation" ], "primary_url": "https://github.com/LlamaTouch/LlamaTouch", "repo_url": "https://github.com/LlamaTouch/LlamaTouch", "demo_url": null, "paper_url": "https://arxiv.org/abs/2404.16054", "enables_en": "Run mobile agents through AgentEnv and evaluate recorded screenshots and view hierarchies with essential-state matching rather than exact action-sequence equality.", "enables_zh": "通过 AgentEnv 运行移动端 Agent，并依据记录的截图及界面层级匹配关键状态，无需动作序列逐项完全一致。", "limits_en": "Requires separate AgentEnv/Evaluator components, Android setup and app accounts/permissions. The documented setup uses Python 3.9 and a Pixel 6a API 31 emulator. Full traces and emulator packages are externally hosted; their downloads were not tested. No explicit project license was located; real app versions and login state affect reproduction.", "limits_zh": "需配置独立的 AgentEnv／Evaluator 组件、Android 环境及应用账号和权限。文档使用 Python 3.9 与 Pixel 6a API 31 模拟器。完整轨迹及模拟器包由外部网盘托管，本次未测试下载。未找到明确的项目许可；真实应用版本及登录状态会影响复现。", "resource_kind": "environment-and-benchmark", "runtime": "Android device or emulator accessed through AgentEnv; essential-state scoring is implemented in the separate Evaluator repository.", "evidence": [ { "url": "https://github.com/LlamaTouch/LlamaTouch#readme", "supports": "Defines task metadata, AITW-derived and newly collected tasks, essential-state annotations and component boundaries." }, { "url": "https://github.com/LlamaTouch/AgentEnv#readme", "supports": "Public device/emulator bridge; documents Python 3.9, Pixel 6a API 31 and account/permission setup." }, { "url": "https://github.com/LlamaTouch/Evaluator#readme", "supports": "Public essential-state evaluator with task-trace inputs and alternative action-matching baselines." }, { "url": "https://github.com/LlamaTouch/LlamaTouch/blob/main/dataset/README.md", "supports": "Documents the externally hosted dataset archive and on-disk structure." } ], "implementation": "unknown", "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "openapps", "name": "OpenApps", "description_en": "A browser-hosted ecosystem of configurable simulated apps for training UI agents and measuring reliability across interface and content variations.", "description_zh": "在浏览器中运行的可配置模拟应用生态，用于训练 UI Agent，并衡量界面及内容变化下的可靠性。", "content_type": "website", "platforms": [ "web" ], "uses": [ "training", "evaluation" ], "primary_url": "https://github.com/facebookresearch/OpenApps", "repo_url": "https://github.com/facebookresearch/OpenApps", "demo_url": null, "paper_url": "https://arxiv.org/abs/2511.20766", "enables_en": "Vary simulated app appearance and state while scoring tasks against the underlying Python application state.", "enables_zh": "改变模拟应用的界面及状态，并依据底层 Python 应用状态对任务完成情况判分。", "limits_en": "Uses CC BY-NC 4.0. Requires Python 3.11–3.13 and Chromium for agent interaction. Desktop/phone styling remains browser simulation. The v1.0-paper tag preserves the paper-era configuration surface but includes later reward fixes; it is not an exact snapshot of the original runs.", "limits_zh": "采用 CC BY-NC 4.0。需 Python 3.11–3.13；Agent 交互还需 Chromium。桌面或手机外观均为浏览器模拟。v1.0-paper 标签保留论文时期的配置接口，但包含后续奖励修正，并非原始实验的完整快照。", "resource_kind": "environment-framework", "runtime": "Python/FastHTML web apps, configurable tasks and BrowserGym interaction; agent credentials depend on the selected model.", "platform_note": "Only web is tagged. Phone/tablet layouts and macOS/Windows-style chrome are not native operating-system environments.", "evidence": [ { "url": "https://github.com/facebookresearch/OpenApps#readme", "supports": "Documents browser-hosted apps, state-based rewards, training/evaluation, BrowserGym and paper-tag limitations." }, { "url": "https://github.com/facebookresearch/OpenApps/blob/main/pyproject.toml", "supports": "Current package requires Python >=3.11,<3.14." }, { "url": "https://github.com/facebookresearch/OpenApps/blob/main/LICENSE", "supports": "Project license is Creative Commons Attribution-NonCommercial 4.0." }, { "url": "https://facebookresearch.github.io/OpenApps/", "supports": "Documents theme, layout, content and device-size configuration for simulated apps." } ], "implementation": "unknown", "reviewed_at": "2026-10-16", "verification": "source-reviewed", "license_en": "CC BY-NC 4.0; the repository license includes a noncommercial condition.", "license_zh": "CC BY-NC 4.0，仓库许可包含非商业使用条件。", "license_url": "https://github.com/facebookresearch/OpenApps/blob/main/LICENSE" }, { "id": "timewarp", "name": "TimeWarp", "description_en": "A web-agent benchmark with controlled Wiki, News and Shop environments rendered in multiple historical interface styles.", "description_zh": "提供 Wiki、News 和 Shop 受控网站环境的网页 Agent 基准，以多个历史时期的界面风格测试泛化。", "content_type": "website", "platforms": [ "web" ], "uses": [ "training", "evaluation" ], "primary_url": "https://github.com/sparklabutah/timewarp", "repo_url": "https://github.com/sparklabutah/timewarp", "demo_url": null, "paper_url": "https://arxiv.org/abs/2603.04949", "enables_en": "Compare agents across website versions and train with the project's plan-distillation and trajectory workflow.", "enables_zh": "跨网站版本比较 Agent，并使用项目提供的计划蒸馏与轨迹流程进行训练。", "limits_en": "Self-hosted historical-style variants, not a continually changing live web. Start the environment services and configure the agent; residual tasks require a model judge. Public data cards and README indicate MIT, but pyproject declares Apache-2.0 and no standalone license was found. Record UI version, task revision and judge configuration.", "limits_zh": "这是自行部署的历史风格变体，不是持续变化的真实互联网。需启动环境服务并配置 Agent，部分剩余任务需模型判分。公开数据卡及 README 标注 MIT，但 pyproject 声明 Apache-2.0，且未找到独立许可文件。结果需记录界面版本、任务修订及评审模型配置。", "resource_kind": "environment-and-benchmark", "runtime": "Documented Conda/Python 3.11 setup, local Wiki/News/Shop services and BrowserGym integration; scoring combines rule checks with model judging for designated tasks.", "evidence": [ { "url": "https://github.com/sparklabutah/timewarp#readme", "supports": "Defines controlled UI variants, local setup, BrowserGym, training traces and the residual llm_judge tasks despite a broader deterministic-scoring claim." }, { "url": "https://huggingface.co/datasets/sparklabutah/timewarp", "supports": "Public task dataset marked MIT." }, { "url": "https://huggingface.co/datasets/sparklabutah/timewarp-env-data", "supports": "Public environment assets marked MIT, including the Shop catalog." }, { "url": "https://github.com/sparklabutah/timewarp/blob/master/pyproject.toml", "supports": "Package metadata declares Apache-2.0, inconsistent with the README MIT badge." } ], "implementation": "unknown", "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "knows", "name": "KNOWS", "description_en": "A browser-agent benchmark for researching sources and producing Google Docs, Sheets and Slides artifacts through multi-step workflows.", "description_zh": "面向网页 Agent 的基准，要求通过多步工作流研究资料并生成 Google Docs、Sheets 和 Slides 成果。", "content_type": "website", "platforms": [ "web" ], "uses": [ "evaluation" ], "primary_url": "https://github.com/alexgill321/KNOWS-benchmark", "repo_url": "https://github.com/alexgill321/KNOWS-benchmark", "demo_url": null, "paper_url": "https://arxiv.org/abs/2609.30604", "enables_en": "Evaluate research-to-artifact workflows with task-specific structure, content, layout and browsing-trace checks plus model judges.", "enables_zh": "结合任务专属的结构、内容、布局及浏览轨迹检查和模型判分，评估从资料研究到成果制作的流程。", "limits_en": "Uses real Google Workspace, requiring accounts, enabled APIs and evaluator access to files; some tasks need fresh write targets. Code/data are Apache-2.0. The public Hub dataset contains prompts and rubrics, while scoring needs repository evaluators and release assets. Configure the judge and report the release tag; external pages can change. The Hub card describes v1.1.0, while the current GitHub release is v1.2.0; do not mix prompt and evaluator/gold-data versions. Official evaluation rules prohibit evaluated agents from accessing evaluator code, checkpoints.md or reference data.", "limits_zh": "使用真实 Google Workspace，需账号、已启用的 API 及评测方对文件的访问权限；部分任务需新建写入目标。代码和数据采用 Apache-2.0。公开 Hub 数据只含提示及评分细则，判分需仓库中的评测程序和发布资产。需配置评审模型、注明版本标签；外部网页可能变化。 Hub 数据卡对应 v1.1.0，当前 GitHub 发布为 v1.2.0，提示与评分器／标准资料版本不能混用。官方评测规则禁止被评测 Agent 访问评分器代码、checkpoints.md 或参考资料。", "resource_kind": "benchmark-suite", "runtime": "Python >=3.10 package or standalone evaluators; Google Workspace API credentials and Gemini API or supported local judges; browser harnesses use Chromium.", "evidence": [ { "url": "https://github.com/alexgill321/KNOWS-benchmark#readme", "supports": "Defines real Workspace artifact tasks, credentials/provisioning, hybrid scoring, BrowserGym package and versioned releases." }, { "url": "https://github.com/alexgill321/KNOWS-benchmark/blob/main/RUNNING_STANDALONE.md", "supports": "Describes artifact provisioning, history capture and evaluation outside the reference harness." }, { "url": "https://huggingface.co/datasets/utahnlp/knows-benchmark", "supports": "Public Apache-2.0 task prompts/rubrics; these data do not replace the executable evaluators." }, { "url": "https://github.com/alexgill321/KNOWS-benchmark/blob/main/LICENSE", "supports": "Repository license is Apache-2.0." }, { "url": "https://github.com/alexgill321/KNOWS-benchmark#readme", "supports": "Official evaluation rules restrict agent access to evaluators, checkpoints.md and reference data; public release/version evidence must be kept aligned with prompts and gold assets." }, { "url": "https://github.com/alexgill321/KNOWS-benchmark/releases/tag/v1.2.0", "supports": "Public v1.2.0 release distributes matching evaluator gold data; it must not be silently mixed with another prompt release." }, { "url": "https://github.com/alexgill321/KNOWS-benchmark/blob/main/ASSETS.md", "supports": "Documents anyone-with-link write access on provisioned targets and OAuth requirement for the slides_17 instance 1 copy." } ], "implementation": "unknown", "reviewed_at": "2026-10-16", "verification": "source-reviewed", "runtime_note_en": "The reference provisioner creates fresh Drive targets with anyone-with-link write access. The slides_17 instance 1 presentation copy requires OAuth; folder-only provisioning can use either supported credential type.", "runtime_note_zh": "参考配置脚本新建的 Drive 目标允许持链接者编辑。slides_17 第 1 个实例复制演示文稿时需要 OAuth；仅创建文件夹可使用两种受支持凭据中的任一种。" }, { "id": "osworld-2", "name": "OSWorld 2.0", "description_en": "A distinct 108-task computer-use benchmark for long-horizon workflows spanning Ubuntu desktop applications and web services.", "description_zh": "包含 108 项独立任务的计算机操作基准，覆盖 Ubuntu 桌面应用与网页服务中的长流程工作。", "content_type": "app", "platforms": [ "linux", "web" ], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/xlang-ai/OSWorld-V2", "repo_url": "https://github.com/xlang-ai/OSWorld-V2", "demo_url": null, "paper_url": "https://arxiv.org/abs/2606.29537", "enables_en": "Evaluate workflows requiring persistent context, cross-source information and coordinated actions across multiple applications.", "enables_zh": "评估需要持续跟踪上下文、综合多个信息源及协调跨应用动作的工作流。", "limits_en": "Public Apache-2.0 harness; task code is auto-gated and labeled Apache-2.0. Complete assets are separately gated with no license declaration in their public card. The current osworld-v2.1 release needs Python >=3.12, an Ubuntu VM, matching self-hosted websites and some task-specific services. Pin all components; its tasks and scores differ from original OSWorld/Verified. The release manifest reports a Task029 setup timeout and incomplete Docker boot verification.", "limits_zh": "公开框架采用 Apache-2.0；任务代码需通过自动批准的访问门槛，标注 Apache-2.0。完整资产另设访问门槛，公开卡片未声明许可。当前 osworld-v2.1 需 Python >=3.12、Ubuntu 虚拟机、配套自建网站及部分任务专属服务。需固定所有组件版本；任务及成绩不能与原始 OSWorld／Verified 混用。 该发布清单记录 Task029 初始化超时，Docker 启动验证未完成。", "resource_kind": "environment-and-benchmark", "runtime": "The current manifest pins Ubuntu images for AWS and Docker, plus task/assets/website versions. It reports a Task029 setup timeout and incomplete Docker boot verification. GitLab-backed tasks need a private GitLab service.", "platform_note": "Linux describes the pinned Ubuntu guests; web describes tasks using mocked website services. Legacy provider code and host compatibility do not establish Windows or macOS task coverage.", "evidence": [ { "url": "https://github.com/xlang-ai/OSWorld-V2#readme", "supports": "Released harness, Python >=3.12, Ubuntu provider images, self-hosted services and separate gates for task code and complete assets." }, { "url": "https://github.com/xlang-ai/OSWorld-V2/blob/main/desktop_env/desktop_env.py", "supports": "Public environment control and task-evaluation dispatch implementation, not a paper-only placeholder." }, { "url": "https://github.com/xlang-ai/OSWorld-V2/blob/main/benchmark_releases/osworld-v2.1.json", "supports": "Active 108-task release pins Ubuntu images and component versions; records Task029 timeout and incomplete Docker verification." }, { "url": "https://github.com/xlang-ai/OSWorld-V2/blob/main/LICENSE", "supports": "Public harness uses Apache-2.0." }, { "url": "https://huggingface.co/datasets/xlangai/osworld_v2_tasks", "supports": "Public dataset metadata declares automatic gating and Apache-2.0; gated task files were not accessed." }, { "url": "https://huggingface.co/datasets/xlangai/osworld_v2_assets_gated", "supports": "Complete asset dataset is automatically gated; public metadata has no license card." }, { "url": "https://github.com/Task-Web/OSWorld-web/tree/osworld-v2.1", "supports": "Companion website deployment source is pinned by the current benchmark release." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "webgen-bench", "name": "WebGen-Bench", "description_en": "A benchmark for generating interactive, multi-file websites from instructions, with browser-agent functional checks and appearance evaluation.", "description_zh": "面向指令驱动、多文件交互网站生成的基准，提供浏览器 Agent 功能检查及外观评测。", "content_type": "website", "platforms": [ "web" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/mnluzimu/WebGen-Bench", "repo_url": "https://github.com/mnluzimu/WebGen-Bench", "demo_url": null, "paper_url": "https://arxiv.org/abs/2505.03733", "enables_en": "Compare generated web applications against requested interaction flows and train generation agents with the released instruction data.", "enables_zh": "按要求的交互流程比较生成的网站，并使用已发布的指令数据训练生成 Agent。", "limits_en": "Functional scoring uses a WebVoyager VLM judge, so outcomes can vary with the judge and configuration. Requires a generator, browser stack and model access. Public Hub data is MIT-labeled; no project-wide license for the original code was located. Harbor changes timeouts and uses work-in-progress reference apps; report the harness and judge.", "limits_zh": "功能判分使用 WebVoyager 视觉语言模型评审，结果会随评审模型及配置变化。需配置生成器、浏览器组件和模型访问。公开 Hub 数据标注 MIT；未找到原始代码的项目级许可。Harbor 调整了超时设置，参考应用仍在完善；结果需注明框架及评审模型。", "resource_kind": "benchmark-suite", "runtime": "Original scripts document Windows 11 and Linux workflows. The separate Harbor adapter packages Python 3.11, Node.js, Selenium and Chromium; its setup supports Linux/macOS. These are runner requirements, while generated artifacts target the web.", "platform_note": "Web describes the generated websites. Host operating systems are not native-app targets.", "evidence": [ { "url": "https://github.com/mnluzimu/WebGen-Bench#readme", "supports": "Author release of task data, generation integrations, WebVoyager functional evaluation and appearance scoring." }, { "url": "https://huggingface.co/datasets/luzimu/WebGen-Bench", "supports": "Public MIT-labeled instruction data with train and test splits." }, { "url": "https://github.com/mnluzimu/WebGen-Bench/blob/main/src/ui_test_bolt/compute_acc.py", "supports": "Inspectable aggregation of the judge's functional-test outcomes." }, { "url": "https://github.com/harbor-framework/harbor/blob/2de4605b9404eada7d341388291e6e469c90330b/adapters/webgen_bench/README.md", "supports": "Pinned adapter links the canonical dataset and documents VLM variability, timeout changes and incomplete reference-solution validation." }, { "url": "https://arxiv.org/abs/2505.03733", "supports": "Original website-generation benchmark and training-data study." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "android-bench", "name": "Android Bench", "description_en": "The public 1.0 task set for Android software engineering: agents edit Kotlin/Java repositories and have their patches checked by build and test harnesses.", "description_zh": "Android 软件工程的公开 1.0 任务集：Agent 修改 Kotlin／Java 代码库，由构建与测试框架检查补丁。", "content_type": "app", "platforms": [ "android" ], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://hub.harborframework.com/datasets/android-bench/android-bench/latest", "repo_url": "https://github.com/android-bench/android-bench", "demo_url": null, "paper_url": null, "enables_en": "Evaluate Android API, Gradle and application/library maintenance tasks using the public 100-task release.", "enables_zh": "使用公开的 100 项任务评估 Android API、Gradle 及应用／库维护能力。", "limits_en": "This entry covers public 1.0 tasks; Android Bench 2.0 has a separate private 30-task set. The original Apache-2.0 harness is archived and points to Harbor. Evaluated repositories retain their own licenses, including GPL/AGPL. Local Harbor evaluation requires x86_64/KVM and substantial resources; the archived README and package metadata disagree on Python minimums.", "limits_zh": "本条目覆盖公开 1.0 任务；Android Bench 2.0 另有私有的 30 项任务。原始 Apache-2.0 框架已归档并转向 Harbor；被测仓库保留各自许可，包括 GPL／AGPL。本地 Harbor 评测需要 x86_64／KVM 及较多资源；归档 README 与包元数据的最低 Python 版本不一致。", "resource_kind": "benchmark-suite", "runtime": "The current public Harbor README lists 8 CPU cores, 32 GB RAM and 120 GB free disk with KVM for Android emulators. The archived harness README says Python 3.14+, while pyproject.toml says >=3.11; neither is presented as a verified current Harbor requirement.", "platform_note": "Android is the target development platform; this is code-editing evaluation, not a mobile GUI navigation benchmark. KVM/Linux host requirements do not add a native Linux app tag.", "evidence": [ { "url": "https://developer.android.com/bench", "supports": "The official site explicitly labels its Harbor link as the 1.0 dataset." }, { "url": "https://hub.harborframework.com/datasets/android-bench/android-bench/latest", "supports": "Public 100-task package and documented x86_64/KVM, CPU, RAM and storage requirements." }, { "url": "https://github.com/android-bench/android-bench#readme", "supports": "Archived harness, Harbor migration, patch/test workflow and documented Python prerequisite." }, { "url": "https://github.com/android-bench/android-bench/blob/main/pyproject.toml", "supports": "Package metadata allows Python >=3.11, differing from the README." }, { "url": "https://github.com/android-bench/android-bench/blob/main/LICENSE", "supports": "Harness license is Apache-2.0." }, { "url": "https://github.com/android-bench/android-bench/blob/main/docs/tech_report.md", "supports": "Original dated 100-task report; its repository-source analysis includes GPL/AGPL projects." }, { "url": "https://developer.android.com/bench/methodology/2", "supports": "Official 2.0 methodology specifies 30 long-horizon tasks and explicitly states that its dataset is private." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "swe-lancer", "name": "SWE-Lancer", "description_en": "A software-engineering benchmark whose public Diamond release combines application-code fixes with choosing among implementation proposals from real freelance work.", "description_zh": "软件工程基准，其公开 Diamond 版本结合真实自由职业项目中的应用代码修复及实现方案选择。", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/openai/frontier-evals/blob/main/project/swelancer/README.md", "repo_url": "https://github.com/openai/frontier-evals/tree/main/project/swelancer", "demo_url": null, "paper_url": "https://arxiv.org/abs/2502.12115", "enables_en": "Evaluate end-to-end code changes and engineering-manager proposal decisions as separate tracks within one benchmark family.", "enables_zh": "在同一基准族中分别评估端到端代码修改及工程管理者的方案选择。", "limits_en": "The public release is a subset of the paper's broader task collection. Diamond's 463 tasks comprise 198 coding and 265 manager tasks; combined and split packages must not be added together. Code is MIT, with underlying application terms separate. Valid original runs disable internet on Linux. Harbor changes prompts/test visibility and omits the monetary metric.", "limits_zh": "公开版本只是论文更大任务集合的子集。Diamond 的 463 项任务由 198 项编码任务和 265 项管理任务构成，不能将总包与子集相加。代码采用 MIT，底层应用另有条款。原始协议要求在 Linux 上禁用网络；Harbor 修改了提示及测试可见性，且不报告金额指标。", "resource_kind": "benchmark-suite", "runtime": "Python >=3.11, uv and the project's Docker/shared evaluation components. Manager tasks use the monolithic image; image builds and caches can be large. Model credentials are configuration-dependent.", "platform_note": "Targets repository changes and proposal decisions. Runner OS and the test browser are not treated as native-app deployment platforms.", "evidence": [ { "url": "https://github.com/openai/frontier-evals/blob/main/project/swelancer/README.md", "supports": "Original run protocol, IC/manager distinction, offline Linux requirement and removal of 39 internet-dependent IC tasks." }, { "url": "https://github.com/openai/frontier-evals/blob/main/project/swelancer/LICENSE.md", "supports": "SWE-Lancer project code is MIT licensed." }, { "url": "https://github.com/openai/frontier-evals/blob/main/project/swelancer/pyproject.toml", "supports": "Python >=3.11 and shared local evaluation dependencies." }, { "url": "https://github.com/harbor-framework/harbor/blob/2de4605b9404eada7d341388291e6e469c90330b/adapters/swelancer/README.md", "supports": "Pinned adapter reconciles 198 IC plus 265 manager tasks and documents prompt, test-visibility and metric differences from the original harness." }, { "url": "https://arxiv.org/abs/2502.12115", "supports": "Original freelance-task study distinguishes code execution, managerial decisions and the public Diamond split." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "swe-smith", "name": "SWE-smith", "description_en": "A toolkit that builds repository execution environments and synthesizes code-repair tasks by introducing changes that fail existing tests.", "description_zh": "通过构建代码库执行环境，并引入会使现有测试失败的修改来合成代码修复任务的工具包。", "content_type": "app", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/SWE-bench/SWE-smith", "repo_url": "https://github.com/SWE-bench/SWE-smith", "demo_url": null, "paper_url": "https://arxiv.org/abs/2504.21798", "enables_en": "Generate test-backed software-engineering training tasks, collect agent trajectories and evaluate repair attempts in repository-specific containers.", "enables_zh": "生成有测试依据的软件工程训练任务、收集 Agent 轨迹，并在各代码库的容器中评估修复尝试。", "limits_en": "Training-oriented tasks are not a held-out SWE-bench evaluation set. Pin the task revision and partition: published dataset sizes have changed, and Harbor's 100-task package is only a slice. Toolkit and Hub data declare MIT; source repositories retain their terms. Requires Docker on supported Ubuntu; the authors do not support Windows/macOS. Harbor's adapter reports some synthetic oracle failures.", "limits_zh": "面向训练的任务不是独立保留的 SWE-bench 评测集。需固定任务版本及划分：发布数据规模已有变化，Harbor 的 100 项任务包只是子集。工具包和 Hub 数据声明 MIT，源代码库保留各自条款。需在受支持的 Ubuntu 上使用 Docker；作者不支持 Windows／macOS。Harbor 适配器提示部分合成参考补丁可能失败。", "resource_kind": "environment-framework", "runtime": "Python >=3.10 per package metadata; Docker repository images and test dependencies. The authors document Ubuntu 22.04.4 LTS. Harbor's reviewed adapter emits Python-profile tasks and may install verifier dependencies at run time.", "platform_note": "Repository-code tasks are not tagged by the operating system of the runner.", "evidence": [ { "url": "https://github.com/SWE-bench/SWE-smith#readme", "supports": "Environment construction, task synthesis, test validation, trajectory resources and Ubuntu/Docker support boundaries." }, { "url": "https://github.com/SWE-bench/SWE-smith/blob/main/LICENSE", "supports": "Toolkit code uses MIT." }, { "url": "https://github.com/SWE-bench/SWE-smith/blob/main/pyproject.toml", "supports": "Package requires Python >=3.10." }, { "url": "https://huggingface.co/datasets/SWE-bench/SWE-smith", "supports": "Public MIT-labeled training-task data, including test lists and image references." }, { "url": "https://github.com/harbor-framework/harbor/blob/2de4605b9404eada7d341388291e6e469c90330b/adapters/swesmith/README.md", "supports": "Pinned adapter's default 100-task slice, Python-profile restriction and possible synthetic-oracle failures." }, { "url": "https://arxiv.org/abs/2504.21798", "supports": "Original work on synthesizing software-engineering training tasks and execution environments." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "unity-ml-agents", "name": "Unity ML-Agents", "description_en": "A Unity SDK and Python interface that turn game and simulation scenes into interactive environments for reinforcement learning, imitation learning and multi-agent research.", "description_zh": "通过 Unity SDK 和 Python 接口，将游戏与模拟场景连接为可交互环境，用于强化学习、模仿学习及多 Agent 研究。", "content_type": "game", "platforms": [ "windows", "macos", "linux" ], "uses": [ "training", "evaluation" ], "implementation": "unknown", "resource_kind": "environment-framework", "creator": "Unity Technologies and ML-Agents contributors", "primary_url": "https://github.com/Unity-Technologies/ml-agents", "repo_url": "https://github.com/Unity-Technologies/ml-agents", "demo_url": null, "paper_url": "https://arxiv.org/abs/1809.02627", "enables_en": "Train and compare policies in instrumented Unity scenes using vector or visual observations, continuous or discrete actions, demonstration data and cooperative or competitive agents.", "enables_zh": "在接入接口的 Unity 场景中，通过向量或视觉观测、连续或离散动作、演示数据及合作或竞争 Agent，训练和比较策略。", "limits_en": "An environment toolkit, not a fixed benchmark or game-generation system. Release 23 documents Unity 6000.0+, C# package 4.0.0 and Python packages 1.1.0 with Python 3.10.1–3.10.12; source and package versions must be pinned separately. The reviewed OpenEnv wrapper uses only the first agent and does not support multi-agent environments; its x86_64 Docker binaries do not run on Apple Silicon.", "limits_zh": "这是环境工具包，不是固定基准或游戏生成系统。Release 23 文档指定 Unity 6000.0+、C# 包 4.0.0、Python 包 1.1.0 及 Python 3.10.1–3.10.12；源码与发布包需分别固定版本。已核查的 OpenEnv 封装仅使用首个 Agent，不支持多 Agent 环境，其 x86_64 Docker 二进制不能在 Apple Silicon 上运行。", "runtime_note_en": "Authoring or modifying scenes needs the Unity Editor and matching ML-Agents package; Python controls the Editor or a platform-specific executable. The pinned OpenEnv wrapper can download example binaries and run a local server; graphics mode needs a display. Desktop tags describe documented environment runtimes, not coverage of every Unity export platform.", "runtime_note_zh": "创作或修改场景需要 Unity Editor 及匹配的 ML-Agents 包；Python 控制 Editor 或特定平台的可执行文件。固定版本的 OpenEnv 封装可下载示例二进制并启动本地服务，图形模式需要显示环境。桌面平台标签表示文档列出的环境运行平台，不代表所有 Unity 导出平台均已覆盖。", "license_en": "ML-Agents toolkit code is Apache-2.0, with separate third-party notices. Unity Editor software has its own terms; the toolkit license does not replace engine, dependency or external-asset terms.", "license_zh": "ML-Agents 工具包代码采用 Apache-2.0，并附有独立第三方声明。Unity Editor 有自己的软件条款；工具包许可不替代引擎、依赖或外部素材条款。", "license_url": "https://github.com/Unity-Technologies/ml-agents/blob/a2777719560e4676be99e2ea128c5eb1fbeb3dbb/LICENSE.md", "reviewed_version": { "source_commit": "a2777719560e4676be99e2ea128c5eb1fbeb3dbb", "source_ref": "release_23 / release_23_tag", "documented_unity_package": "4.0.0", "documented_python_package": "1.1.0", "source_python_version": "1.2.0.dev0", "openenv_wrapper_commit": "55347f62e22f83a618b49ee984e2f684b69d8fbd" }, "evidence": [ { "url": "https://github.com/Unity-Technologies/ml-agents/blob/a2777719560e4676be99e2ea128c5eb1fbeb3dbb/Readme.md", "supports": "Canonical SDK/Python toolkit, training methods, example scenes, release-component table and original reference paper." }, { "url": "https://github.com/Unity-Technologies/ml-agents/blob/a2777719560e4676be99e2ea128c5eb1fbeb3dbb/com.unity.ml-agents/Documentation~/Installation.md", "supports": "Unity 6000.0+, Python 3.10.1–3.10.12, package versus source installation, desktop virtual environments and Windows setup." }, { "url": "https://github.com/Unity-Technologies/ml-agents/blob/a2777719560e4676be99e2ea128c5eb1fbeb3dbb/com.unity.ml-agents/package.json", "supports": "Unity package 4.0.0 and Unity 6000.0 requirement at the reviewed commit." }, { "url": "https://github.com/Unity-Technologies/ml-agents/blob/a2777719560e4676be99e2ea128c5eb1fbeb3dbb/ml-agents/mlagents/trainers/__init__.py", "supports": "Source reports 1.2.0.dev0; this is preserved separately from the documented PyPI 1.1.0 package." }, { "url": "https://pypi.org/pypi/mlagents/1.1.0/json", "supports": "Published package version and Python upper/lower bounds." }, { "url": "https://github.com/Unity-Technologies/ml-agents/blob/a2777719560e4676be99e2ea128c5eb1fbeb3dbb/LICENSE.md", "supports": "Apache-2.0 toolkit license." }, { "url": "https://github.com/Unity-Technologies/ml-agents/blob/a2777719560e4676be99e2ea128c5eb1fbeb3dbb/Third%20Party%20Notices.md", "supports": "Separately attributed third-party components." }, { "url": "https://unity.com/legal/editor-terms-of-service/software", "supports": "Unity Editor has separate software terms; no claim that the engine is licensed by the toolkit Apache license." }, { "url": "https://github.com/huggingface/OpenEnv/blob/55347f62e22f83a618b49ee984e2f684b69d8fbd/envs/unity_env/README.md", "supports": "Exact unity_env wrapper relationship, supported example names, first-agent-only scope, native desktop binaries and Apple-Silicon Docker limitation." } ] }, { "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "textarena", "name": "TextArena", "description_en": "A collection of competitive and cooperative text-based games with a shared string-observation/string-action interface for training and evaluating language-model agents.", "description_zh": "提供竞争与合作类文字游戏，以统一的字符串观测与动作接口训练、评估语言模型 Agent。", "content_type": "game", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "resource_kind": "environment-and-benchmark", "creator": "Leon Guertler, Bobby Cheng and TextArena contributors", "primary_url": "https://github.com/TextArena/TextArena", "repo_url": "https://github.com/TextArena/TextArena", "demo_url": null, "paper_url": "https://arxiv.org/abs/2504.11442", "enables_en": "Run self-play or agent-versus-agent games to study sequential decisions, negotiation and social reasoning; choose a specific game, opponent and language configuration for comparison.", "enables_zh": "运行自我对弈或 Agent 对战，研究序列决策、协商和社会推理；比较时需选定具体游戏、对手及语言配置。", "limits_en": "Requires Python 3.10+ and game-specific dependencies or word data. Provider-backed example agents need credentials; local game logic does not itself supply a model. PyPI 0.7.4 and the reviewed main-branch metadata (0.7.3) differ, so pin the package or commit and game ID. Hosted ratings are separate from a fixed offline test; multilingual coverage and review quality vary by game and language.", "limits_zh": "需要 Python 3.10+，部分游戏另需依赖或词表。使用外部模型服务的示例 Agent 需要凭据，本地游戏逻辑本身不提供模型。PyPI 0.7.4 与已核查主分支的版本元数据 0.7.3 不一致，应固定包或提交及游戏 ID。托管评分不同于固定离线测试，多语言覆盖和核查程度因游戏、语言而异。", "runtime_note_en": "Use the Python package with an agent callable that accepts text and returns text. The OpenEnv textarena_env record is a configurable HTTP wrapper whose documented default is one-player Wordle-v0; do not treat that wrapper or its sample as an additional benchmark family. No native OS support claim is inferred from Python alone.", "runtime_note_zh": "通过 Python 包调用接收文本并返回文本的 Agent。OpenEnv 的 textarena_env 是可配置 HTTP 封装，文档默认使用单人 Wordle-v0；不能将封装或示例另计为独立基准族，也不因使用 Python 就推断原生操作系统支持。", "license_en": "Core repository code is MIT. Optional model integrations and word-data dependencies have separate terms; the wordgames extra documents wordfreq as Apache-2.0.", "license_zh": "核心仓库代码采用 MIT；可选模型接口及词表依赖另有条款，wordgames 扩展将 wordfreq 标注为 Apache-2.0。", "license_url": "https://github.com/TextArena/TextArena/blob/1e3160dea659971f120bf42708fafa8ea3001b81/LICENSE", "reviewed_version": { "source_commit": "1e3160dea659971f120bf42708fafa8ea3001b81", "source_metadata_version": "0.7.3", "pypi_version": "0.7.4", "openenv_wrapper_commit": "55347f62e22f83a618b49ee984e2f684b69d8fbd" }, "evidence": [ { "url": "https://github.com/TextArena/TextArena/blob/1e3160dea659971f120bf42708fafa8ea3001b81/README.md", "supports": "Game-interface scope, self-play, local examples, hosted play, original paper and tiered multilingual coverage." }, { "url": "https://github.com/TextArena/TextArena/blob/1e3160dea659971f120bf42708fafa8ea3001b81/pyproject.toml", "supports": "Python 3.10+, source metadata version, core/optional dependencies and wordgames extra." }, { "url": "https://github.com/TextArena/TextArena/blob/1e3160dea659971f120bf42708fafa8ea3001b81/LICENSE", "supports": "MIT license and named copyright holders." }, { "url": "https://pypi.org/pypi/textarena/0.7.4/json", "supports": "Published 0.7.4 package metadata is distinct from reviewed main source metadata." }, { "url": "https://github.com/huggingface/OpenEnv/blob/55347f62e22f83a618b49ee984e2f684b69d8fbd/envs/textarena_env/README.md", "supports": "Canonical project link, configurable wrapper, default Wordle-v0, default one player and optional NLTK downloads." }, { "url": "https://arxiv.org/abs/2504.11442", "supports": "Original TextArena paper; historical paper environment count is not presented as current coverage." } ] }, { "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "momaland", "name": "MOMAland", "description_en": "A multi-objective multi-agent environment suite whose PettingZoo-style interfaces return reward vectors, including board games, cooperative physics tasks and resource-allocation simulations.", "description_zh": "多目标、多 Agent 环境套件，采用类似 PettingZoo 的接口返回奖励向量，包含棋盘游戏、合作物理任务和资源分配模拟。", "content_type": "game", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "resource_kind": "environment-and-benchmark", "creator": "Farama Foundation and MOMAland contributors", "primary_url": "https://github.com/Farama-Foundation/MOMAland", "repo_url": "https://github.com/Farama-Foundation/MOMAland", "demo_url": null, "paper_url": "https://arxiv.org/abs/2407.16312", "enables_en": "Compare multi-agent policies under competing objectives and utility choices, including multi-objective Connect4, Breakthrough and Pistonball, using versioned environments and supplied learning baselines.", "enables_zh": "使用带版本的环境及配套学习基线，比较多 Agent 策略在相互竞争的目标与效用选择下的表现，例如多目标 Connect4、Breakthrough 和 Pistonball。", "limits_en": "Reward vectors need an explicit utility, scalarization or multi-objective evaluation protocol; scores are not interchangeable with scalar-reward PettingZoo results. PyPI 0.2.0 metadata permits Python 3.9+, while the reviewed main source requires 3.11+. Optional learning extras add JAX and other dependencies. LICENSE and the package license field say GPLv3, but an MIT classifier conflicts with them.", "limits_zh": "奖励向量需要明确的效用、标量化或多目标评估协议，成绩不能与 PettingZoo 标量奖励结果直接互换。PyPI 0.2.0 元数据允许 Python 3.9+，已核查主分支源码要求 3.11+。可选学习扩展还需 JAX 等依赖。LICENSE 及包的 license 字段写明 GPLv3，但同时存在冲突的 MIT 分类标签。", "runtime_note_en": "Python environment APIs and optional renderers; some environments adapt existing PettingZoo tasks and others come from multi-objective literature. Pin the library commit/package, versioned environment ID and objective configuration. Desktop OS support was not established by this review.", "runtime_note_zh": "提供 Python 环境 API 及可选渲染器；部分环境改编自 PettingZoo，其他来自多目标研究。需固定库提交或包、带版本的环境 ID 及目标配置。本次未确立特定桌面操作系统支持。", "license_en": "Repository LICENSE is GNU GPLv3 and pyproject license text agrees; the same pyproject also includes a contradictory MIT classifier. Preserve that inconsistency rather than describing the suite as MIT. Dependencies retain their own licenses.", "license_zh": "仓库 LICENSE 为 GNU GPLv3，pyproject 的许可文本与之相同；但同一文件又包含冲突的 MIT 分类标签。应保留此不一致信息，不能将套件概括为 MIT。依赖项各自保留其许可。", "license_url": "https://github.com/Farama-Foundation/MOMAland/blob/b81f2330df45b2c39ef56aa3a94e7539023d5f8c/LICENSE", "reviewed_version": { "source_commit": "b81f2330df45b2c39ef56aa3a94e7539023d5f8c", "source_metadata_version": "0.2.0", "source_requires_python": ">=3.11", "pypi_version": "0.2.0", "pypi_requires_python": ">=3.9" }, "evidence": [ { "url": "https://github.com/Farama-Foundation/MOMAland/blob/b81f2330df45b2c39ef56aa3a94e7539023d5f8c/README.md", "supports": "Vector rewards, PettingZoo-style API, optional dependencies, learning algorithms, environment versioning and current publisher citation." }, { "url": "https://momaland.farama.org/environments/all-envs/", "supports": "Official environment list and competing objectives, including Connect4, Breakthrough, Pistonball and adapted PettingZoo environments." }, { "url": "https://github.com/Farama-Foundation/MOMAland/blob/b81f2330df45b2c39ef56aa3a94e7539023d5f8c/pyproject.toml", "supports": "Main source Python 3.11+, optional JAX-based learning dependencies, GPLv3 license text and conflicting MIT classifier." }, { "url": "https://github.com/Farama-Foundation/MOMAland/blob/b81f2330df45b2c39ef56aa3a94e7539023d5f8c/LICENSE", "supports": "GPLv3 license text." }, { "url": "https://pypi.org/pypi/momaland/0.2.0/json", "supports": "Published 0.2.0 package metadata allows Python 3.9+, distinct from the reviewed source checkout." }, { "url": "https://momaland.farama.org/", "supports": "Original arXiv citation for the environment suite." }, { "url": "https://arxiv.org/abs/2407.16312", "supports": "Original multi-objective multi-agent benchmark paper and author metadata." } ] }, { "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "gpt5-markdown-to-slides", "name": "Markdown → Slides", "description_en": "An OpenAI-published generated browser tool that turns typed Markdown into a slide preview, using lines of three hyphens as slide breaks.", "description_zh": "OpenAI 发布的生成式浏览器工具，将输入的 Markdown 转为幻灯片预览，以独立一行的三个连字符分隔页面。", "content_type": "website", "platforms": [ "web" ], "uses": [ "showcase" ], "implementation": "ai-built", "resource_kind": "ai-generated-output", "creator": "OpenAI", "primary_url": "https://github.com/openai/gpt-5-coding-examples/tree/77f6322723703941f0b884d1a1cb77c60aad5187/apps/markdown-to-slides", "repo_url": "https://github.com/openai/gpt-5-coding-examples", "source_file_url": "https://github.com/openai/gpt-5-coding-examples/blob/77f6322723703941f0b884d1a1cb77c60aad5187/apps/markdown-to-slides/index.html", "demo_url": "https://gpt5-coding-examples.vercel.app/markdown-to-slides", "paper_url": null, "provenance_source": "https://github.com/openai/gpt-5-coding-examples/blob/77f6322723703941f0b884d1a1cb77c60aad5187/README.md", "provenance_note_en": "OpenAI describes this collection as application code generated from single GPT-5 or GPT-5.2 prompts without handwritten application code. The exact pinned manifest identifies this base example but does not name an individual model checkpoint. This is publisher-reported provenance, not an independent authorship audit.", "provenance_note_zh": "OpenAI 将该集合描述为通过单次 GPT-5 或 GPT-5.2 提示生成应用代码、未手写应用代码。固定清单明确标识这个基础示例，但未给出具体模型检查点。这属于发布者开发来源声明，并非独立作者身份审计。", "enables_en": "Inspect a generated document-to-preview workflow with Markdown editing, live slide rendering, previous/next and keyboard navigation, theme selection and browser fullscreen controls.", "enables_zh": "检查生成的文档转预览流程，包括 Markdown 编辑、实时页面渲染、前后翻页及键盘导航、主题选择和浏览器全屏控制。", "limits_en": "A small custom Markdown parser powers this preview/presentation demo. There is no built-in PPTX, PDF or HTML export. Only the theme is saved to localStorage; the document restarts from a sample. A separate browser record covers a two-slide synthetic document and previous/next navigation, not full Markdown compatibility. Remote images can need network access.", "limits_zh": "这个预览与演示示例使用简化的自定义 Markdown 解析器，没有内置 PPTX、PDF 或 HTML 导出。localStorage 只保存主题，文档会从示例重新初始化。独立浏览器记录覆盖两页合成文本及前后翻页，不代表完整 Markdown 兼容性；远程图片可能需要联网。", "runtime_note_en": "Single HTML/CSS/JavaScript artifact with a textarea and client-side rendering. The demo HTML matched the pinned source at source review. A later browser check rendered two typed slides and navigated between them; fullscreen, theme persistence, images and broader syntax were not tested.", "runtime_note_zh": "单文件 HTML/CSS/JavaScript 作品，提供文本输入框并在浏览器端渲染。源码核查时演示 HTML 与固定源码一致；后续浏览器检查渲染了两页输入文本并进行翻页，未测试全屏、主题持久化、图片或更多语法。", "license_en": "Repository application code is MIT. External images referenced by a deck have separate ownership and terms.", "license_zh": "仓库应用代码采用 MIT；幻灯片引用的外部图片另有权属及条款。", "license_url": "https://github.com/openai/gpt-5-coding-examples/blob/77f6322723703941f0b884d1a1cb77c60aad5187/LICENSE", "reviewed_version": { "source_commit": "77f6322723703941f0b884d1a1cb77c60aad5187", "manifest_path": "examples/markdown-to-slides.yaml", "application_blob": "10efecbe9a139a6efd0b5c87b94cad42eca35070", "html_sha256": "d96d39a8a07e11ef3212b80add16b7f5702f020381a0244d2717ed2ddb59ea64" }, "evidence": [ { "url": "https://github.com/openai/gpt-5-coding-examples/blob/77f6322723703941f0b884d1a1cb77c60aad5187/README.md", "supports": "First-party single-prompt generation claim, human-code scope and link to the hosted collection." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/77f6322723703941f0b884d1a1cb77c60aad5187/examples/markdown-to-slides.yaml", "supports": "Exact example ID, title and generating prompt; no explicit per-example model checkpoint field." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/77f6322723703941f0b884d1a1cb77c60aad5187/apps/markdown-to-slides/index.html", "supports": "Textarea input, slide-splitting/parser, navigation/fullscreen/theme logic, theme-only storage and absence of a file-export path." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/77f6322723703941f0b884d1a1cb77c60aad5187/LICENSE", "supports": "MIT license for repository code." }, { "url": "https://gpt5-coding-examples.vercel.app/markdown-to-slides", "supports": "Source-supported hosted route whose fetched HTML matched the pinned source. Later limited browser interactions are recorded separately in observations.json." } ] }, { "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "gpt5-csv-to-charts", "name": "CSV To Charts", "description_en": "An OpenAI-published generated browser tool that reads a CSV file and configures bar, line, pie or doughnut charts with column mapping, colors and PNG download logic.", "description_zh": "OpenAI 发布的生成式浏览器工具，读取 CSV 文件并配置柱状图、折线图、饼图或环形图，包含列映射、颜色设置及 PNG 下载逻辑。", "content_type": "website", "platforms": [ "web" ], "uses": [ "showcase" ], "implementation": "ai-built", "resource_kind": "ai-generated-output", "creator": "OpenAI", "primary_url": "https://github.com/openai/gpt-5-coding-examples/tree/77f6322723703941f0b884d1a1cb77c60aad5187/apps/csv-to-charts", "repo_url": "https://github.com/openai/gpt-5-coding-examples", "source_file_url": "https://github.com/openai/gpt-5-coding-examples/blob/77f6322723703941f0b884d1a1cb77c60aad5187/apps/csv-to-charts/index.html", "demo_url": "https://gpt5-coding-examples.vercel.app/csv-to-charts", "paper_url": null, "provenance_source": "https://github.com/openai/gpt-5-coding-examples/blob/77f6322723703941f0b884d1a1cb77c60aad5187/README.md", "provenance_note_en": "OpenAI attributes the application collection to single GPT-5 or GPT-5.2 prompts without handwritten application code; the exact example manifest has no model-checkpoint field. The generated application uses the separately authored Chart.js library, so the authorship claim does not extend to that dependency.", "provenance_note_zh": "OpenAI 将该应用集合归因于单次 GPT-5 或 GPT-5.2 提示生成、未手写应用代码；该示例清单未给出模型检查点。生成应用使用独立开发的 Chart.js，因此开发来源声明不包括这一依赖。", "enables_en": "Inspect a generated file-to-chart workflow: local FileReader input, header/delimiter controls, numeric-column selection, chart/color changes and a canvas-to-PNG download handler.", "enables_zh": "检查生成的文件转图表流程：本地 FileReader 读取、表头与分隔符控制、数值列选择、图表及颜色切换，以及画布转 PNG 下载处理。", "limits_en": "A separate browser observation covers the built-in sample, chart-type change and dataset visibility. File import and PNG output are unverified. Chart.js loads from an unversioned CDN. Source inspection shows empty strings become zero and commas are stripped; invalid numbers are skipped in pie/doughnut charts or become gaps in bar/line data. Analytical correctness and CSV compatibility were not established.", "limits_zh": "独立浏览器记录覆盖内置样例、图表类型切换及数据列显隐。文件导入和 PNG 输出未验证。Chart.js 从未固定版本的 CDN 加载。源码显示空字符串会转成零，逗号会被删除；无效数值在饼图／环形图中被跳过，在柱状图／折线图中变为缺口。未确立分析正确性及 CSV 兼容性。", "runtime_note_en": "Single HTML application with an embedded sample and file picker/dropzone, without paste-text input. Source inspection shows local FileReader parsing and a PNG data-URL handler. A later browser check used only the built-in sample; Save as image was clicked, but download completion was not verified.", "runtime_note_zh": "单文件 HTML 应用，内置样例并提供文件选择／拖放区域，没有粘贴文本输入界面。源码包含本地 FileReader 解析及 PNG 数据地址处理。后续浏览器检查只使用内置样例；已点击 Save as image，但未确认下载完成。", "license_en": "Repository application code is MIT; Chart.js is separately MIT-licensed. The application does not pin the Chart.js version served by the CDN.", "license_zh": "仓库应用代码采用 MIT，Chart.js 也独立采用 MIT；应用未固定 CDN 提供的 Chart.js 版本。", "license_url": "https://github.com/openai/gpt-5-coding-examples/blob/77f6322723703941f0b884d1a1cb77c60aad5187/LICENSE", "reviewed_version": { "source_commit": "77f6322723703941f0b884d1a1cb77c60aad5187", "manifest_path": "examples/csv-to-charts.yaml", "application_blob": "0df508248f9b0a13438b3476daf1f7e102c1d298", "html_sha256": "cdf87d597ac5e713b76ecee5df4ee208811a563baa59c1826f9c91fcf00ff8b0", "cdn_chartjs_version_observed": "4.5.1", "cdn_dependency_is_unpinned": true }, "evidence": [ { "url": "https://github.com/openai/gpt-5-coding-examples/blob/77f6322723703941f0b884d1a1cb77c60aad5187/README.md", "supports": "First-party application-code generation claim and hosted collection link." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/77f6322723703941f0b884d1a1cb77c60aad5187/examples/csv-to-charts.yaml", "supports": "Exact base-example ID and prompt for CSV chart generation, chart choices, colors and image saving." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/77f6322723703941f0b884d1a1cb77c60aad5187/apps/csv-to-charts/index.html", "supports": "FileReader input, CSV parsing, chart type and column controls, empty-string numeric coercion, unversioned Chart.js CDN and toBase64Image PNG download handler." }, { "url": "https://github.com/openai/gpt-5-coding-examples/blob/77f6322723703941f0b884d1a1cb77c60aad5187/LICENSE", "supports": "MIT license for repository code." }, { "url": "https://github.com/chartjs/Chart.js/blob/master/LICENSE.md", "supports": "Chart.js is a separately authored MIT-licensed dependency." }, { "url": "https://cdn.jsdelivr.net/npm/chart.js/package.json", "supports": "CDN metadata reported Chart.js 4.5.1 at review time; the application source does not pin it." }, { "url": "https://gpt5-coding-examples.vercel.app/csv-to-charts", "supports": "HTTP-fetched HTML matched the pinned source. Later bounded browser observations are recorded separately; PNG output was not verified." } ] }, { "id": "ade-bench", "name": "ADE-bench", "description_en": "A container-based benchmark for agents that repair and modify dbt projects, SQL transformations and warehouse outputs.", "description_zh": "基于容器的基准，评估 Agent 修复或修改 dbt 项目、SQL 转换及数据仓库结果的能力。", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "resource_kind": "environment-and-benchmark", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "primary_url": "https://github.com/dbt-labs/ade-bench", "repo_url": "https://github.com/dbt-labs/ade-bench", "paper_url": null, "enables_en": "Evaluate changes to data models and analytics pipelines using task-specific dbt tests and expected-table comparisons.", "enables_zh": "通过任务专用 dbt 测试和预期结果表比对，评估数据模型及分析流水线的修改。", "limits_en": "DuckDB tasks copy a local database into a task container. Snowflake tasks use a real account, create or replace trial databases/users/roles, and leave objects behind; upstream advises a separate test account. Database/project variants differ by task. The frozen Harbor package has 48 DuckDB+dbt tasks and does not represent every task in the current source.", "limits_zh": "DuckDB 任务会把本地数据库复制到任务容器。Snowflake 任务使用真实账号，创建或替换试验数据库、用户及角色，且会保留这些对象；上游建议使用独立测试账号。数据库及项目变体因任务而异。固定的 Harbor 包含 48 项 DuckDB+dbt 任务，不能代表当前源码中的全部任务。", "license_en": "Repository code is Apache-2.0. The database release and inspected metadata do not establish separate terms for every source dataset; do not assume all warehouse contents share the code license.", "license_zh": "仓库代码采用 Apache-2.0。数据库发布及已查阅元数据未确立每个原始数据集的独立条款，不能假定所有仓库内容均沿用代码许可。", "license_url": "https://github.com/dbt-labs/ade-bench/blob/c305600ebf17e7e7c4550bd401b6ddb530beb7c8/LICENSE", "runtime_note_en": "Python >=3.10, Docker/Compose, shared dbt projects and separately distributed DuckDB files. Model credentials depend on the selected agent; the bundled reference-solution agent can check task setup without a model. Source inspection found 60 ready task folders at the pinned revision; readiness is not independent runtime verification.", "runtime_note_zh": "需 Python >=3.10、Docker/Compose、共享 dbt 项目及另行分发的 DuckDB 文件。模型凭证取决于所选 Agent；内置参考解法 Agent 可在不调用模型时检查任务配置。固定版本的源码有 60 个标为 ready 的任务目录，该状态不代表已独立验证运行。", "evidence": [ { "url": "https://github.com/dbt-labs/ade-bench/blob/c305600ebf17e7e7c4550bd401b6ddb530beb7c8/README.md", "supports": "Canonical task lifecycle, dbt checks, Docker isolation, local DuckDB copies and real Snowflake object changes/persistence." }, { "url": "https://github.com/dbt-labs/ade-bench/blob/c305600ebf17e7e7c4550bd401b6ddb530beb7c8/tasks/airbnb001/task.yaml", "supports": "A ready task repairs a dbt macro migration and compares five output tables; database/project variants are explicit." }, { "url": "https://github.com/dbt-labs/ade-bench/blob/c305600ebf17e7e7c4550bd401b6ddb530beb7c8/ade_bench/setup/snowflake_setup.py", "supports": "Source issues DROP/CREATE DATABASE, ROLE and USER commands against configured Snowflake credentials." }, { "url": "https://github.com/dbt-labs/ade-bench/blob/c305600ebf17e7e7c4550bd401b6ddb530beb7c8/pyproject.toml", "supports": "Python >=3.10; package metadata version 1.0.0rc1." }, { "url": "https://github.com/dbt-labs/ade-bench/releases/tag/databases", "supports": "Public release lists shared DuckDB binaries; metadata inspected, binaries not downloaded." }, { "url": "https://github.com/dbt-labs/ade-bench/blob/c305600ebf17e7e7c4550bd401b6ddb530beb7c8/shared/databases/catalog.yaml", "supports": "The inspected database metadata catalog is empty, so it does not resolve per-database source terms." }, { "url": "https://github.com/laude-institute/harbor/blob/3e9f14a7c1033896cc600d27de69d6948bcb3410/adapters/adebench/README.md", "supports": "Harbor adapter describes 48 DuckDB+dbt tasks and excludes Snowflake/dbt-fusion variants." } ] }, { "id": "bixbench", "name": "BixBench", "description_en": "A bioinformatics benchmark with a released notebook execution environment for agents that inspect data, run Python/R/Bash analyses and interpret results.", "description_zh": "生物信息学基准，提供笔记本执行环境，让 Agent 检查数据、运行 Python／R／Bash 分析并解释结果。", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "resource_kind": "environment-and-benchmark", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "primary_url": "https://github.com/Future-House/BixBench", "repo_url": "https://github.com/Future-House/BixBench", "paper_url": "https://arxiv.org/abs/2503.00096", "enables_en": "Evaluate multi-step computational analyses with open answers or multiple-choice answers, separately from zero-shot question-answering baselines.", "enables_zh": "通过开放式或选择题答案评估多步计算分析，并与不执行分析的零样本问答基线分别比较。", "limits_en": "v1.5 revises and flattens the benchmark to 205 questions; v1.0 has 296 questions and is not score-equivalent. The public split is named train but includes benchmark answers and a no-training-corpus notice. Open answers can require model judging. Capsule counts differ across paper, README and current metadata. Harbor notebook/CLI variants alter tools, prompts and grading details.", "limits_zh": "v1.5 修订并展平为 205 道问题；v1.0 有 296 道问题，两者成绩不能直接等同。公开划分名为 train，但包含基准答案及不得进入训练语料的声明。开放式答案可能需要模型判分。论文、README 与当前元数据的胶囊数量不一致；Harbor 的笔记本／CLI 变体也改变了工具、提示及评分细节。", "license_en": "Repository code and the public dataset card declare Apache-2.0. Third-party capsule files were not unpacked or individually licensed in this review. The dataset contamination notice is an evaluation-integrity instruction, separate from the license declaration.", "license_zh": "仓库代码及公开数据卡声明 Apache-2.0。本次未解压并逐项核查胶囊中的第三方文件许可。数据中的防污染声明属于评测完整性要求，应与许可声明分别理解。", "license_url": "https://github.com/Future-House/BixBench/blob/49311180bdacb324c596f2e07596c126f2004008/LICENSE", "runtime_note_en": "Python >=3.12 with Aviary/fhda and a Jupyter kernel; upstream recommends its Docker image. Data capsules and model/judge access are separate requirements. Hub metadata is public and ungated despite the README login instruction. Pin dataset revision, execution mode, grader and replica settings.", "runtime_note_zh": "需 Python >=3.12、Aviary／fhda 及 Jupyter 内核；上游推荐其 Docker 镜像。数据胶囊和模型／评审服务需另行准备。虽然 README 有登录说明，当前 Hub 元数据及任务文件公开且无访问门槛。应固定数据版本、执行模式、评分器及重复运行设置。", "evidence": [ { "url": "https://github.com/Future-House/BixBench/blob/49311180bdacb324c596f2e07596c126f2004008/README.md", "supports": "Released notebook trajectory generation, zero-shot baseline distinction, Docker and model/grader setup; README states 60 capsules." }, { "url": "https://github.com/Future-House/BixBench/blob/49311180bdacb324c596f2e07596c126f2004008/bixbench/generate_trajectories.py", "supports": "Public implementation loads question records and capsules, creates DataAnalysisEnv instances and performs environment steps." }, { "url": "https://github.com/Future-House/BixBench/blob/49311180bdacb324c596f2e07596c126f2004008/bixbench/graders.py", "supports": "Grading includes string, range and model-based paths; open-ended range grading uses a model, so eval_mode names alone do not prove deterministic scoring." }, { "url": "https://github.com/Future-House/BixBench/blob/49311180bdacb324c596f2e07596c126f2004008/pyproject.toml", "supports": "Python >=3.12 and pinned fhda v1.5.0 dependency." }, { "url": "https://huggingface.co/datasets/futurehouse/BixBench/blob/f8cc3bdcc6357c88b8c3648306522b9c422dc95a/README.md", "supports": "Apache-2.0 declaration; September 2025 v1.5 revision note; original v1.0 retained; public split named train." }, { "url": "https://huggingface.co/datasets/futurehouse/BixBench/blob/f8cc3bdcc6357c88b8c3648306522b9c422dc95a/BixBench.jsonl", "supports": "205 public question rows, 59 unique capsule_uuid/data_folder values, reference answers and no-training-corpus notices; no separate held-out split established." }, { "url": "https://arxiv.org/html/2503.00096v3", "supports": "Paper v3 describes 205 questions and 61 scenarios, which differs from current file metadata." }, { "url": "https://github.com/laude-institute/harbor/blob/3e9f14a7c1033896cc600d27de69d6948bcb3410/adapters/bixbench/README.md", "supports": "Notebook and CLI adapter variants are one project family; tool access, prompts and judge wording differ." } ] }, { "id": "dabstep", "name": "DABstep", "description_en": "Multi-file data-analysis tasks combining payment records, JSON business data and documentation, with a public code-executing agent baseline and answer scorer.", "description_zh": "多文件数据分析任务，结合支付记录、JSON 业务数据及说明文档，并公开代码执行型 Agent 基线和答案评分器。", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "resource_kind": "dataset-and-benchmark", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "primary_url": "https://huggingface.co/datasets/adyen/DABstep", "repo_url": "https://huggingface.co/spaces/adyen/DABstep/tree/main", "paper_url": "https://arxiv.org/abs/2506.23719", "enables_en": "Evaluate data joins, aggregation, business-rule interpretation and precise result reporting across heterogeneous files; local scoring is available when reference answers are provided.", "enables_zh": "评估跨异构文件的数据关联、聚合、业务规则理解及精确结果报告；在具备参考答案时可本地判分。", "limits_en": "The public default file has 450 questions with empty answer fields; the 10 answered dev questions share six of those IDs. Scoring checks final answers, not notebook validity or GUI completion. Harbor reconstructs default labels from scored public submissions, so its scores are not established as equivalent to official hidden-answer evaluation. The current Space closes new manual validation requests while keeping unvalidated submissions open.", "limits_zh": "公开 default 文件有 450 道问题，答案字段为空；10 道附答案的 dev 题中有 6 个 ID 与其重叠。评分检查最终答案，不验证笔记本正确性或 GUI 任务完成。Harbor 从已评分的公开提交重建 default 标签，其成绩尚不能视作官方隐藏答案评测的等价结果。当前 Space 已关闭新的人工验证申请，但仍开放未验证提交。", "license_en": "Dataset and Space LICENSE files declare CC BY 4.0. Space setup.py instead lists an MIT classifier; package-code licensing metadata is inconsistent and should not be presented as uniformly resolved.", "license_zh": "数据集和 Space 的 LICENSE 文件声明 CC BY 4.0，但 Space 的 setup.py 标注 MIT 分类；软件包代码的许可元数据存在冲突，不能说已统一明确。", "license_url": "https://huggingface.co/datasets/adyen/DABstep/blob/523e6627415c6ebafa0954ce0df9cd93b6c0914d/LICENSE", "runtime_note_en": "The author baseline documents Python 3.10, pinned smolagents dependencies, a model endpoint and seven shared context files. It executes Python for analysis; source review did not verify sandbox isolation. The Space includes the scorer and leaderboard service, not an official locally complete hidden-answer test set.", "runtime_note_zh": "作者基线文档使用 Python 3.10、固定的 smolagents 依赖、模型端点及七个共享背景文件，并通过执行 Python 完成分析；本次源码核查未验证其沙箱隔离。Space 提供评分器和榜单服务，不等于官方完整隐藏答案测试集可在本地取得。", "evidence": [ { "url": "https://huggingface.co/datasets/adyen/DABstep/blob/523e6627415c6ebafa0954ce0df9cd93b6c0914d/README.md", "supports": "Task/submission/task-score configurations; tasks has default/dev splits. The README colloquially calls configurations splits." }, { "url": "https://huggingface.co/datasets/adyen/DABstep/blob/523e6627415c6ebafa0954ce0df9cd93b6c0914d/data/tasks/all.jsonl", "supports": "450 public question rows, 378 hard and 72 easy; answer fields are empty." }, { "url": "https://huggingface.co/datasets/adyen/DABstep/blob/523e6627415c6ebafa0954ce0df9cd93b6c0914d/data/tasks/dev.jsonl", "supports": "10 answered dev questions, with six task IDs also present in default." }, { "url": "https://huggingface.co/spaces/adyen/DABstep/blob/d4431c2e4a695cbe43c33aab2adaa304a37ae64a/baseline/README.md", "supports": "Public Python 3.10 baseline invocation for default/dev tasks." }, { "url": "https://huggingface.co/spaces/adyen/DABstep/blob/d4431c2e4a695cbe43c33aab2adaa304a37ae64a/baseline/utils.py", "supports": "CodeAgent setup and Python execution plus seven context-file downloads; source read only." }, { "url": "https://huggingface.co/spaces/adyen/DABstep/blob/d4431c2e4a695cbe43c33aab2adaa304a37ae64a/dabstep_benchmark/evaluation/scorer.py", "supports": "Deterministic answer comparison with numeric tolerances, normalized/fuzzy strings and list handling; no model judge in this scorer." }, { "url": "https://huggingface.co/spaces/adyen/DABstep/blob/d4431c2e4a695cbe43c33aab2adaa304a37ae64a/dabstep_benchmark/content.py", "supports": "Current distinction between closed manual validations and open unvalidated submissions; v2 is announced, not independently verified as released." }, { "url": "https://huggingface.co/spaces/adyen/DABstep/blob/d4431c2e4a695cbe43c33aab2adaa304a37ae64a/LICENSE", "supports": "Space root license declares CC BY 4.0." }, { "url": "https://huggingface.co/spaces/adyen/DABstep/blob/d4431c2e4a695cbe43c33aab2adaa304a37ae64a/setup.py", "supports": "Package classifier says MIT, conflicting with the root license declaration." }, { "url": "https://github.com/laude-institute/harbor/blob/3e9f14a7c1033896cc600d27de69d6948bcb3410/adapters/dabstep/run_adapter.py", "supports": "Adapter source reconstructs labels from public scored submissions and cleans the selected answers; no extraction was run." }, { "url": "https://arxiv.org/abs/2506.23719", "supports": "Original paper identifies financial data-analysis scope, hidden-answer evaluation and small public developer set." } ] }, { "id": "photocraft", "name": "PhotoCraft", "description_en": "An alpha Rust image editor with layered documents, masks, brushes, editable type, adjustments and PSD/PSB import and saving.", "description_zh": "处于 alpha 阶段的 Rust 图像编辑器，提供分层文档、蒙版、画笔、可编辑文字、调整及 PSD／PSB 读写。", "content_type": "app", "platforms": [ "windows", "macos", "linux", "web" ], "uses": [ "showcase" ], "implementation": "ai-assisted", "resource_kind": "ai-generated-output", "creator": "ArtCraft team and contributors; creator announcement by u/ai_art_is_art", "primary_url": "https://getartcraft.com/apps/photocraft", "repo_url": "https://github.com/storytold/photocraft", "source_file_url": "https://github.com/storytold/photocraft/blob/5f65579ecb44047c2a619036c18ff273202e1389/apps/photocraft/src/main.rs", "download_url": "https://github.com/storytold/photocraft/releases/tag/v0.9.0", "demo_url": null, "paper_url": null, "provenance_source": "https://www.reddit.com/r/GeminiAI/comments/1wwo52q/ive_created_open_source_100_rust_replacements_for/", "provenance_note_en": "Author-reported LLM assistance; human direction and separately authored components remain.", "provenance_note_zh": "作者报告使用 LLM 辅助开发；仍包含人工指导及独立开发的组件。", "enables_en": "Study an AI-assisted implementation of layered image editing and scripted image operations through a native interface, CLI and documented command channel.", "enables_zh": "研究 AI 辅助开发的分层图像编辑实现，以及通过原生界面、CLI 和文档化命令通道进行图像操作的方式。", "limits_en": "Early alpha. The maintainer says it is not ready to replace Photoshop in daily professional work; typography, plug-ins and other workflows remain incomplete. Application-level PSD saves are not byte-identical round trips, even when rendering matches. AVIF reading and JPEG XL are unavailable in the reviewed source. No editing or file-fidelity tests were run in this review.", "limits_zh": "早期 alpha。维护者明确表示尚不能替代 Photoshop 的日常专业工作，排版、插件及其他流程仍不完整。即使渲染相同，应用级 PSD 保存也不是逐字节一致的往返；已核查源码尚不能读取 AVIF 或 JPEG XL。本次未进行编辑或文件保真测试。", "runtime_note_en": "Inspected source commit and v0.9.0 release separately. Release assets include native Windows MSI/portable packages, a universal macOS DMG, Linux packages and a self-hosted WebAssembly archive. Windows x64 MSI and macOS DMG returned HTTP 200 to HEAD requests; files were not downloaded or executed. Reviewed application build guidance requires Rust 1.95+. No hosted interactive web demo was established.", "runtime_note_zh": "分别核查固定源码提交与 v0.9.0 发布。发布文件包含原生 Windows MSI／便携包、通用 macOS DMG、Linux 包及需自行托管的 WebAssembly 压缩包。Windows x64 MSI 与 macOS DMG 的 HEAD 请求返回 HTTP 200；未下载或执行文件。已核查的应用构建说明要求 Rust 1.95+，未确立公开托管的交互网页演示。", "license_en": "Application code is dual-licensed MIT OR Apache-2.0. Bundled assets and fonts retain their listed licenses; ArtCraft branding has separate trademark restrictions and is excluded from the code-license grant.", "license_zh": "应用代码采用 MIT 或 Apache-2.0 双重许可。随附素材与字体保留其列明的许可；ArtCraft 品牌有单独商标限制，不包含在代码许可授权中。", "license_url": "https://github.com/storytold/photocraft/blob/5f65579ecb44047c2a619036c18ff273202e1389/LICENSE-MIT", "reviewed_at": "2026-10-16", "verification": "source-reviewed", "reviewed_version": { "source_commit": "5f65579ecb44047c2a619036c18ff273202e1389", "source_version": "0.9.0", "observed_release": "v0.9.0", "source_and_release_not_runtime_compared": true }, "distribution_evidence": { "asset_listing_url": "https://github.com/storytold/photocraft/releases/expanded_assets/v0.9.0", "artifacts": { "windows": "https://github.com/storytold/photocraft/releases/download/v0.9.0/photocraft-0.9.0-windows-x64.msi", "macos": "https://github.com/storytold/photocraft/releases/download/v0.9.0/photocraft-0.9.0-macos-universal.dmg", "linux": "https://github.com/storytold/photocraft/releases/download/v0.9.0/photocraft-0.9.0-linux-x86_64.AppImage", "web": "https://github.com/storytold/photocraft/releases/download/v0.9.0/photocraft-web-0.9.0.zip" }, "head_checks": { "windows": { "artifact_url": "https://github.com/storytold/photocraft/releases/download/v0.9.0/photocraft-0.9.0-windows-x64.msi", "status": 200, "content_length": "69443584", "content_type": "application/octet-stream" }, "macos": { "artifact_url": "https://github.com/storytold/photocraft/releases/download/v0.9.0/photocraft-0.9.0-macos-universal.dmg", "status": 200, "content_length": "90386785", "content_type": "application/octet-stream" } }, "architecture_note_en": "Release filenames document Windows x64/x86/ARM64 and macOS Universal variants; only Windows x64 and macOS Universal asset availability received HEAD checks." }, "evidence": [ { "url": "https://www.reddit.com/r/GeminiAI/comments/1wwo52q/ive_created_open_source_100_rust_replacements_for/", "supports": "Creator-linked implementation provenance." }, { "url": "https://github.com/storytold/photocraft/blob/5f65579ecb44047c2a619036c18ff273202e1389/README.md", "supports": "Project-specific capabilities, build/distribution documentation, maturity and known limitations." }, { "url": "https://github.com/storytold/photocraft/blob/5f65579ecb44047c2a619036c18ff273202e1389/Cargo.toml", "supports": "Inspectable application workspace metadata, code version, Rust edition/build floor and dual code license." }, { "url": "https://github.com/storytold/photocraft/blob/5f65579ecb44047c2a619036c18ff273202e1389/apps/photocraft/src/main.rs", "supports": "Published native application entry point; this is an application-source repository, not a release-only listing." }, { "url": "https://github.com/storytold/photocraft/releases/tag/v0.9.0", "supports": "Specific public release and platform asset inventory." }, { "url": "https://github.com/storytold/photocraft/releases/download/v0.9.0/photocraft-0.9.0-windows-x64.msi", "supports": "Windows x64 installer availability verified by HTTP HEAD; execution was not tested." }, { "url": "https://github.com/storytold/photocraft/releases/download/v0.9.0/photocraft-0.9.0-macos-universal.dmg", "supports": "Universal macOS disk-image availability verified by HTTP HEAD; execution was not tested." }, { "url": "https://github.com/storytold/photocraft/blob/5f65579ecb44047c2a619036c18ff273202e1389/LICENSE-MIT", "supports": "MIT license option." }, { "url": "https://github.com/storytold/photocraft/blob/5f65579ecb44047c2a619036c18ff273202e1389/LICENSE-APACHE", "supports": "Apache-2.0 license option." } ] }, { "id": "vectorcraft", "name": "VectorCraft", "description_en": "An alpha Rust vector editor with paths, shapes, text, layers and native/SVG/PDF document workflows, distributed as desktop and WebAssembly builds.", "description_zh": "处于 alpha 阶段的 Rust 矢量编辑器，提供路径、形状、文字、图层及原生／SVG／PDF 文档流程，并发布桌面和 WebAssembly 构建。", "content_type": "app", "platforms": [ "windows", "macos", "linux", "web" ], "uses": [ "showcase" ], "implementation": "ai-assisted", "resource_kind": "ai-generated-output", "creator": "ArtCraft team and contributors; creator announcement by u/ai_art_is_art", "primary_url": "https://getartcraft.com/apps/vectorcraft", "repo_url": "https://github.com/storytold/vectorcraft", "source_file_url": "https://github.com/storytold/vectorcraft/blob/8bc65329286dc1b84aa392997bb205613c3e96cb/apps/vectorcraft/src/main.rs", "download_url": "https://github.com/storytold/vectorcraft/releases/tag/v0.12.0", "demo_url": null, "paper_url": null, "provenance_source": "https://www.reddit.com/r/GeminiAI/comments/1wwo52q/ive_created_open_source_100_rust_replacements_for/", "provenance_note_en": "Author-reported LLM assistance; human direction and separately authored components remain.", "provenance_note_zh": "作者报告使用 LLM 辅助开发；仍包含人工指导及独立开发的组件。", "enables_en": "Inspect vector drawing, path operations and document export implemented across a native editor and command interface.", "enables_zh": "检查原生编辑器及命令接口中的矢量绘制、路径运算和文档导出实现。", "limits_en": "Alpha. Reliable exchange of Illustrator editing data remains incomplete, Affinity files are read-only, and raster effects, 3D/material workflows and vertical CJK composition have gaps. The upstream readiness percentages are self-assessments and are not copied as validation. No native interaction or cross-application round trip was tested here.", "limits_zh": "Alpha 阶段。Illustrator 编辑数据的可靠交换仍未完成，Affinity 文件仅支持读取，栅格特效、3D／材质流程及竖排 CJK 排版仍有缺口。未把上游自行评估的成熟度百分比当作验证结果。本次未测试原生交互或跨应用往返。", "runtime_note_en": "Inspected source commit and v0.12.0 release separately. Release assets include native Windows MSI/portable packages, a universal macOS DMG, Linux packages and a self-hosted WebAssembly archive. Windows x64 MSI and macOS DMG returned HTTP 200 to HEAD requests; files were not downloaded or executed. Reviewed application build guidance requires Rust 1.95+. No hosted interactive web demo was established.", "runtime_note_zh": "分别核查固定源码提交与 v0.12.0 发布。发布文件包含原生 Windows MSI／便携包、通用 macOS DMG、Linux 包及需自行托管的 WebAssembly 压缩包。Windows x64 MSI 与 macOS DMG 的 HEAD 请求返回 HTTP 200；未下载或执行文件。已核查的应用构建说明要求 Rust 1.95+，未确立公开托管的交互网页演示。", "license_en": "Application code is dual-licensed MIT OR Apache-2.0. Bundled assets and fonts retain their listed licenses; ArtCraft branding has separate trademark restrictions and is excluded from the code-license grant.", "license_zh": "应用代码采用 MIT 或 Apache-2.0 双重许可。随附素材与字体保留其列明的许可；ArtCraft 品牌有单独商标限制，不包含在代码许可授权中。", "license_url": "https://github.com/storytold/vectorcraft/blob/8bc65329286dc1b84aa392997bb205613c3e96cb/LICENSE-MIT", "reviewed_at": "2026-10-16", "verification": "source-reviewed", "reviewed_version": { "source_commit": "8bc65329286dc1b84aa392997bb205613c3e96cb", "source_version": "0.12.0", "observed_release": "v0.12.0", "source_and_release_not_runtime_compared": true }, "distribution_evidence": { "asset_listing_url": "https://github.com/storytold/vectorcraft/releases/expanded_assets/v0.12.0", "artifacts": { "windows": "https://github.com/storytold/vectorcraft/releases/download/v0.12.0/vectorcraft-0.12.0-windows-x64.msi", "macos": "https://github.com/storytold/vectorcraft/releases/download/v0.12.0/vectorcraft-0.12.0-macos-universal.dmg", "linux": "https://github.com/storytold/vectorcraft/releases/download/v0.12.0/vectorcraft-0.12.0-linux-x86_64.AppImage", "web": "https://github.com/storytold/vectorcraft/releases/download/v0.12.0/vectorcraft-web-0.12.0.zip" }, "head_checks": { "windows": { "artifact_url": "https://github.com/storytold/vectorcraft/releases/download/v0.12.0/vectorcraft-0.12.0-windows-x64.msi", "status": 200, "content_length": "101388288", "content_type": "application/octet-stream" }, "macos": { "artifact_url": "https://github.com/storytold/vectorcraft/releases/download/v0.12.0/vectorcraft-0.12.0-macos-universal.dmg", "status": 200, "content_length": "136458785", "content_type": "application/octet-stream" } }, "architecture_note_en": "Release filenames document Windows x64/x86/ARM64 and macOS Universal variants; only Windows x64 and macOS Universal asset availability received HEAD checks." }, "evidence": [ { "url": "https://www.reddit.com/r/GeminiAI/comments/1wwo52q/ive_created_open_source_100_rust_replacements_for/", "supports": "Creator-linked implementation provenance." }, { "url": "https://github.com/storytold/vectorcraft/blob/8bc65329286dc1b84aa392997bb205613c3e96cb/README.md", "supports": "Project-specific capabilities, build/distribution documentation, maturity and known limitations." }, { "url": "https://github.com/storytold/vectorcraft/blob/8bc65329286dc1b84aa392997bb205613c3e96cb/Cargo.toml", "supports": "Inspectable application workspace metadata, code version, Rust edition/build floor and dual code license." }, { "url": "https://github.com/storytold/vectorcraft/blob/8bc65329286dc1b84aa392997bb205613c3e96cb/apps/vectorcraft/src/main.rs", "supports": "Published native application entry point; this is an application-source repository, not a release-only listing." }, { "url": "https://github.com/storytold/vectorcraft/releases/tag/v0.12.0", "supports": "Specific public release and platform asset inventory." }, { "url": "https://github.com/storytold/vectorcraft/releases/download/v0.12.0/vectorcraft-0.12.0-windows-x64.msi", "supports": "Windows x64 installer availability verified by HTTP HEAD; execution was not tested." }, { "url": "https://github.com/storytold/vectorcraft/releases/download/v0.12.0/vectorcraft-0.12.0-macos-universal.dmg", "supports": "Universal macOS disk-image availability verified by HTTP HEAD; execution was not tested." }, { "url": "https://github.com/storytold/vectorcraft/blob/8bc65329286dc1b84aa392997bb205613c3e96cb/LICENSE-MIT", "supports": "MIT license option." }, { "url": "https://github.com/storytold/vectorcraft/blob/8bc65329286dc1b84aa392997bb205613c3e96cb/LICENSE-APACHE", "supports": "Apache-2.0 license option." } ] }, { "id": "lightcraft", "name": "LightCraft", "description_en": "An alpha Rust photo library and RAW developer with catalog organization, nondestructive adjustments, masking and image-export workflows.", "description_zh": "处于 alpha 阶段的 Rust 照片库与 RAW 显影应用，包含照片整理、非破坏性调整、蒙版及图像导出流程。", "content_type": "app", "platforms": [ "windows", "macos", "linux", "web" ], "uses": [ "showcase" ], "implementation": "ai-assisted", "resource_kind": "ai-generated-output", "creator": "ArtCraft team and contributors; creator announcement by u/ai_art_is_art", "primary_url": "https://getartcraft.com/apps/lightcraft", "repo_url": "https://github.com/storytold/lightcraft", "source_file_url": "https://github.com/storytold/lightcraft/blob/f34c7a8a89e0d1bade0a6889185f40acecc84a9f/apps/lightcraft/src/main.rs", "download_url": "https://github.com/storytold/lightcraft/releases/tag/v0.8.0", "demo_url": null, "paper_url": null, "provenance_source": "https://www.reddit.com/r/GeminiAI/comments/1wwo52q/ive_created_open_source_100_rust_replacements_for/", "provenance_note_en": "Author-reported LLM assistance; human direction and separately authored components remain.", "provenance_note_zh": "作者报告使用 LLM 辅助开发；仍包含人工指导及独立开发的组件。", "enables_en": "Inspect a native photo-management and development pipeline, including configurable edits and automation through CLI/MCP entry points.", "enables_zh": "检查原生照片管理与显影流程，包括可配置编辑及通过 CLI／MCP 入口进行自动化。", "limits_en": "Alpha. Camera calibration is incomplete; unsupported RAW variants may fall back to embedded JPEG previews. Subject/sky selection currently uses classical heuristics, and on-screen HDR, video and some Lightroom Classic modules are missing. The README requires Rust 1.95 for workspace builds although root Cargo metadata still says 1.90. No camera-file fidelity was independently tested.", "limits_zh": "Alpha 阶段。相机色彩标定尚不完整，不支持的 RAW 变体可能退回内嵌 JPEG 预览。主体／天空选择目前使用传统启发式方法，屏幕 HDR、视频及部分 Lightroom Classic 模块尚缺失。README 要求工作区构建使用 Rust 1.95，根 Cargo 元数据仍写 1.90。本次未独立测试相机文件保真度。", "runtime_note_en": "Inspected source commit and v0.8.0 release separately. Release assets include native Windows MSI/portable packages, a universal macOS DMG, Linux packages and a self-hosted WebAssembly archive. Windows x64 MSI and macOS DMG returned HTTP 200 to HEAD requests; files were not downloaded or executed. Reviewed application build guidance requires Rust 1.95+. No hosted interactive web demo was established.", "runtime_note_zh": "分别核查固定源码提交与 v0.8.0 发布。发布文件包含原生 Windows MSI／便携包、通用 macOS DMG、Linux 包及需自行托管的 WebAssembly 压缩包。Windows x64 MSI 与 macOS DMG 的 HEAD 请求返回 HTTP 200；未下载或执行文件。已核查的应用构建说明要求 Rust 1.95+，未确立公开托管的交互网页演示。", "license_en": "Application code is dual-licensed MIT OR Apache-2.0. Bundled assets and fonts retain their listed licenses; ArtCraft branding has separate trademark restrictions and is excluded from the code-license grant.", "license_zh": "应用代码采用 MIT 或 Apache-2.0 双重许可。随附素材与字体保留其列明的许可；ArtCraft 品牌有单独商标限制，不包含在代码许可授权中。", "license_url": "https://github.com/storytold/lightcraft/blob/f34c7a8a89e0d1bade0a6889185f40acecc84a9f/LICENSE-MIT", "reviewed_at": "2026-10-16", "verification": "source-reviewed", "reviewed_version": { "source_commit": "f34c7a8a89e0d1bade0a6889185f40acecc84a9f", "source_version": "0.8.0", "observed_release": "v0.8.0", "source_and_release_not_runtime_compared": true }, "distribution_evidence": { "asset_listing_url": "https://github.com/storytold/lightcraft/releases/expanded_assets/v0.8.0", "artifacts": { "windows": "https://github.com/storytold/lightcraft/releases/download/v0.8.0/lightcraft-0.8.0-windows-x64.msi", "macos": "https://github.com/storytold/lightcraft/releases/download/v0.8.0/lightcraft-0.8.0-macos-universal.dmg", "linux": "https://github.com/storytold/lightcraft/releases/download/v0.8.0/lightcraft-0.8.0-linux-x86_64.AppImage", "web": "https://github.com/storytold/lightcraft/releases/download/v0.8.0/lightcraft-web-0.8.0.zip" }, "head_checks": { "windows": { "artifact_url": "https://github.com/storytold/lightcraft/releases/download/v0.8.0/lightcraft-0.8.0-windows-x64.msi", "status": 200, "content_length": "87281664", "content_type": "application/octet-stream" }, "macos": { "artifact_url": "https://github.com/storytold/lightcraft/releases/download/v0.8.0/lightcraft-0.8.0-macos-universal.dmg", "status": 200, "content_length": "107963600", "content_type": "application/octet-stream" } }, "architecture_note_en": "Release filenames document Windows x64/x86/ARM64 and macOS Universal variants; only Windows x64 and macOS Universal asset availability received HEAD checks." }, "evidence": [ { "url": "https://www.reddit.com/r/GeminiAI/comments/1wwo52q/ive_created_open_source_100_rust_replacements_for/", "supports": "Creator-linked implementation provenance." }, { "url": "https://github.com/storytold/lightcraft/blob/f34c7a8a89e0d1bade0a6889185f40acecc84a9f/README.md", "supports": "Project-specific capabilities, build/distribution documentation, maturity and known limitations." }, { "url": "https://github.com/storytold/lightcraft/blob/f34c7a8a89e0d1bade0a6889185f40acecc84a9f/Cargo.toml", "supports": "Inspectable application workspace metadata, code version, Rust edition/build floor and dual code license." }, { "url": "https://github.com/storytold/lightcraft/blob/f34c7a8a89e0d1bade0a6889185f40acecc84a9f/apps/lightcraft/src/main.rs", "supports": "Published native application entry point; this is an application-source repository, not a release-only listing." }, { "url": "https://github.com/storytold/lightcraft/releases/tag/v0.8.0", "supports": "Specific public release and platform asset inventory." }, { "url": "https://github.com/storytold/lightcraft/releases/download/v0.8.0/lightcraft-0.8.0-windows-x64.msi", "supports": "Windows x64 installer availability verified by HTTP HEAD; execution was not tested." }, { "url": "https://github.com/storytold/lightcraft/releases/download/v0.8.0/lightcraft-0.8.0-macos-universal.dmg", "supports": "Universal macOS disk-image availability verified by HTTP HEAD; execution was not tested." }, { "url": "https://github.com/storytold/lightcraft/blob/f34c7a8a89e0d1bade0a6889185f40acecc84a9f/LICENSE-MIT", "supports": "MIT license option." }, { "url": "https://github.com/storytold/lightcraft/blob/f34c7a8a89e0d1bade0a6889185f40acecc84a9f/LICENSE-APACHE", "supports": "Apache-2.0 license option." } ] }, { "id": "pdfcraft", "name": "PdfCraft", "description_en": "An alpha Rust PDF workbench for viewing, searching, annotations, forms and page organization, with native applications and command-line tools.", "description_zh": "处于 alpha 阶段的 Rust PDF 工作台，用于查看、搜索、批注、表单和页面整理，提供原生应用及命令行工具。", "content_type": "app", "platforms": [ "windows", "macos", "linux", "web" ], "uses": [ "showcase" ], "implementation": "ai-assisted", "resource_kind": "ai-generated-output", "creator": "ArtCraft team and contributors; creator announcement by u/ai_art_is_art", "primary_url": "https://getartcraft.com/apps/pdfcraft", "repo_url": "https://github.com/storytold/pdfcraft", "source_file_url": "https://github.com/storytold/pdfcraft/blob/924c4da4ac2879ac9107c6c82e20ac6be5570f0a/apps/pdfcraft/src/main.rs", "download_url": "https://github.com/storytold/pdfcraft/releases/tag/v0.8.0", "demo_url": null, "paper_url": null, "provenance_source": "https://www.reddit.com/r/GeminiAI/comments/1wwo52q/ive_created_open_source_100_rust_replacements_for/", "provenance_note_en": "Author-reported LLM assistance; human direction and separately authored components remain.", "provenance_note_zh": "作者报告使用 LLM 辅助开发；仍包含人工指导及独立开发的组件。", "enables_en": "Inspect document viewing and page-operation workflows, including combining, splitting and automation through a shared PDF engine.", "enables_zh": "检查文档查看及页面操作流程，包括合并、拆分和通过共享 PDF 引擎进行自动化。", "limits_en": "Alpha. Existing-text editing, non-Latin OCR, Office interchange and advanced signature/preflight workflows remain limited; Word export can lose layout and Office-to-PDF is absent. Rendering still uses the separately authored hayro crate. The maintainers report unresolved crashes/hangs from fuzzing. This review did not validate editing, signatures or document fidelity.", "limits_zh": "Alpha 阶段。已有文字编辑、非拉丁文字 OCR、Office 交换及高级签名／印前流程仍有限；Word 导出可能丢失布局，Office 转 PDF 尚未提供。渲染仍使用独立开发的 hayro 库。维护者记录了模糊测试发现的未解决崩溃或卡死。本次未验证编辑、签名或文档保真度。", "runtime_note_en": "Inspected source commit and v0.8.0 release separately. Release assets include native Windows MSI/portable packages, a universal macOS DMG, Linux packages and a self-hosted WebAssembly archive. Windows x64 MSI and macOS DMG returned HTTP 200 to HEAD requests; files were not downloaded or executed. Reviewed application build guidance requires Rust 1.95+. No hosted interactive web demo was established.", "runtime_note_zh": "分别核查固定源码提交与 v0.8.0 发布。发布文件包含原生 Windows MSI／便携包、通用 macOS DMG、Linux 包及需自行托管的 WebAssembly 压缩包。Windows x64 MSI 与 macOS DMG 的 HEAD 请求返回 HTTP 200；未下载或执行文件。已核查的应用构建说明要求 Rust 1.95+，未确立公开托管的交互网页演示。", "license_en": "Application code is dual-licensed MIT OR Apache-2.0. Bundled assets and fonts retain their listed licenses; ArtCraft branding has separate trademark restrictions and is excluded from the code-license grant.", "license_zh": "应用代码采用 MIT 或 Apache-2.0 双重许可。随附素材与字体保留其列明的许可；ArtCraft 品牌有单独商标限制，不包含在代码许可授权中。", "license_url": "https://github.com/storytold/pdfcraft/blob/924c4da4ac2879ac9107c6c82e20ac6be5570f0a/LICENSE-MIT", "reviewed_at": "2026-10-16", "verification": "source-reviewed", "reviewed_version": { "source_commit": "924c4da4ac2879ac9107c6c82e20ac6be5570f0a", "source_version": "0.8.0", "observed_release": "v0.8.0", "source_and_release_not_runtime_compared": true }, "distribution_evidence": { "asset_listing_url": "https://github.com/storytold/pdfcraft/releases/expanded_assets/v0.8.0", "artifacts": { "windows": "https://github.com/storytold/pdfcraft/releases/download/v0.8.0/pdfcraft-0.8.0-windows-x64.msi", "macos": "https://github.com/storytold/pdfcraft/releases/download/v0.8.0/pdfcraft-0.8.0-macos-universal.dmg", "linux": "https://github.com/storytold/pdfcraft/releases/download/v0.8.0/pdfcraft-0.8.0-linux-x86_64.AppImage", "web": "https://github.com/storytold/pdfcraft/releases/download/v0.8.0/pdfcraft-web-0.8.0.zip" }, "head_checks": { "windows": { "artifact_url": "https://github.com/storytold/pdfcraft/releases/download/v0.8.0/pdfcraft-0.8.0-windows-x64.msi", "status": 200, "content_length": "71266304", "content_type": "application/octet-stream" }, "macos": { "artifact_url": "https://github.com/storytold/pdfcraft/releases/download/v0.8.0/pdfcraft-0.8.0-macos-universal.dmg", "status": 200, "content_length": "93992515", "content_type": "application/octet-stream" } }, "architecture_note_en": "Release filenames document Windows x64/x86/ARM64 and macOS Universal variants; only Windows x64 and macOS Universal asset availability received HEAD checks." }, "evidence": [ { "url": "https://www.reddit.com/r/GeminiAI/comments/1wwo52q/ive_created_open_source_100_rust_replacements_for/", "supports": "Creator-linked implementation provenance." }, { "url": "https://github.com/storytold/pdfcraft/blob/924c4da4ac2879ac9107c6c82e20ac6be5570f0a/README.md", "supports": "Project-specific capabilities, build/distribution documentation, maturity and known limitations." }, { "url": "https://github.com/storytold/pdfcraft/blob/924c4da4ac2879ac9107c6c82e20ac6be5570f0a/Cargo.toml", "supports": "Inspectable application workspace metadata, code version, Rust edition/build floor and dual code license." }, { "url": "https://github.com/storytold/pdfcraft/blob/924c4da4ac2879ac9107c6c82e20ac6be5570f0a/apps/pdfcraft/src/main.rs", "supports": "Published native application entry point; this is an application-source repository, not a release-only listing." }, { "url": "https://github.com/storytold/pdfcraft/releases/tag/v0.8.0", "supports": "Specific public release and platform asset inventory." }, { "url": "https://github.com/storytold/pdfcraft/releases/download/v0.8.0/pdfcraft-0.8.0-windows-x64.msi", "supports": "Windows x64 installer availability verified by HTTP HEAD; execution was not tested." }, { "url": "https://github.com/storytold/pdfcraft/releases/download/v0.8.0/pdfcraft-0.8.0-macos-universal.dmg", "supports": "Universal macOS disk-image availability verified by HTTP HEAD; execution was not tested." }, { "url": "https://github.com/storytold/pdfcraft/blob/924c4da4ac2879ac9107c6c82e20ac6be5570f0a/LICENSE-MIT", "supports": "MIT license option." }, { "url": "https://github.com/storytold/pdfcraft/blob/924c4da4ac2879ac9107c6c82e20ac6be5570f0a/LICENSE-APACHE", "supports": "Apache-2.0 license option." }, { "url": "https://github.com/storytold/printcraft", "supports": "The creator-linked former repository URL redirects to storytold/pdfcraft; this is the renamed project, not a separate entry." } ] }, { "id": "filmcraft", "name": "FilmCraft", "description_en": "An alpha Rust video editor with a multitrack timeline, trimming, titles, color and audio controls, plus documented media-export and timeline-interchange paths.", "description_zh": "处于 alpha 阶段的 Rust 视频编辑器，包含多轨时间线、修剪、字幕、颜色与音频控制，并提供文档化的媒体导出及时间线交换流程。", "content_type": "app", "platforms": [ "windows", "macos", "linux", "web" ], "uses": [ "showcase" ], "implementation": "ai-assisted", "resource_kind": "ai-generated-output", "creator": "ArtCraft team and contributors; creator announcement by u/ai_art_is_art", "primary_url": "https://getartcraft.com/apps/filmcraft", "repo_url": "https://github.com/storytold/filmcraft", "source_file_url": "https://github.com/storytold/filmcraft/blob/09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5/apps/filmcraft/src/main.rs", "download_url": "https://github.com/storytold/filmcraft/releases/tag/v0.8.0", "demo_url": null, "paper_url": null, "provenance_source": "https://www.reddit.com/r/GeminiAI/comments/1wwo52q/ive_created_open_source_100_rust_replacements_for/", "provenance_note_en": "Author-reported LLM assistance; human direction and separately authored components remain.", "provenance_note_zh": "作者报告使用 LLM 辅助开发；仍包含人工指导及独立开发的组件。", "enables_en": "Inspect an AI-assisted desktop editing implementation that connects timeline changes, media processing and scripted export through shared engine commands.", "enables_zh": "检查 AI 辅助开发的桌面视频编辑实现，了解时间线修改、媒体处理和脚本导出如何共享引擎命令。", "limits_en": "Alpha. Premiere .prproj import and VST3/Audio Units/OpenFX hosting are absent. Codec and hardware paths vary: H.264 is the documented software delivery export, while H.265 needs supported hardware. Some effects/export rendering remain CPU-bound; the authors report less testing on Windows/Linux than macOS. No playback, export, codec-conformance or hardware test was performed here.", "limits_zh": "Alpha 阶段。尚无 Premiere .prproj 导入及 VST3／Audio Units／OpenFX 插件宿主。编解码与硬件路径存在差异：文档中的软件交付编码为 H.264，H.265 需要受支持硬件。部分特效及导出渲染仍依赖 CPU；作者称 Windows／Linux 的测试少于 macOS。本次未测试播放、导出、编解码一致性或硬件。", "runtime_note_en": "Inspected source commit and v0.8.0 release separately. Release assets include native Windows MSI/portable packages, a universal macOS DMG, Linux packages and a self-hosted WebAssembly archive. Windows x64 MSI and macOS DMG returned HTTP 200 to HEAD requests; files were not downloaded or executed. Reviewed application build guidance requires Rust 1.95+. No hosted interactive web demo was established.", "runtime_note_zh": "分别核查固定源码提交与 v0.8.0 发布。发布文件包含原生 Windows MSI／便携包、通用 macOS DMG、Linux 包及需自行托管的 WebAssembly 压缩包。Windows x64 MSI 与 macOS DMG 的 HEAD 请求返回 HTTP 200；未下载或执行文件。已核查的应用构建说明要求 Rust 1.95+，未确立公开托管的交互网页演示。", "license_en": "Application code is dual-licensed MIT OR Apache-2.0. Bundled assets and fonts retain their listed licenses; ArtCraft branding has separate trademark restrictions and is excluded from the code-license grant.", "license_zh": "应用代码采用 MIT 或 Apache-2.0 双重许可。随附素材与字体保留其列明的许可；ArtCraft 品牌有单独商标限制，不包含在代码许可授权中。", "license_url": "https://github.com/storytold/filmcraft/blob/09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5/LICENSE-MIT", "reviewed_at": "2026-10-16", "verification": "source-reviewed", "reviewed_version": { "source_commit": "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5", "source_version": "0.8.0", "observed_release": "v0.8.0", "source_and_release_not_runtime_compared": true }, "distribution_evidence": { "asset_listing_url": "https://github.com/storytold/filmcraft/releases/expanded_assets/v0.8.0", "artifacts": { "windows": "https://github.com/storytold/filmcraft/releases/download/v0.8.0/filmcraft-0.8.0-windows-x64.msi", "macos": "https://github.com/storytold/filmcraft/releases/download/v0.8.0/filmcraft-0.8.0-macos-universal.dmg", "linux": "https://github.com/storytold/filmcraft/releases/download/v0.8.0/filmcraft-0.8.0-linux-x86_64.AppImage", "web": "https://github.com/storytold/filmcraft/releases/download/v0.8.0/filmcraft-web-0.8.0.zip" }, "head_checks": { "windows": { "artifact_url": "https://github.com/storytold/filmcraft/releases/download/v0.8.0/filmcraft-0.8.0-windows-x64.msi", "status": 200, "content_length": "57221120", "content_type": "application/octet-stream" }, "macos": { "artifact_url": "https://github.com/storytold/filmcraft/releases/download/v0.8.0/filmcraft-0.8.0-macos-universal.dmg", "status": 200, "content_length": "73692833", "content_type": "application/octet-stream" } }, "architecture_note_en": "Release filenames document Windows x64/x86/ARM64 and macOS Universal variants; only Windows x64 and macOS Universal asset availability received HEAD checks." }, "evidence": [ { "url": "https://www.reddit.com/r/GeminiAI/comments/1wwo52q/ive_created_open_source_100_rust_replacements_for/", "supports": "Creator-linked implementation provenance." }, { "url": "https://github.com/storytold/filmcraft/blob/09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5/README.md", "supports": "Project-specific capabilities, build/distribution documentation, maturity and known limitations." }, { "url": "https://github.com/storytold/filmcraft/blob/09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5/Cargo.toml", "supports": "Inspectable application workspace metadata, code version, Rust edition/build floor and dual code license." }, { "url": "https://github.com/storytold/filmcraft/blob/09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5/apps/filmcraft/src/main.rs", "supports": "Published native application entry point; this is an application-source repository, not a release-only listing." }, { "url": "https://github.com/storytold/filmcraft/releases/tag/v0.8.0", "supports": "Specific public release and platform asset inventory." }, { "url": "https://github.com/storytold/filmcraft/releases/download/v0.8.0/filmcraft-0.8.0-windows-x64.msi", "supports": "Windows x64 installer availability verified by HTTP HEAD; execution was not tested." }, { "url": "https://github.com/storytold/filmcraft/releases/download/v0.8.0/filmcraft-0.8.0-macos-universal.dmg", "supports": "Universal macOS disk-image availability verified by HTTP HEAD; execution was not tested." }, { "url": "https://github.com/storytold/filmcraft/blob/09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5/LICENSE-MIT", "supports": "MIT license option." }, { "url": "https://github.com/storytold/filmcraft/blob/09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5/LICENSE-APACHE", "supports": "Apache-2.0 license option." } ] }, { "id": "effectcraft", "name": "EffectCraft", "description_en": "An alpha Rust motion-graphics and compositing app that combines layers, keyframes and effects, with native project files and documented Lottie/media export.", "description_zh": "处于 alpha 阶段的 Rust 动态图形与合成应用，结合图层、关键帧和特效，提供原生项目格式及文档化的 Lottie／媒体导出。", "content_type": "app", "platforms": [ "windows", "macos", "linux", "web" ], "uses": [ "showcase" ], "implementation": "ai-assisted", "resource_kind": "ai-generated-output", "creator": "ArtCraft team and contributors; creator announcement by u/ai_art_is_art", "primary_url": "https://getartcraft.com/apps/effectcraft", "repo_url": "https://github.com/storytold/effectcraft", "source_file_url": "https://github.com/storytold/effectcraft/blob/8f02f52a44a4b1ea807d8824b92301b96f896a06/apps/effectcraft/src/main.rs", "download_url": "https://github.com/storytold/effectcraft/releases/tag/v0.10.0", "demo_url": null, "paper_url": null, "provenance_source": "https://www.reddit.com/r/GeminiAI/comments/1wwo52q/ive_created_open_source_100_rust_replacements_for/", "provenance_note_en": "Author-reported LLM assistance; human direction and separately authored components remain.", "provenance_note_zh": "作者报告使用 LLM 辅助开发；仍包含人工指导及独立开发的组件。", "enables_en": "Inspect layer-based animation and compositing, including command-driven property changes, rendering and reusable project structures.", "enables_zh": "检查基于图层的动画与合成，包括通过命令修改属性、渲染及复用项目结构。", "limits_en": "Alpha. It cannot open After Effects .aep/.aepx projects or run its plug-ins; rendering equivalence has not been measured against After Effects. macOS is the most-tested platform. Upstream Windows ARM64 checks install the package and run the CLI but do not open the GUI. Optional tracking/masking models require separate downloads. No interaction or render/export test was performed here.", "limits_zh": "Alpha 阶段。不能打开 After Effects .aep／.aepx 项目或运行其插件，尚未与 After Effects 测量渲染等价性。macOS 是测试最多的平台。上游 Windows ARM64 检查会安装包并运行 CLI，但不打开 GUI。可选跟踪／蒙版模型需另行下载。本次未测试交互、渲染或导出。", "runtime_note_en": "Inspected source commit and v0.10.0 release separately. Release assets include native Windows MSI/portable packages, a universal macOS DMG, Linux packages and a self-hosted WebAssembly archive. Windows x64 MSI and macOS DMG returned HTTP 200 to HEAD requests; files were not downloaded or executed. Reviewed application build guidance requires Rust 1.95+. No hosted interactive web demo was established.", "runtime_note_zh": "分别核查固定源码提交与 v0.10.0 发布。发布文件包含原生 Windows MSI／便携包、通用 macOS DMG、Linux 包及需自行托管的 WebAssembly 压缩包。Windows x64 MSI 与 macOS DMG 的 HEAD 请求返回 HTTP 200；未下载或执行文件。已核查的应用构建说明要求 Rust 1.95+，未确立公开托管的交互网页演示。", "license_en": "Application code is dual-licensed MIT OR Apache-2.0. Bundled assets and fonts retain their listed licenses; ArtCraft branding has separate trademark restrictions and is excluded from the code-license grant.", "license_zh": "应用代码采用 MIT 或 Apache-2.0 双重许可。随附素材与字体保留其列明的许可；ArtCraft 品牌有单独商标限制，不包含在代码许可授权中。", "license_url": "https://github.com/storytold/effectcraft/blob/8f02f52a44a4b1ea807d8824b92301b96f896a06/LICENSE-MIT", "reviewed_at": "2026-10-16", "verification": "source-reviewed", "reviewed_version": { "source_commit": "8f02f52a44a4b1ea807d8824b92301b96f896a06", "source_version": "0.10.0", "observed_release": "v0.10.0", "source_and_release_not_runtime_compared": true }, "distribution_evidence": { "asset_listing_url": "https://github.com/storytold/effectcraft/releases/expanded_assets/v0.10.0", "artifacts": { "windows": "https://github.com/storytold/effectcraft/releases/download/v0.10.0/effectcraft-0.10.0-windows-x64.msi", "macos": "https://github.com/storytold/effectcraft/releases/download/v0.10.0/effectcraft-0.10.0-macos-universal.dmg", "linux": "https://github.com/storytold/effectcraft/releases/download/v0.10.0/effectcraft-0.10.0-linux-x86_64.AppImage", "web": "https://github.com/storytold/effectcraft/releases/download/v0.10.0/effectcraft-web-0.10.0.zip" }, "head_checks": { "windows": { "artifact_url": "https://github.com/storytold/effectcraft/releases/download/v0.10.0/effectcraft-0.10.0-windows-x64.msi", "status": 200, "content_length": "50470912", "content_type": "application/octet-stream" }, "macos": { "artifact_url": "https://github.com/storytold/effectcraft/releases/download/v0.10.0/effectcraft-0.10.0-macos-universal.dmg", "status": 200, "content_length": "64038819", "content_type": "application/octet-stream" } }, "architecture_note_en": "Release filenames document Windows x64/x86/ARM64 and macOS Universal variants; only Windows x64 and macOS Universal asset availability received HEAD checks." }, "evidence": [ { "url": "https://www.reddit.com/r/GeminiAI/comments/1wwo52q/ive_created_open_source_100_rust_replacements_for/", "supports": "Creator-linked implementation provenance." }, { "url": "https://github.com/storytold/effectcraft/blob/8f02f52a44a4b1ea807d8824b92301b96f896a06/README.md", "supports": "Project-specific capabilities, build/distribution documentation, maturity and known limitations." }, { "url": "https://github.com/storytold/effectcraft/blob/8f02f52a44a4b1ea807d8824b92301b96f896a06/Cargo.toml", "supports": "Inspectable application workspace metadata, code version, Rust edition/build floor and dual code license." }, { "url": "https://github.com/storytold/effectcraft/blob/8f02f52a44a4b1ea807d8824b92301b96f896a06/apps/effectcraft/src/main.rs", "supports": "Published native application entry point; this is an application-source repository, not a release-only listing." }, { "url": "https://github.com/storytold/effectcraft/releases/tag/v0.10.0", "supports": "Specific public release and platform asset inventory." }, { "url": "https://github.com/storytold/effectcraft/releases/download/v0.10.0/effectcraft-0.10.0-windows-x64.msi", "supports": "Windows x64 installer availability verified by HTTP HEAD; execution was not tested." }, { "url": "https://github.com/storytold/effectcraft/releases/download/v0.10.0/effectcraft-0.10.0-macos-universal.dmg", "supports": "Universal macOS disk-image availability verified by HTTP HEAD; execution was not tested." }, { "url": "https://github.com/storytold/effectcraft/blob/8f02f52a44a4b1ea807d8824b92301b96f896a06/LICENSE-MIT", "supports": "MIT license option." }, { "url": "https://github.com/storytold/effectcraft/blob/8f02f52a44a4b1ea807d8824b92301b96f896a06/LICENSE-APACHE", "supports": "Apache-2.0 license option." } ] }, { "id": "scienceboard", "name": "ScienceBoard", "description_en": "A scientific desktop environment and benchmark covering molecular visualization, astronomy, GIS, algebra, theorem proving and scientific document workflows.", "description_zh": "科学桌面环境及基准，覆盖分子可视化、天文学、GIS、代数、定理证明和科研文档流程。", "content_type": "app", "platforms": [ "linux" ], "uses": [ "evaluation" ], "implementation": "unknown", "resource_kind": "environment-and-benchmark", "primary_url": "https://github.com/OS-Copilot/ScienceBoard", "repo_url": "https://github.com/OS-Copilot/ScienceBoard", "demo_url": null, "paper_url": "https://arxiv.org/abs/2505.19897", "enables_en": "Evaluate agents operating instrumented scientific applications and check task completion through application-specific state and artifact evaluators.", "enables_zh": "评估 Agent 操作接入评测接口的科学应用，并通过应用专属状态及文件检查判定任务完成。", "limits_en": "The original Ubuntu VM is public. Current source adds a Docker-native backend, but its prebuilt Docker image is documented as a private review artifact pending redistribution review. Some tasks depend on Google sign-in or changing web services. The reported 20-task Docker acceptance run is an upstream subset check, not full-suite reproduction by this catalog.", "limits_zh": "原始 Ubuntu 虚拟机公开。当前源码新增 Docker 原生后端，但预构建 Docker 镜像被文档列为等待再分发审查的私有评审资产。部分任务依赖 Google 登录或会变化的网页服务。上游报告的 20 项 Docker 验收仅是子集检查，不代表本站复现整个基准。", "license_en": "Framework code is MIT and the original VM Hub card is MIT-labeled. Bundled applications retain separate terms; the Docker delivery guide explicitly identifies ChimeraX redistribution/commercial-license constraints. The trajectory dataset has no license declaration in its inspected card.", "license_zh": "框架代码采用 MIT，原始虚拟机 Hub 卡片也标注 MIT；内置应用保留独立条款。Docker 分发指南明确指出 ChimeraX 的再分发及商业许可限制；所查轨迹数据卡未声明许可。", "runtime": "Python 3.11; Ubuntu 22.04 guest/application stack. Legacy operation uses VMware; Docker operation uses x86-64 Linux, Docker and an authorized locally prepared image without runtime KVM. Application state plugins and modified scientific applications are part of the setup.", "platform_note": "Linux describes the Ubuntu scientific desktop. Windows support in legacy instructions is a VMware host choice, not Windows task coverage.", "reviewed_version": { "source_commit": "21b5eb0f1d00898f7de9e84c3d33a87c2f2b3c3e", "vm_hub_commit": "3c7edc429c4035215def670804d9eb51ed4090b0" }, "evidence": [ { "url": "https://github.com/OS-Copilot/ScienceBoard/blob/21b5eb0f1d00898f7de9e84c3d33a87c2f2b3c3e/README.md", "supports": "Scientific application families, original VM release, trajectories and the newer Docker backend." }, { "url": "https://github.com/OS-Copilot/ScienceBoard/blob/21b5eb0f1d00898f7de9e84c3d33a87c2f2b3c3e/vm_config/manual.md", "supports": "Ubuntu 22.04 setup and modified ChimeraX, KAlgebra, Celestia, GRASS GIS, TeXstudio and Lean stack." }, { "url": "https://github.com/OS-Copilot/ScienceBoard/blob/21b5eb0f1d00898f7de9e84c3d33a87c2f2b3c3e/sci/docker/desktop_env.py", "supports": "Inspectable Docker lifecycle/reset implementation; source read without execution." }, { "url": "https://github.com/OS-Copilot/ScienceBoard/blob/21b5eb0f1d00898f7de9e84c3d33a87c2f2b3c3e/docs/image-delivery.md", "supports": "Docker image is a private review artifact; public redistribution requires separate application-license review." }, { "url": "https://github.com/OS-Copilot/ScienceBoard/blob/21b5eb0f1d00898f7de9e84c3d33a87c2f2b3c3e/docs/known-issues.md", "supports": "Authentication/network dependencies, amd64 limits and the scope of upstream acceptance checks." }, { "url": "https://huggingface.co/OS-Copilot/ScienceBoard-Env", "supports": "Public ungated original VM archive and MIT card; no archive downloaded." }, { "url": "https://github.com/OS-Copilot/ScienceBoard/blob/21b5eb0f1d00898f7de9e84c3d33a87c2f2b3c3e/LICENSE", "supports": "MIT framework license." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "cadworld", "name": "CADWorld", "description_en": "A FreeCAD computer-use benchmark that evaluates screenshot-driven GUI work through executable checks on saved CAD projects and engineering outputs.", "description_zh": "FreeCAD 计算机操作基准，通过对保存的 CAD 项目及工程输出执行检查，评估基于截图的 GUI 操作。", "content_type": "app", "platforms": [ "linux" ], "uses": [ "evaluation" ], "implementation": "unknown", "resource_kind": "environment-and-benchmark", "primary_url": "https://github.com/Zdong104/CADWORLD", "repo_url": "https://github.com/Zdong104/CADWORLD", "demo_url": null, "paper_url": "https://arxiv.org/abs/2609.16251", "enables_en": "Evaluate modeling, sketch constraints, assembly, manufacturing and simulation workflows against task-specific artifact requirements.", "enables_zh": "根据任务专属文件要求，评估建模、草图约束、装配、制造及仿真流程。", "limits_en": "Requires a Linux/KVM Docker host and the Ubuntu FreeCAD VM. CAM-task evaluation, live or offline, needs a host FreeCAD 1.1.x console; most other checks parse files in Python. Omitting FreeCAD can invalidate CAM scores. Keep the 200-task GUI benchmark distinct from the paper's 50-task terminal-only ablation. A project-wide code license was not found in the reviewed root or package metadata.", "limits_zh": "需要 Linux／KVM Docker 主机及 Ubuntu FreeCAD 虚拟机。CAM 任务的在线评分和离线重评需要主机上的 FreeCAD 1.1.x 控制台；大多数其他检查通过 Python 解析文件。缺失 FreeCAD 可能影响 CAM 评分。200 项 GUI 基准应与论文中的 50 项纯终端消融分别比较。所查根目录及包元数据未找到项目级代码许可。", "license_en": "The task dataset declares Apache-2.0; the VM Hub card separately declares MIT. These declarations do not replace FreeCAD or other bundled software terms and do not resolve the missing project-wide GitHub code license.", "license_zh": "任务数据集声明 Apache-2.0，虚拟机 Hub 卡片另行声明 MIT。这些声明不替代 FreeCAD 或其他内置软件条款，也不能补足 GitHub 项目级代码许可的缺失。", "runtime": "Pinned source uses Python 3.12, Docker/QEMU and KVM. The documented VM defaults to 8 CPUs, 8 GB RAM and a 64 GB virtual disk; image storage and a matching host-side FreeCAD console are additional setup requirements.", "platform_note": "Linux refers to the released Ubuntu task guest. FreeCAD's availability on other operating systems is not used as a benchmark platform claim.", "reviewed_version": { "source_commit": "e5d0ebab8bd28da6e98c68c61db05747d1647f7b", "task_dataset_commit": "fa02aa03b944daf2ae5b23e56d135bb7cbb75994", "vm_hub_commit": "8dd055555291c787c9f9bbed2664ed416dfcbabb" }, "evidence": [ { "url": "https://github.com/Zdong104/CADWORLD/blob/e5d0ebab8bd28da6e98c68c61db05747d1647f7b/README.md", "supports": "Ubuntu screenshot/pyautogui workflow, KVM/Docker setup and host FreeCAD 1.1.x requirement." }, { "url": "https://github.com/Zdong104/CADWORLD/blob/e5d0ebab8bd28da6e98c68c61db05747d1647f7b/desktop_env/evaluators/metrics/freecad_cam.py", "supports": "Inspectable CAM evaluator invokes a host FreeCAD console." }, { "url": "https://github.com/Zdong104/CADWORLD/blob/e5d0ebab8bd28da6e98c68c61db05747d1647f7b/pyproject.toml", "supports": "Python package requirements; no project license field." }, { "url": "https://huggingface.co/datasets/Zihan1004/CADWORLD", "supports": "Public tasks/evaluation assets with Apache-2.0 dataset declaration." }, { "url": "https://huggingface.co/Zihan1004/CADWorld", "supports": "Public FreeCAD Ubuntu VM repository with separate MIT card." }, { "url": "https://cad-world.github.io/", "supports": "Distinguishes the main GUI benchmark from the 50-task terminal-only ablation." }, { "url": "https://github.com/Zdong104/CADWORLD/blob/e5d0ebab8bd28da6e98c68c61db05747d1647f7b/README.md#L76-L89", "supports": "Explicitly distinguishes mostly pure-Python metrics from 15 freecad-cam-* console checks, in live and archived scoring; recommends FreeCAD 1.1.x." }, { "url": "https://github.com/Zdong104/CADWORLD/blob/e5d0ebab8bd28da6e98c68c61db05747d1647f7b/README.md#L109-L111", "supports": "Offline re-evaluation/diagnostics otherwise run with Python >=3.10 and requests; full live-run environment is separate." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "engiworld", "name": "EngiWorld", "description_en": "A professional engineering benchmark with GUI and CLI workflows across CAD, simulation, manufacturing, building design, electronics and 3D content creation.", "description_zh": "专业工程基准，覆盖 CAD、仿真、制造、建筑设计、电子设计及三维内容制作的 GUI 与 CLI 工作流。", "content_type": "app", "platforms": [ "linux", "windows" ], "uses": [ "evaluation" ], "implementation": "unknown", "resource_kind": "environment-and-benchmark", "primary_url": "https://github.com/Hongcheng-Gao/EngiWorld", "repo_url": "https://github.com/Hongcheng-Gao/EngiWorld", "demo_url": null, "paper_url": "https://arxiv.org/abs/2609.37686", "enables_en": "Evaluate individual applications, software selection, cross-software coordination and optimization using task-specific checks on native engineering artifacts.", "enables_zh": "通过对原生工程文件执行任务专属检查，评估单软件操作、软件选择、跨软件协作及设计优化。", "limits_en": "The release provides 1,301 task definitions, while the paper's main experiments use 306 tasks. Public images cover only 15 open-source software environments, all Ubuntu in the inspected manifest; proprietary-software images require separate licensed access or user preparation. Respect each task's GUI/CLI contract and pin the task taxonomy, assets and matching image.", "limits_zh": "发布内容包含 1,301 项任务定义，论文主要实验使用其中 306 项。公开镜像仅覆盖 15 个开源软件环境，所查清单均为 Ubuntu；专有软件镜像需另行取得合许可访问或自行准备。须遵循每项任务的 GUI／CLI 接口要求，并固定任务分类、资产及匹配镜像。", "license_en": "Original code and the task dataset declare MIT. The OSWorld-derived runtime engine retains Apache-2.0 notices. Environment images have software-specific terms; no single permissive image license is declared.", "license_zh": "原创代码及任务数据集声明 MIT；基于 OSWorld 的运行引擎保留 Apache-2.0 声明。环境镜像适用各软件自身条款，未声明统一的宽松镜像许可。", "runtime": "Python 3.10+ on an x86_64 Linux Docker host, zstd-compressed VM images and model access. KVM accelerates execution; the documented runner can attempt slower software emulation. Complete task resources and matching images are distributed separately.", "platform_note": "Task definitions target Windows and Ubuntu guests. The public image catalog currently contains Ubuntu 24.04 images only; Windows/proprietary runtime availability is not implied by the Windows task tag.", "reviewed_version": { "source_commit": "dc4936fffd46b764e0443c840081e7f1bd60d515", "task_dataset_commit": "a9958dfec3cb9fea16be25551336edaa555a257d", "image_dataset_commit": "a6690a8f8de7470a676b87032d7b3ab08f23a0d2" }, "evidence": [ { "url": "https://github.com/Hongcheng-Gao/EngiWorld/blob/dc4936fffd46b764e0443c840081e7f1bd60d515/README.md", "supports": "1,301 tasks, 306-task experiment subset, GUI/CLI interfaces and Windows/Ubuntu runtime design." }, { "url": "https://github.com/Hongcheng-Gao/EngiWorld/blob/dc4936fffd46b764e0443c840081e7f1bd60d515/task/single-software-execution/gui/librecad/task-08/eval.py", "supports": "Concrete public evaluator checks engineering artifacts and GUI task constraints." }, { "url": "https://github.com/Hongcheng-Gao/EngiWorld/blob/dc4936fffd46b764e0443c840081e7f1bd60d515/runtime/pyproject.toml", "supports": "Python >=3.10 and optional runtime dependency groups." }, { "url": "https://huggingface.co/datasets/HongchengGao/EngiWorld-Images/blob/a6690a8f8de7470a676b87032d7b3ab08f23a0d2/README.md", "supports": "Only open-source software images are public; proprietary images are license-dependent." }, { "url": "https://huggingface.co/datasets/HongchengGao/EngiWorld-Images/blob/a6690a8f8de7470a676b87032d7b3ab08f23a0d2/manifest.json", "supports": "15 published image manifests specify Ubuntu 24.04/amd64." }, { "url": "https://github.com/Hongcheng-Gao/EngiWorld/blob/dc4936fffd46b764e0443c840081e7f1bd60d515/THIRD_PARTY_NOTICES.md", "supports": "Separates original MIT code from Apache-2.0 OSWorld-derived components." }, { "url": "https://huggingface.co/datasets/HongchengGao/EngiWorld-Tasks", "supports": "Public task/evaluator artifact repository with MIT card." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "blendergym", "name": "BlenderGym", "description_en": "A 3D graphics-editing benchmark that asks model systems to modify Blender Python code so a starting scene matches a target scene.", "description_zh": "三维图形编辑基准，要求模型系统修改 Blender Python 代码，使初始场景匹配目标场景。", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "resource_kind": "benchmark-suite", "primary_url": "https://github.com/richard-guyunqi/BlenderGym-Open", "repo_url": "https://github.com/richard-guyunqi/BlenderGym-Open", "demo_url": null, "paper_url": "https://arxiv.org/abs/2504.01786", "enables_en": "Compare code-based edits for object placement, lighting, materials, blend shapes and procedural geometry using rendered and geometric metrics.", "enables_zh": "通过渲染及几何指标，比较物体放置、照明、材质、形态混合和程序化几何的代码编辑结果。", "limits_en": "This evaluates code-based scene reconstruction, not mouse-and-keyboard Blender operation. Public evaluation code and a dataset archive are available, but no benchmark-wide code or data license was found. Setup uses Python 3.10, an Infinigen Blender dependency and GPU-oriented packages; the starter script clones a dependency through SSH. Archives and execution were not tested.", "limits_zh": "评估对象是基于代码的场景重建，不是鼠标键盘操作 Blender。评测代码及数据档案公开，但未找到覆盖整个基准的代码或数据许可。配置使用 Python 3.10、Infinigen Blender 依赖及偏向 GPU 的软件包，启动脚本通过 SSH 克隆依赖。本次未下载档案或执行验证。", "license_en": "No project-level license file was present in the reviewed repository root, and the author-linked dataset card states no license. Blender/Infinigen and model dependencies have their own terms; those do not establish benchmark-data rights.", "license_zh": "所查仓库根目录没有项目级许可文件，作者链接的数据卡也未声明许可。Blender／Infinigen 及模型依赖各有条款，但不能由此推断基准数据的使用许可。", "runtime": "Conda/Python 3.10, the released TaskSolver and graphics-editing pipeline, Infinigen-provided Blender, and configured local or API-backed models. The package pins Torch/Transformers and includes FlashAttention/bitsandbytes dependencies.", "platform_note": "Blender is the target application, but no operating-system tag is inferred from its cross-platform product support or shell-based installation instructions.", "reviewed_version": { "source_commit": "67c8b48bf8c548c701b6873db6fcc6527d756385", "dataset_commit": "e519b50a5e87901f43b5a894a56e820bdc3a68a9" }, "evidence": [ { "url": "https://github.com/richard-guyunqi/BlenderGym-Open/blob/67c8b48bf8c548c701b6873db6fcc6527d756385/README.md", "supports": "Official inference/evaluation release and author-linked dataset." }, { "url": "https://blendergym.github.io/", "supports": "Five code-editing task categories and start/goal scene artifact design." }, { "url": "https://github.com/richard-guyunqi/BlenderGym-Open/blob/67c8b48bf8c548c701b6873db6fcc6527d756385/starter_setup.sh", "supports": "Documented setup installs TaskSolver and clones the Infinigen dependency by SSH." }, { "url": "https://github.com/richard-guyunqi/BlenderGym-Open/blob/67c8b48bf8c548c701b6873db6fcc6527d756385/TaskSolver/setup.py", "supports": "Pinned model/runtime dependencies, including GPU-oriented packages." }, { "url": "https://github.com/richard-guyunqi/BlenderGym-Open/blob/67c8b48bf8c548c701b6873db6fcc6527d756385/evaluation.py", "supports": "Public artifact/render evaluation implementation." }, { "url": "https://huggingface.co/datasets/richard-guyunqi/BG_bench_data", "supports": "Public ungated benchmark archive; no dataset license declaration." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "officebench", "name": "OfficeBench", "description_en": "An office-workflow benchmark in which language agents coordinate document, spreadsheet, email and calendar operations through structured tool commands.", "description_zh": "办公工作流基准，让语言 Agent 通过结构化工具命令协调文档、电子表格、邮件及日历操作。", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "resource_kind": "environment-and-benchmark", "primary_url": "https://github.com/zlwang-cs/OfficeBench", "repo_url": "https://github.com/zlwang-cs/OfficeBench", "demo_url": null, "paper_url": "https://arxiv.org/abs/2407.19056", "enables_en": "Evaluate planning and application switching across local office artifacts using exact, fuzzy and execution-based task checks.", "enables_zh": "使用精确匹配、模糊匹配及执行式任务检查，评估围绕本地办公文件的规划和应用切换。", "limits_en": "The released applications are Python-backed tools in a Docker workspace, not native Microsoft Office GUI sessions. Email writes local .eml files and calendar actions write .ics files; they are not live account integrations. Setup requires Python 3.10, Docker, document/OCR dependencies and model access. The legacy image recipe expects a local model-key file and uses an unpinned Ubuntu base.", "limits_zh": "发布的“应用”是在 Docker 工作区内由 Python 实现的工具，不是原生 Microsoft Office GUI 会话。邮件操作写入本地 .eml 文件，日历操作写入 .ics 文件，并非真实账号集成。需 Python 3.10、Docker、文档／OCR 依赖及模型访问；旧镜像配置需要本地模型密钥文件，且 Ubuntu 基础镜像未固定版本。", "license_en": "The repository declares Apache-2.0. Included frameworks, document libraries and operating-system packages retain their respective terms.", "license_zh": "仓库声明 Apache-2.0；所包含框架、文档库及操作系统软件包保留各自条款。", "runtime": "The Dockerfile installs LibreOffice and Tesseract. Excel/Word actions use openpyxl/python-docx; mail and calendar tools manipulate local files. Model/provider configuration and task-specific assets remain necessary.", "platform_note": "The target is a structured office-tool environment. An Ubuntu container and Office file formats do not establish native Linux or Windows application coverage.", "reviewed_version": { "source_commit": "b978b808667c32b52ce19a67ce1def1de9ae02b7" }, "evidence": [ { "url": "https://github.com/zlwang-cs/OfficeBench/blob/b978b808667c32b52ce19a67ce1def1de9ae02b7/README.md", "supports": "Cross-application transition/action design, setup and task evaluation." }, { "url": "https://github.com/zlwang-cs/OfficeBench/blob/b978b808667c32b52ce19a67ce1def1de9ae02b7/apps/email_app/email_send_email.py", "supports": "Email operation writes .eml files under the local testbed rather than using an SMTP account." }, { "url": "https://github.com/zlwang-cs/OfficeBench/blob/b978b808667c32b52ce19a67ce1def1de9ae02b7/apps/calendar_app/calendar_create_event.py", "supports": "Calendar operation writes local .ics files." }, { "url": "https://github.com/zlwang-cs/OfficeBench/blob/b978b808667c32b52ce19a67ce1def1de9ae02b7/apps/excel_app/excel_set_cell.py", "supports": "Spreadsheet changes use openpyxl rather than an Excel GUI." }, { "url": "https://github.com/zlwang-cs/OfficeBench/blob/b978b808667c32b52ce19a67ce1def1de9ae02b7/apps/word_app/word_write_to_file.py", "supports": "Document changes use python-docx rather than Word GUI control." }, { "url": "https://github.com/zlwang-cs/OfficeBench/blob/b978b808667c32b52ce19a67ce1def1de9ae02b7/docker/Dockerfile", "supports": "Ubuntu/LibreOffice/Tesseract image and local key-file build assumption." }, { "url": "https://github.com/zlwang-cs/OfficeBench/blob/b978b808667c32b52ce19a67ce1def1de9ae02b7/LICENSE", "supports": "Apache-2.0 repository license." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "spreadsheetbench", "name": "SpreadsheetBench", "description_en": "A real-world spreadsheet-editing dataset and benchmark with multiple workbook test cases for each user instruction.", "description_zh": "真实场景电子表格编辑数据集与基准，为每条用户指令提供多个工作簿测试用例。", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "resource_kind": "dataset-and-benchmark", "primary_url": "https://github.com/RUCKBReasoning/SpreadsheetBench", "repo_url": "https://github.com/RUCKBReasoning/SpreadsheetBench", "demo_url": null, "paper_url": "https://arxiv.org/abs/2406.14991", "enables_en": "Evaluate spreadsheet edits against expected cells and formatting across varied input workbooks, including irregular tables and multi-sheet files.", "enables_zh": "在不同输入工作簿上按预期单元格及格式评估编辑结果，涵盖不规则表格与多工作表文件。", "limits_en": "The complete release has 912 instructions; Verified is an expert-annotated 400-instance subset, not 400 additional tasks. The code also retains older 200-example setup paths. Formula results need recalculation through LibreOffice or Excel before checking; record the workbook release and calculation backend. This is artifact editing, not a packaged GUI-navigation benchmark.", "limits_zh": "完整发布包含 912 条指令；Verified 是经专家标注的 400 项子集，不是新增 400 项任务。代码仍保留较早的 200 示例配置路径。判分前需通过 LibreOffice 或 Excel 重算公式结果，并记录工作簿版本及计算后端。这是文件编辑评测，不是封装好的 GUI 导航基准。", "license_en": "The project README and official dataset card declare CC BY-SA 4.0. Code-execution infrastructure and document software dependencies have separate licenses.", "license_zh": "项目 README 及官方数据卡声明 CC BY-SA 4.0；代码执行基础设施和文档软件依赖另有许可。", "runtime": "Released Python inference/evaluation code with a CodeAct-derived Docker execution service. Current recalculation supports LibreOffice 7.5+ or Windows Excel/win32com; backend availability does not imply benchmark execution was independently tested.", "platform_note": "Spreadsheet files and their evaluators are the target artifacts; cross-platform runner support is not converted into native-app platform tags.", "reviewed_version": { "source_commit": "49b73a94775fb489063f60ca1865e3a650079a79", "dataset_commit": "ab0b742b0fc95b946f212d80ac7771b5531272e4" }, "evidence": [ { "url": "https://github.com/RUCKBReasoning/SpreadsheetBench/blob/49b73a94775fb489063f60ca1865e3a650079a79/README.md", "supports": "912-instruction release, 400-instance Verified subset, legacy sample paths and CC BY-SA 4.0 declaration." }, { "url": "https://github.com/RUCKBReasoning/SpreadsheetBench/blob/49b73a94775fb489063f60ca1865e3a650079a79/evaluation/open_spreadsheet.py", "supports": "LibreOffice and Excel COM recalculation backends in the current source." }, { "url": "https://huggingface.co/datasets/KAKA22/SpreadsheetBench", "supports": "Public complete and Verified archives with CC BY-SA 4.0 card." }, { "url": "https://arxiv.org/abs/2406.14991", "supports": "Original benchmark design with multiple test workbooks per instruction." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "spreadsheetbench-2", "name": "SpreadsheetBench 2", "description_en": "A distinct benchmark for end-to-end business spreadsheet generation, debugging and visualization across complex multi-sheet workbooks.", "description_zh": "面向复杂多工作表的独立业务流程基准，覆盖端到端电子表格生成、调试及可视化。", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "resource_kind": "dataset-and-benchmark", "primary_url": "https://github.com/RUCKBReasoning/SpreadsheetBench-2", "repo_url": "https://github.com/RUCKBReasoning/SpreadsheetBench-2", "demo_url": null, "paper_url": "https://arxiv.org/abs/2606.29955", "enables_en": "Evaluate delivered financial models, repaired formulas, template completions and charts through cell/formula checks and visualization checklists.", "enables_zh": "通过单元格／公式检查及可视化清单，评估交付的财务模型、公式修复、模板补全及图表。", "limits_en": "This is a new workflow task set, not the original benchmark's Verified subset. Recalculate input, reference and output workbooks before scoring. Extracting charts from XLSX for VLM grading requires Windows Excel/WPS COM; the evaluator also accepts PNG outputs directly. Data is MIT-labeled, but no root license for original evaluation code was found. The README's root Dockerfile command disagrees with its actual SWE-agent/ location.", "limits_zh": "这是新的工作流任务集，不是原基准的 Verified 子集。评分前需重算输入、参考及输出工作簿。从 XLSX 提取图表进行视觉语言模型评分，需要 Windows 上的 Excel／WPS COM；评分器也直接接受 PNG 输出。数据标注 MIT，但未找到原创评测代码的根目录许可。README 的根目录 Dockerfile 命令与文件实际位于 SWE-agent/ 的位置不符。", "license_en": "The official dataset card declares MIT. The bundled SWE-agent component has its own license; that does not automatically establish a license for all original repository code or external spreadsheet software.", "license_zh": "官方数据卡声明 MIT。内置 SWE-agent 组件有自己的许可，但不能自动据此认定全部原创仓库代码及外部电子表格软件的许可。", "runtime": "Python 3.11, the bundled SWE-agent workflow and a Docker image with LibreOffice. Numeric/formula checking and Windows COM/VLM visualization checking are separate evaluation paths. Pin data archive, harness, spreadsheet engine and judge.", "platform_note": "Targets workbook artifacts. Windows is required by the current chart-export evaluator, not asserted as the platform of every benchmark workflow.", "reviewed_version": { "source_commit": "5842db8f836887db15144b267467339bb7458f92", "dataset_commit": "5a2215ed4121945ab09d8723df2995602090b042" }, "evidence": [ { "url": "https://github.com/RUCKBReasoning/SpreadsheetBench-2/blob/5842db8f836887db15144b267467339bb7458f92/README.md", "supports": "Separate workflow dataset, four implementation folders, recalculation protocol and Windows COM/VLM visualization path." }, { "url": "https://github.com/RUCKBReasoning/SpreadsheetBench-2/blob/5842db8f836887db15144b267467339bb7458f92/SWE-agent/spreadsheet.Dockerfile", "supports": "Actual Dockerfile location and Python 3.11/LibreOffice execution environment." }, { "url": "https://github.com/RUCKBReasoning/SpreadsheetBench-2/blob/5842db8f836887db15144b267467339bb7458f92/evaluation/evaluation.py", "supports": "Inspectable cell, formula and formatting comparison implementation." }, { "url": "https://github.com/RUCKBReasoning/SpreadsheetBench-2/blob/5842db8f836887db15144b267467339bb7458f92/evaluation/run_visual_vlm_checklist_eval.py", "supports": "Excel/WPS COM chart export and separate VLM checklist grading." }, { "url": "https://huggingface.co/datasets/KAKA22/SpreadsheetBench-v2", "supports": "Public workflow-data archives with MIT dataset card." }, { "url": "https://arxiv.org/abs/2606.29955", "supports": "Original paper defines a new workflow-level task set, substantively distinct from SpreadsheetBench 1." }, { "url": "https://github.com/RUCKBReasoning/SpreadsheetBench-2/blob/5842db8f836887db15144b267467339bb7458f92/evaluation/run_visual_vlm_checklist_eval.py#L1197-L1222", "supports": "Finds .xlsx and .png; if png_path assigns image_input directly; elif xlsx_path invokes chart_exporter." }, { "url": "https://github.com/RUCKBReasoning/SpreadsheetBench-2/blob/5842db8f836887db15144b267467339bb7458f92/evaluation/run_visual_vlm_checklist_eval.py#L246-L278", "supports": "COM detection checks Excel.Application and Ket.Application and is used by the chart-image exporter." } ], "reviewed_at": "2026-10-16", "verification": "source-reviewed" }, { "id": "os-sentinel", "name": "OS-Sentinel / MobileRisk-Live", "resource_kind": "environment-and-benchmark", "content_type": "app", "platforms": [ "android" ], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/OS-Copilot/OS-Sentinel", "repo_url": "https://github.com/OS-Copilot/OS-Sentinel", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "description_en": "An Android safety-research framework combining state/rule checks and a vision-language judge, with mobile task scenarios and recorded-risk trajectory evaluation.", "description_zh": "Android 安全研究框架，结合状态／规则检查与视觉语言模型评审，提供移动任务场景及风险轨迹评测。", "paper_url": "https://arxiv.org/abs/2510.24411", "enables_en": "Evaluate action-level and trajectory-level safety detection, or record agents operating in the AndroidWorld-based MobileSafetyBench implementation.", "enables_zh": "评估动作级和轨迹级安全检测，或记录 Agent 在基于 AndroidWorld 的 MobileSafetyBench 实现中的操作。", "limits_en": "Public code and task scenarios exist, but setup requires separately supplied Android apps. Four APK download URLs in root.py are literal placeholders; follow the app-version file manually. Safety detection is not a guarantee of safe behavior or task completion. The README MIT badge conflicts with the Apache-2.0 LICENSE; MobileRisk data has no public license card.", "limits_zh": "代码及任务场景已公开，但需另行准备 Android 应用。root.py 中四个 APK 下载地址仍为占位符，需人工按版本文件准备。安全检测不等于安全行为或任务完成保证。README 的 MIT 徽章与 Apache-2.0 LICENSE 冲突；MobileRisk 数据未提供公开许可卡。", "runtime_note_en": "Requires AndroidWorld/emulator setup, specified APK versions, Node.js/Appium/UIAutomator2 and task configuration. External VLM modes need model access. The repository calls its runtime MobileSafetyBench; the current paper calls the environment MobileRisk-Live. Recorded-data archives were listed, not downloaded.", "runtime_note_zh": "需配置 AndroidWorld／模拟器、指定版本 APK、Node.js／Appium／UIAutomator2 及任务参数；外部视觉模型模式需模型访问。仓库运行环境名为 MobileSafetyBench，当前论文称 MobileRisk-Live。已确认记录数据档案列表，但未下载。", "license_en": "Root LICENSE is Apache-2.0 while the README badge says MIT. MobileRisk has no public license declaration. Third-party Android applications retain their own terms.", "license_zh": "根目录 LICENSE 为 Apache-2.0，但 README 徽章标注 MIT；MobileRisk 未公开声明许可。第三方 Android 应用保留各自条款。", "license_url": "https://github.com/OS-Copilot/OS-Sentinel/blob/fd7f57aa92aab55b74defc2c804aaf842aaed3c2/LICENSE", "evidence": [ { "url": "https://github.com/OS-Copilot/OS-Sentinel/blob/fd7f57aa92aab55b74defc2c804aaf842aaed3c2/README.md", "supports": "AndroidWorld/Appium setup, live step/record modes, recorded-data evaluators and MIT badge." }, { "url": "https://github.com/OS-Copilot/OS-Sentinel/blob/fd7f57aa92aab55b74defc2c804aaf842aaed3c2/msb.py", "supports": "Substantive MobileSafetyEnv agent runner with task/scenario selection; source read only." }, { "url": "https://github.com/OS-Copilot/OS-Sentinel/blob/fd7f57aa92aab55b74defc2c804aaf842aaed3c2/pipeline/eval.py", "supports": "Released rule/state, trajectory and action-level VLM safety evaluation." }, { "url": "https://github.com/OS-Copilot/OS-Sentinel/blob/fd7f57aa92aab55b74defc2c804aaf842aaed3c2/asset/tasks/tasks.json", "supports": "80 scenario rows across 21 unique task_id values; these are not80 independent base workflows." }, { "url": "https://github.com/OS-Copilot/OS-Sentinel/blob/fd7f57aa92aab55b74defc2c804aaf842aaed3c2/root.py", "supports": "Four APK URLs are literal placeholders; missing files must be supplied separately." }, { "url": "https://github.com/OS-Copilot/OS-Sentinel/blob/fd7f57aa92aab55b74defc2c804aaf842aaed3c2/asset/environments/apk_versions.txt", "supports": "App versions and manual APK preparation; Simple Calendar filename differs from root.py." }, { "url": "https://github.com/OS-Copilot/OS-Sentinel/blob/fd7f57aa92aab55b74defc2c804aaf842aaed3c2/LICENSE", "supports": "Apache-2.0 text conflicts with the badge." }, { "url": "https://huggingface.co/api/datasets/OS-Copilot/MobileRisk", "supports": "Public ungated archive listing, no license card; archive contents not inspected." }, { "url": "https://arxiv.org/abs/2510.24411", "supports": "Current paper names MobileRisk-Live and describes the hybrid safety detector." } ] }, { "id": "webarena-verified", "name": "WebArena-Verified", "resource_kind": "benchmark-suite", "content_type": "website", "platforms": [ "web" ], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/ServiceNow/webarena-verified", "repo_url": "https://github.com/ServiceNow/webarena-verified", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "description_en": "A separately maintained WebArena evaluation artifact with revised task definitions, structured response checks and offline scoring from captured network traces.", "description_zh": "独立维护的 WebArena 评测产物，提供修订任务、结构化答案检查及基于已记录网络轨迹的离线判分。", "paper_url": "https://openreview.net/forum?id=94tlGxmqkN", "enables_en": "Re-score recorded web-agent outputs with deterministic normalization and structural checks, or collect new runs using supported self-hosted sites.", "enables_zh": "通过确定性的规范化及结构检查，对已记录网页 Agent 输出重新评分，或在支持的自建网站上收集新运行。", "limits_en": "This revises the original WebArena task family. Its 812 full tasks and 258-task Hard subset in v1.2.3 are not independent tasks to add to WebArena. Offline scoring requires the expected response and trace artifacts; new trajectories still need sites and resets. Pin evaluator/data/image versions; scores need not match the original evaluator.", "limits_zh": "它修订原始 WebArena 任务族。v1.2.3 的812项完整任务及258项 Hard 子集，不是可叠加到 WebArena 的新独立任务。离线判分需指定的答案和轨迹产物；收集新轨迹仍需网站及重置。应固定评分器、数据和镜像版本，成绩不一定与原始评分器一致。", "runtime_note_en": "Python >=3.11 with CLI/package or Docker. The evaluator consumes agent responses and recorded network events. Website containers and optional map/Wikipedia downloads are separate. No container, task replay or state reset was executed.", "runtime_note_zh": "支持 Python >=3.11 的 CLI／软件包或 Docker。评分器读取 Agent 答案和网络事件记录；网站容器及可选地图／维基百科下载需另行配置。本次未启动容器、重放任务或重置状态。", "license_en": "Evaluator code and the v1.2.3 dataset card declare Apache-2.0. Underlying website software and content retain their own terms.", "license_zh": "评分代码及 v1.2.3 数据卡声明 Apache-2.0，底层网站软件与内容仍适用各自条款。", "license_url": "https://github.com/ServiceNow/webarena-verified/blob/6473f72db5dcefc97b5725b59e734504edc28a21/LICENSE", "evidence": [ { "url": "https://github.com/ServiceNow/webarena-verified/blob/6473f72db5dcefc97b5725b59e734504edc28a21/README.md", "supports": "Released CLI/evaluator, offline traces, full/Hard sets, site deployment and author paper citation." }, { "url": "https://github.com/ServiceNow/webarena-verified/blob/6473f72db5dcefc97b5725b59e734504edc28a21/src/webarena_verified/core/evaluation/value_comparator.py", "supports": "Inspectable recursive structural comparisons and assertion statuses." }, { "url": "https://github.com/ServiceNow/webarena-verified/blob/6473f72db5dcefc97b5725b59e734504edc28a21/pyproject.toml", "supports": "Python >=3.11 and package/build configuration." }, { "url": "https://github.com/ServiceNow/webarena-verified/blob/6473f72db5dcefc97b5725b59e734504edc28a21/LICENSE", "supports": "Apache-2.0 code license." }, { "url": "https://huggingface.co/datasets/AmineHA/WebArena-Verified/blob/0566051c3050bc41ee425cacc58c82528666d1b5/README.md", "supports": "Full 812/Hard 258, Apache-2.0, v1.2.3 and matching source commit." }, { "url": "https://huggingface.co/datasets/AmineHA/WebArena-Verified/blob/0566051c3050bc41ee425cacc58c82528666d1b5/version.json", "supports": "Dataset hash and source release metadata." } ] }, { "id": "clawbench", "name": "ClawBench", "resource_kind": "benchmark-suite", "content_type": "website", "platforms": [ "web" ], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/TIGER-AI-Lab/ClawBench", "repo_url": "https://github.com/TIGER-AI-Lab/ClawBench", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "description_en": "A live-website evaluation framework with everyday task specifications, browser-agent harnesses, trace capture and request-based or trajectory-based scoring.", "description_zh": "面向真实网站的评测框架，提供日常任务定义、浏览器 Agent 框架、轨迹记录及请求或轨迹评分。", "paper_url": "https://arxiv.org/abs/2604.08523", "enables_en": "Evaluate document-informed online workflows and inspect recorded actions, requests and model messages under a chosen corpus, harness and rubric.", "enables_zh": "评估需要利用文档信息的在线流程，并在明确语料版本、框架和评分规则后检查动作、请求及模型消息。", "limits_en": "Runs touch production websites and may need accounts or external services. Interception blocks only requests matching a task schema; nonmatching requests continue live, and placeholder schemas do not block submissions. A blocked request plus judge approval is not a committed real-world result. Current V1/V2 folders have 152/129 tasks; historical counts and scoring protocols differ.", "limits_zh": "运行涉及真实生产网站，可能需要账号或外部服务。拦截仅阻止匹配任务规则的请求；未匹配请求仍发送到真实网站，占位规则不会阻止提交。请求被拦截且评审通过，不代表真实结果已提交。当前 V1／V2 目录有152／129项任务，历史数量及评分协议不同。", "runtime_note_en": "Package v0.11.0 requires Python >=3.11, Docker/Podman or documented remote-browser setup, and model/judge configuration. Native interception scoring and historical V1 human-reference trajectory judging are separate protocols. Live sites, login and browser restrictions remain external dependencies.", "runtime_note_zh": "v0.11.0 软件包要求 Python >=3.11、Docker／Podman 或文档支持的远程浏览器，并配置模型／评审服务。本地拦截评分与历史 V1 人工参考轨迹评审属于不同协议。真实网站、登录及浏览器限制仍是外部依赖。", "license_en": "Repository code and inspected official task/V1-trace metadata declare Apache-2.0. This does not license third-party websites or authorize live task actions.", "license_zh": "仓库代码及已查阅的官方任务／V1轨迹元数据声明 Apache-2.0；该许可不涵盖第三方网站，也不代表获得执行真实任务的授权。", "license_url": "https://github.com/TIGER-AI-Lab/ClawBench/blob/07eb50a3d4c5a799f5ad4eebd57c67c10417787f/LICENSE", "evidence": [ { "url": "https://github.com/TIGER-AI-Lab/ClawBench/blob/07eb50a3d4c5a799f5ad4eebd57c67c10417787f/README.md", "supports": "Released harnesses, current corpus counts and historical paper-count differences." }, { "url": "https://github.com/TIGER-AI-Lab/ClawBench/blob/07eb50a3d4c5a799f5ad4eebd57c67c10417787f/docs/faq.md", "supports": "Account setup, nonmatching live requests, placeholder interception and distinct V1 trajectory scoring." }, { "url": "https://github.com/TIGER-AI-Lab/ClawBench/blob/07eb50a3d4c5a799f5ad4eebd57c67c10417787f/test-cases/v1/001-daily-life-food-uber-eats/task.json", "supports": "One inspected task uses __PLACEHOLDER_WILL_NOT_MATCH__; no task was run." }, { "url": "https://github.com/TIGER-AI-Lab/ClawBench/blob/07eb50a3d4c5a799f5ad4eebd57c67c10417787f/src/clawbench/runner/judge.py", "supports": "Public request/answer judging implementation and model configuration." }, { "url": "https://github.com/TIGER-AI-Lab/ClawBench/blob/07eb50a3d4c5a799f5ad4eebd57c67c10417787f/eval/scoring.md", "supports": "Two-stage request scoring and older corpus denominators; not a universal state-commit guarantee." }, { "url": "https://github.com/TIGER-AI-Lab/ClawBench/blob/07eb50a3d4c5a799f5ad4eebd57c67c10417787f/pyproject.toml", "supports": "Package 0.11.0, Python >=3.11 and bundled task/CLI implementation." }, { "url": "https://github.com/TIGER-AI-Lab/ClawBench/blob/07eb50a3d4c5a799f5ad4eebd57c67c10417787f/LICENSE", "supports": "Apache-2.0 code license." }, { "url": "https://huggingface.co/api/datasets/NAIL-Group/ClawBench", "supports": "Public ungated task metadata declares Apache-2.0." }, { "url": "https://huggingface.co/api/datasets/NAIL-Group/ClawBenchV1Trace", "supports": "Public ungated trace metadata declares Apache-2.0; trace contents not downloaded." } ] }, { "id": "mind2web-2", "name": "Mind2Web 2", "resource_kind": "benchmark-suite", "content_type": "website", "platforms": [ "web" ], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/OSU-NLP-Group/Mind2Web-2", "repo_url": "https://github.com/OSU-NLP-Group/Mind2Web-2", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "description_en": "An agentic-search benchmark for long-horizon web research and citation-backed synthesis, with a released judge-agent framework and versioned task-specific evaluators.", "description_zh": "面向长流程网页研究及带来源综合回答的 Agent 搜索基准，提供公开评审 Agent 框架及带版本的任务专用评分器。", "paper_url": "https://arxiv.org/abs/2506.21506", "enables_en": "Assess completeness and source attribution of synthesized answers with task-specific verification trees and cached evidence pages.", "enables_zh": "利用任务专用验证树及缓存证据网页，评估综合回答的完整性和来源归属。", "limits_en": "Distinct from offline Mind2Web action prediction and Online-Mind2Web navigation. Public Git code includes 10 development evaluators; current full task/evaluator files on Hub are auto-gated and were not accessed. Full evaluator release was announced in October 2025, so the paper-era private-test description is historical. Live pages and model judgments require version/date controls.", "limits_zh": "区别于离线 Mind2Web 动作预测和 Online-Mind2Web 导航。公开 Git 代码包含10个开发集评分器；Hub 当前完整任务／评分文件设有自动批准的访问门槛，本次未访问。完整评分器于2025年10月宣布发布，因此论文时期的私有测试集说明属于历史状态。实时网页及模型判分需固定版本和日期。", "runtime_note_en": "Python >=3.11, browser/webpage-cache services and judge models; some tasks use Google Maps APIs. The runner defaults to 2025_07_14 evaluators while Hub lists 2025_10_23 scripts. Align script release, task split and caches. No browser or evaluator was executed.", "runtime_note_zh": "需 Python >=3.11、浏览器／网页缓存服务及评审模型；部分任务使用 Google Maps API。运行器默认评分版本为2025_07_14，而 Hub 列出2025_10_23脚本。应对齐脚本版本、任务划分及缓存；本次未运行浏览器或评分器。", "license_en": "Git repository code is MIT. Public Hub metadata declares CC BY 4.0 for the gated task/evaluator distribution. Live source pages retain their own terms.", "license_zh": "Git 仓库代码采用 MIT；公开 Hub 元数据为需通过访问门槛的任务／评分分发声明 CC BY 4.0。实时来源网页仍适用各自条款。", "license_url": "https://github.com/OSU-NLP-Group/Mind2Web-2/blob/1fcbfda98bf5c669daa2ed0a27f1d2f2499475ee/LICENSE", "evidence": [ { "url": "https://github.com/OSU-NLP-Group/Mind2Web-2/blob/1fcbfda98bf5c669daa2ed0a27f1d2f2499475ee/README.md", "supports": "Search/synthesis scope, full-release announcement, caching and API requirements." }, { "url": "https://github.com/OSU-NLP-Group/Mind2Web-2/blob/1fcbfda98bf5c669daa2ed0a27f1d2f2499475ee/run_eval.py", "supports": "Substantive task evaluation dispatcher and default script version." }, { "url": "https://github.com/OSU-NLP-Group/Mind2Web-2/blob/1fcbfda98bf5c669daa2ed0a27f1d2f2499475ee/eval_scripts/dev_set/yu_lineage.py", "supports": "Public task-specific verification-tree and attribution checks." }, { "url": "https://github.com/OSU-NLP-Group/Mind2Web-2/blob/1fcbfda98bf5c669daa2ed0a27f1d2f2499475ee/eval_scripts/README.md", "supports": "Directs users to Hub for current full evaluator scripts." }, { "url": "https://huggingface.co/api/datasets/osunlp/Mind2Web-2", "supports": "gated=auto, dev_set/test_set, CC BY4.0 and 2025_10_23 filenames; gated contents not inspected." }, { "url": "https://github.com/OSU-NLP-Group/Mind2Web-2/blob/1fcbfda98bf5c669daa2ed0a27f1d2f2499475ee/pyproject.toml", "supports": "Python >=3.11 and package dependencies." }, { "url": "https://github.com/OSU-NLP-Group/Mind2Web-2/blob/1fcbfda98bf5c669daa2ed0a27f1d2f2499475ee/LICENSE", "supports": "MIT code license." }, { "url": "https://arxiv.org/abs/2506.21506", "supports": "Original paper describes 130 tasks and a historical 10-development/120-test division." } ] }, { "id": "webbench", "name": "WebBench", "resource_kind": "dataset-and-benchmark", "content_type": "website", "platforms": [ "web" ], "uses": [ "evaluation" ], "implementation": "unknown", "primary_url": "https://github.com/Halluminate/WebBench", "repo_url": "https://github.com/Halluminate/WebBench", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "description_en": "A public collection of live-website task instructions spanning reading, account changes and file operations, accompanied by published evaluation results.", "description_zh": "公开的真实网站任务指令集合，覆盖阅读、账号修改及文件操作，并附有已发布的评测结果。", "paper_url": null, "enables_en": "Select concrete browser workflows by action category and starting URL, then use a separately supplied authorized agent and outcome-review procedure.", "enables_zh": "按操作类别和起始网址选择具体浏览器流程，再使用另行配置且获得授权的 Agent 与结果评审流程。", "limits_en": "The repository publishes CSV tasks and results, not a complete resettable environment or evaluator package. WRITE tasks change real account state and need appropriate test-account authorization. The current main CSV has 2,647 records while the README reports 2,454; pin the file. All 315 HITL rows also occur in the main collection.", "limits_zh": "仓库发布 CSV 任务及结果，并未提供完整可重置环境或评分器软件包。WRITE 任务会改变真实账号状态，需相应测试账号授权。当前主 CSV 有2,647条记录，README 则写2,454；应固定文件版本。315行 HITL 记录也全部存在于主集合。", "runtime_note_en": "Bring your own browser agent, accounts where needed and task-outcome assessment. Public fields are ID, Starting URL, Category and Task; no executable outcome tests are embedded. Sites, authentication and downloads can change. No live task was attempted.", "runtime_note_zh": "需自行准备浏览器 Agent、必要账号及任务结果评估方式。公开字段为 ID、Starting URL、Category 和 Task，不包含可执行结果测试。网站、认证及下载行为可能变化；本次未尝试真实任务。", "license_en": "Repository LICENSE and official Hub card declare MIT. This does not cover third-party website content or authorize account mutations.", "license_zh": "仓库 LICENSE 及官方 Hub 数据卡声明 MIT；该许可不涵盖第三方网站内容，也不授予修改账号状态的权限。", "license_url": "https://github.com/Halluminate/WebBench/blob/ea7a1628443321363989f354401f0653e0cba6f4/LICENSE", "evidence": [ { "url": "https://github.com/Halluminate/WebBench/blob/ea7a1628443321363989f354401f0653e0cba6f4/README.md", "supports": "Task categories, historical 2,454/452 headline; no own paper or complete evaluator identified." }, { "url": "https://github.com/Halluminate/WebBench/blob/ea7a1628443321363989f354401f0653e0cba6f4/webbenchfinal.csv", "supports": "Pinned CSV has 2,647 unique IDs and 453 distinct Starting URL strings; metadata read only." }, { "url": "https://github.com/Halluminate/WebBench/blob/ea7a1628443321363989f354401f0653e0cba6f4/webbench_hitl_final.csv", "supports": "315 rows with Difficulty; all IDs and task texts match records in main CSV." }, { "url": "https://github.com/Halluminate/WebBench/blob/ea7a1628443321363989f354401f0653e0cba6f4/LICENSE", "supports": "MIT repository license." }, { "url": "https://huggingface.co/datasets/Halluminate/WebBench", "supports": "Official MIT-labeled public ungated distribution." }, { "url": "https://github.com/Halluminate/WebBench/tree/ea7a1628443321363989f354401f0653e0cba6f4", "supports": "Source root contains README/license, CSV tasks and results; no runnable evaluator established." } ] }, { "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "demo_url": null, "paper_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "notemd", "name": "note.md", "platforms": [ "macos", "linux" ], "implementation": "ai-built", "creator": "wizlijun, note.md maintainer", "description_en": "A desktop Markdown reader and editor with linked notes, separate annotation files, local vaults and configurable agent integrations.", "description_zh": "桌面 Markdown 阅读与编辑应用，提供双向链接、独立批注文件、本地资料库及可配置 Agent 集成。", "enables_en": "Read and edit Markdown, keep comments beside source documents, and share a file-based vault with search and automation tools.", "enables_zh": "阅读和编辑 Markdown，在原文旁保存独立批注，并让搜索及自动化工具共享文件式资料库。", "limits_en": "The inspected v6.1016.2 release supplies macOS and Linux beta artifacts. Windows source-build support is documented, but no Windows installer was verified in the inspected release inventory. Linux ARM desktop acceptance and some native plugins remain incomplete. Agent features need separately configured providers or local models; their costs are separate. No runtime or document round-trip testing was performed.", "limits_zh": "所查 v6.1016.2 发布提供 macOS 和 Linux beta 文件。文档说明了 Windows 源码构建，但所查发布清单未验证 Windows 安装包。Linux ARM 桌面验收及部分原生插件仍不完整。Agent 功能需另行配置服务商或本地模型，费用另计；本次未测试运行或文档往返。", "primary_url": "https://github.com/wizlijun/note.md", "repo_url": "https://github.com/wizlijun/note.md", "source_file_url": "https://github.com/wizlijun/note.md/blob/f2bdc01b2bfd0bf9f9129e144e550360aab444b8/src-tauri/src/main.rs", "download_url": "https://github.com/wizlijun/note.md/releases/tag/v6.1016.2", "provenance_source": "https://github.com/wizlijun/note.md/blob/f2bdc01b2bfd0bf9f9129e144e550360aab444b8/README.md", "provenance_note_en": "The application README explicitly attributes development and maintenance to AI coding, with human review and smoke testing. This applies to note.md application development, not authorship of every dependency or user document.", "provenance_note_zh": "应用 README 明确将开发和维护归于 AI 编码，同时保留人工审查与冒烟测试；该声明针对 note.md 开发，不涵盖每项依赖或用户文档的作者身份。", "license_en": "The application repository is Apache-2.0; dependency and plugin terms remain separate.", "license_zh": "应用仓库采用 Apache-2.0；依赖项和插件保留各自条款。", "license_url": "https://github.com/wizlijun/note.md/blob/f2bdc01b2bfd0bf9f9129e144e550360aab444b8/LICENSE", "runtime_note_en": "Tauri/Rust desktop binary with an operating-system WebView UI. macOS 13+ has separate Apple Silicon/Intel DMGs; Linux beta targets Debian 12/Omarchy. Source builds require Rust, pnpm and the sibling moraya-core checkout.", "runtime_note_zh": "采用 Tauri／Rust 桌面二进制及操作系统 WebView 界面。macOS 13+ 分别提供 Apple Silicon／Intel DMG，Linux beta 面向 Debian 12／Omarchy。源码构建需 Rust、pnpm 及相邻目录的 moraya-core。", "reviewed_version": { "source_commit": "f2bdc01b2bfd0bf9f9129e144e550360aab444b8", "release_tag": "v6.1016.2" }, "provenance_chain": [ { "url": "https://github.com/wizlijun/note.md/blob/f2bdc01b2bfd0bf9f9129e144e550360aab444b8/README.md", "relationship": "Application-specific AI-development statement and link to official product/download site." }, { "url": "https://github.com/wizlijun/note.md/blob/f2bdc01b2bfd0bf9f9129e144e550360aab444b8/src-tauri/src/main.rs", "relationship": "Actual application entry point; repository also contains frontend and Rust app source." }, { "url": "https://github.com/wizlijun/note.md/releases/tag/v6.1016.2", "relationship": "Specific released desktop artifacts, separate from source support for other platforms." } ], "distribution_evidence": { "artifact_listing_url": "https://github.com/wizlijun/note.md/releases/expanded_assets/v6.1016.2", "head_checks": [ { "url": "https://github.com/wizlijun/note.md/releases/download/v6.1016.2/note.md-6.1016.2-aarch64.dmg", "status": 200, "content_length": "14586207", "content_type": "application/octet-stream" }, { "url": "https://github.com/wizlijun/note.md/releases/download/v6.1016.2/note.md-6.1016.2-x86_64.dmg", "status": 200, "content_length": "15236603", "content_type": "application/octet-stream" }, { "url": "https://github.com/wizlijun/note.md/releases/download/v6.1016.2/note.md_6.1016.2_amd64.deb", "status": 200, "content_length": "15930234", "content_type": "application/octet-stream" }, { "url": "https://github.com/wizlijun/note.md/releases/download/v6.1016.2/note.md_6.1016.2_arm64.deb", "status": 200, "content_length": "15919278", "content_type": "application/octet-stream" } ], "method": "HTTP HEAD availability only; no installer downloaded or executed." }, "evidence": [ { "url": "https://github.com/wizlijun/note.md/blob/f2bdc01b2bfd0bf9f9129e144e550360aab444b8/README.md", "supports": "Explicit application authorship, Tauri/system-WebView architecture, platform limits and build instructions." }, { "url": "https://github.com/wizlijun/note.md/blob/f2bdc01b2bfd0bf9f9129e144e550360aab444b8/package.json", "supports": "Version 6.1016.2, pinned pnpm and local moraya-core dependency." }, { "url": "https://github.com/wizlijun/note.md/blob/f2bdc01b2bfd0bf9f9129e144e550360aab444b8/LICENSE", "supports": "Apache-2.0 license text." }, { "url": "https://github.com/wizlijun/note.md/releases/tag/v6.1016.2", "supports": "Public macOS DMGs and Linux beta packages; file availability separately checked." }, { "url": "https://notemd.net/", "supports": "Official product scope and provider-cost/platform caveats." } ] }, { "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "demo_url": null, "paper_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "rolltake", "name": "Rolltake — Teleprompter", "platforms": [ "ios" ], "implementation": "ai-built", "creator": "Griffin Wooldridge", "description_en": "An iPhone teleprompter with script editing, text/Markdown import, adjustable scrolling, mirror modes and Bluetooth remote controls.", "description_zh": "iPhone 提词器，提供脚本编辑、文本／Markdown 导入、滚动调节、镜像模式及蓝牙遥控。", "enables_en": "Prepare a script, adjust its reading layout and scroll it during rehearsal or presentation.", "enables_zh": "准备脚本、调整阅读布局，并在排练或演示时滚动显示。", "limits_en": "Requires iOS 17+. The reviewed store listing is an iPhone app; Apple Silicon compatibility does not establish a native Mac edition. Camera/Watch capabilities discussed in the creator article were not independently checked and are not used to expand this entry. No script, remote-control or privacy behavior was tested.", "limits_zh": "需要 iOS 17+。所查商店版本面向 iPhone，Apple Silicon 兼容性不代表原生 Mac 版本。作者文章提及的相机／Watch 功能未独立核查，不用于扩展本条目范围。本次未测试脚本、遥控或隐私行为。", "primary_url": "https://apps.apple.com/us/app/rolltake-teleprompter/id6806368467", "repo_url": null, "download_url": "https://apps.apple.com/us/app/rolltake-teleprompter/id6806368467", "provenance_source": "https://griffinwooldridge.com/blog/build-ios-app-with-claude-code", "provenance_note_en": "Griffin reports that Claude Code wrote all application Swift while he made product/design decisions and tested on hardware. The account links the exact shipped store ID; it does not assign AI authorship to Apple frameworks.", "provenance_note_zh": "Griffin 报告应用 Swift 代码由 Claude Code 编写，自己负责产品／设计决策及真机测试。文章直接链接已上架的商店 ID，声明不包括 Apple 框架的作者身份。", "license_en": "No public application source or reusable code license was established in the reviewed sources; the verified artifact is the store distribution.", "license_zh": "所查来源未确立公开应用源码或可复用代码许可；已核实的产物是商店分发版本。", "runtime_note_en": "Native SwiftUI iPhone application according to the creator; install through the App Store. The inspected listing is free and English-only. No Apple account was used during review.", "runtime_note_zh": "作者说明这是原生 SwiftUI iPhone 应用，通过 App Store 安装。所查版本免费且仅列英语；核查未使用 Apple 账户。", "reviewed_version": { "app_store_id": "6806368467", "version": "1.0", "minimum_ios": "17.0" }, "provenance_chain": [ { "url": "https://griffinwooldridge.com/blog/build-ios-app-with-claude-code", "relationship": "First-person implementation account explicitly names Rolltake and links App Store ID 6806368467." }, { "url": "https://apps.apple.com/us/app/rolltake-teleprompter/id6806368467", "relationship": "Live listing identifies seller GRIFFIN MICHAEL WOOLDRIDGE and links rolltake.app." } ], "evidence": [ { "url": "https://griffinwooldridge.com/blog/build-ios-app-with-claude-code", "supports": "Creator-owned account of Swift implementation, human decisions/testing and exact distribution link." }, { "url": "https://apps.apple.com/us/app/rolltake-teleprompter/id6806368467", "supports": "Current iPhone workflow, seller, minimum OS and free distribution." }, { "url": "https://itunes.apple.com/lookup?id=6806368467&country=us", "supports": "Official lookup confirms bundle com.griffin.rolltake, version 1.0 and iOS 17.0 minimum." } ] }, { "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "demo_url": null, "paper_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "review-books", "name": "Review Books: Rate, Rank, Log", "platforms": [ "ios" ], "implementation": "ai-assisted", "creator": "whi / Hitoshi watanabe", "description_en": "An iPhone personal-review log with custom fields, photos, pairwise rankings, search and JSON import/export.", "description_zh": "iPhone 个人评价记录应用，提供自定义字段、照片、两两比较排名、搜索及 JSON 导入导出。", "enables_en": "Create category-specific review books and compare personal favorites without relying on a predefined product database.", "enables_zh": "按类别建立自定义评价册，并比较个人偏好，无需依赖预设商品数据库。", "limits_en": "Requires iOS 16.4+; languages are Japanese and English. The free app includes ads, and its store privacy declaration lists tracking-related data. Local review storage must not be described as a blanket no-network guarantee. No public app source or ranking/export correctness was verified.", "limits_zh": "需要 iOS 16.4+，提供日语和英语。免费版含广告，商店隐私声明列出与跟踪有关的数据；本地评价存储不能概括为完全不联网。本次未确立公开应用源码，也未验证排名或导出正确性。", "primary_url": "https://apps.apple.com/us/app/review-books-rate-rank-log/id6809986409", "repo_url": null, "download_url": "https://apps.apple.com/us/app/review-books-rate-rank-log/id6809986409", "provenance_source": "https://note.com/made_by_whi/n/n9735ba40681e", "provenance_note_en": "The creator describes producing most design notes, code and tests through Claude Code, while choosing requirements and testing. The post links the exact store app and its matching whi product site; website source is not treated as application source.", "provenance_note_zh": "作者描述通过 Claude Code 完成大部分设计记录、代码及测试，自己选择需求并试用核查。帖子链接确切商店应用及相应 whi 产品页；网站源码不视为应用源码。", "license_en": "No public application source or reusable code license was established in the reviewed sources; the verified artifact is the store distribution.", "license_zh": "所查来源未确立公开应用源码或可复用代码许可；已核实的产物是商店分发版本。", "runtime_note_en": "Creator-described Expo/React Native application with local SQLite review storage; App Store distribution for iPhone. Advertising is an additional external-service path.", "runtime_note_zh": "作者说明应用使用 Expo／React Native，并以本地 SQLite 保存评价记录；通过 App Store 向 iPhone 分发，广告另有外部服务路径。", "reviewed_version": { "app_store_id": "6809986409", "version": "1.1.1", "minimum_ios": "16.4" }, "provenance_chain": [ { "url": "https://note.com/made_by_whi/n/n9735ba40681e", "relationship": "First-person AI-development account names the app and links App Store ID 6809986409." }, { "url": "https://apps.apple.com/us/app/review-books-rate-rank-log/id6809986409", "relationship": "Live listing names Hitoshi watanabe, copyrights whi, and links the same whi-dev product site." }, { "url": "https://whi-dev.github.io/review-books-site/", "relationship": "Matching product/distribution page only; no inference of native application source publication." } ], "evidence": [ { "url": "https://note.com/made_by_whi/n/n9735ba40681e", "supports": "Application-specific development contribution, React Native/SQLite stack and release chain." }, { "url": "https://apps.apple.com/us/app/review-books-rate-rank-log/id6809986409", "supports": "Current features, ads, privacy declaration, languages, seller and iOS requirement." }, { "url": "https://itunes.apple.com/lookup?id=6809986409&country=us", "supports": "Official lookup confirms com.whi.reviewbooks, version 1.1.1 and iOS 16.4 minimum." } ] }, { "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "demo_url": null, "paper_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "nami-daily-flow", "name": "Nami — Daily Flow", "platforms": [ "ios" ], "implementation": "ai-assisted", "creator": "Yannik Zuehlke / Tecton & Tide", "description_en": "An iPhone/iPad habit tracker centered on three habits, daily check-ins, reminders and progress history.", "description_zh": "围绕三个习惯设计的 iPhone／iPad 习惯记录应用，提供每日打卡、提醒和进展历史。", "enables_en": "Track a small habit set and inspect consistency, with optional Pro widgets, statistics and iCloud synchronization.", "enables_zh": "记录少量习惯并查看坚持情况，可选 Pro 功能包括小组件、统计及 iCloud 同步。", "limits_en": "The current store requires iOS/iPadOS 26.2+. Pro features use a one-time in-app purchase. The earlier launch account described iPhone-only local storage without CloudKit, but the current listing includes iPad and Pro iCloud sync; this entry follows the current distribution. App behavior, reminders and synchronization were not independently tested.", "limits_zh": "当前商店要求 iOS／iPadOS 26.2+；Pro 功能通过一次性内购解锁。早期开发文章描述仅面向 iPhone、没有 CloudKit 的本地存储，当前商店则列出 iPad 及 Pro iCloud 同步；本条目采用当前分发范围。本次未独立测试应用行为、提醒或同步功能。", "primary_url": "https://apps.apple.com/us/app/nami-daily-flow/id6757530125", "repo_url": null, "download_url": "https://apps.apple.com/us/app/nami-daily-flow/id6757530125", "provenance_source": "https://tectontide.com/en/blog/building-nami-ios-app-with-ai/", "provenance_note_en": "Yannik credits Claude Code with specification/design work and Codex CLI with Swift feature implementation and tests, under his direction. His detailed article links the exact Nami store app, whose seller matches him.", "provenance_note_zh": "Yannik 将规格／设计工作归于 Claude Code，将 Swift 功能实现与测试归于 Codex CLI，并由自己指导。详细文章链接确切的 Nami 商店应用，卖方与作者对应。", "license_en": "No public application source or reusable code license was established in the reviewed sources; the verified artifact is the store distribution.", "license_zh": "所查来源未确立公开应用源码或可复用代码许可；已核实的产物是商店分发版本。", "runtime_note_en": "Creator-described SwiftUI/SwiftData app. The store lists English/German and optional paid Pro functionality; no Mac-native implementation is inferred from Apple Silicon compatibility.", "runtime_note_zh": "作者说明使用 SwiftUI／SwiftData。商店列出英语／德语及可选付费 Pro 功能，不因 Apple Silicon 兼容性推断存在原生 Mac 实现。", "reviewed_version": { "app_store_id": "6757530125", "version": "1.0", "minimum_ios": "26.2" }, "provenance_chain": [ { "url": "https://tectontide.com/en/blog/building-nami-ios-app-with-ai/", "relationship": "Named author describes concrete AI implementation/test work and links store ID 6757530125." }, { "url": "https://apps.apple.com/us/app/nami-daily-flow/id6757530125", "relationship": "Live seller YANNIK ZUHLKE matches the author; developer links return to tectontide.com." } ], "evidence": [ { "url": "https://tectontide.com/en/blog/building-nami-ios-app-with-ai/", "supports": "First-person division of implementation work and exact shipped-artifact link." }, { "url": "https://apps.apple.com/us/app/nami-daily-flow/id6757530125", "supports": "Current three-habit workflow, iPad support, Pro purchase/sync and OS minimum." }, { "url": "https://itunes.apple.com/lookup?id=6757530125&country=us", "supports": "Official lookup confirms bundle com.z5e.nami, version 1.0 and iOS 26.2 minimum." } ] }, { "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "demo_url": null, "paper_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "angle-calculator-tanaka", "name": "Angle-Calculator", "platforms": [ "android" ], "implementation": "ai-assisted", "creator": "Shin / TANAKA KOGYO Lab", "description_en": "An Android right-triangle calculator with a diagram, configurable decimal precision and named saved calculations.", "description_zh": "Android 直角三角形计算器，提供示意图、可配置小数精度及命名保存的计算记录。", "enables_en": "Enter known sides and angles, inspect the calculated triangle and recall previous values for repeated work.", "enables_zh": "输入已知边长与角度，查看三角形计算结果，并为重复工作调取之前的数据。", "limits_en": "The listing specifies Android 7.0+. Calculation accuracy, edge cases and suitability for manufacturing decisions were not independently tested. The creator reports substantial generated logic but also human coding and build/signing work; no whole-app authorship percentage is adopted. Public app source and reusable code licensing were not established.", "limits_zh": "商店说明要求 Android 7.0+。本次未独立测试计算准确性、边界情形或制造决策适用性。作者报告大量逻辑由 AI 生成，同时保留人工编码及构建／签名工作；不采用整个应用的 AI 编写比例。未确立公开应用源码或可复用代码许可。", "primary_url": "https://play.google.com/store/apps/details?id=com.anglecalc.app", "repo_url": null, "download_url": "https://play.google.com/store/apps/details?id=com.anglecalc.app", "provenance_source": "https://note.com/royal_shin1960/n/n610f38a76665", "provenance_note_en": "Shin identifies Claude Code as the implementation/error-fixing tool and describes human Android Studio build/signing steps. The development article directly links the TANAKA KOGYO Lab package com.anglecalc.app.", "provenance_note_zh": "Shin 将 Claude Code 列为实现和修错工具，并描述人工执行的 Android Studio 构建／签名步骤；开发文章直接链接 TANAKA KOGYO Lab 的 com.anglecalc.app。", "license_en": "No public application source or reusable code license was established in the reviewed sources; the verified artifact is the store distribution.", "license_zh": "所查来源未确立公开应用源码或可复用代码许可；已核实的产物是商店分发版本。", "runtime_note_en": "Native Android application according to the creator; Google Play supplies the release. The developer describes offline calculations and device-local storage; these privacy/runtime properties were not independently audited.", "runtime_note_zh": "作者说明这是原生 Android 应用，由 Google Play 提供发布。开发者描述离线计算及设备本地存储，本次未独立审计这些隐私或运行属性。", "reviewed_version": { "android_package": "com.anglecalc.app", "store_updated_at": "2025-12-17", "minimum_android": "7.0 (developer listing)" }, "provenance_chain": [ { "url": "https://note.com/royal_shin1960/n/n610f38a76665", "relationship": "Creator explains building the specific trigonometric app with Claude Code and supplies its exact Play link." }, { "url": "https://play.google.com/store/apps/details?id=com.anglecalc.app", "relationship": "Live Angle-Calculator distribution by TANAKA KOGYO Lab with matching function and Android compatibility." } ], "evidence": [ { "url": "https://note.com/royal_shin1960/n/n610f38a76665", "supports": "Specific AI implementation contribution, retained human work and exact release link." }, { "url": "https://play.google.com/store/apps/details?id=com.anglecalc.app", "supports": "Current artifact identity, right-triangle functions, saved calculations and Android 7.0 requirement." } ] }, { "content_type": "app", "uses": [ "showcase" ], "resource_kind": "ai-generated-output", "demo_url": null, "paper_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "curate-rank-your-media", "name": "Curate: Rank Your Media", "platforms": [ "android" ], "implementation": "ai-assisted", "creator": "Michael Ethan Berkowitz / meberko; creator post u/slashx14", "description_en": "An Android media log that ranks films, television, games and books through pairwise comparisons instead of fixed star scores.", "description_zh": "Android 媒体记录应用，通过两两比较来排列电影、电视剧、游戏和书籍，而非固定星级。", "enables_en": "Search external media catalogs, add items to a personal collection and refine rankings through preference comparisons.", "enables_zh": "搜索外部媒体目录、将项目加入个人收藏，并通过偏好比较调整排名。", "limits_en": "The developer started with handwritten Android/Kotlin code and later used Claude Code; this is a mixed-development project. Media lookup uses TMDB, RAWG, Google Books and Open Library; optional device sync uses Google Sign-In. An iOS launch is mentioned as preparation, not a verified released platform. Ranking, offline and sync behavior were not tested here.", "limits_zh": "开发者先手写 Android／Kotlin 代码，之后使用 Claude Code，因此属于混合开发项目。媒体检索使用 TMDB、RAWG、Google Books 和 Open Library，可选跨设备同步使用 Google 登录。iOS 发布仅被提为准备工作，不列为已验证发布平台。本次未测试排名、离线或同步行为。", "primary_url": "https://play.google.com/store/apps/details?id=com.meberko.curate", "repo_url": null, "download_url": "https://play.google.com/store/apps/details?id=com.meberko.curate", "provenance_source": "https://www.reddit.com/r/ClaudeCode/comments/1rttpr1/sharing_my_claude_coded_media_ranking_app/", "provenance_note_en": "The creator reports an initially handwritten app followed by Claude-generated implementation that he reviewed. The same first-person post links package com.meberko.curate, connecting the contribution claim to the released app rather than an AI-powered feature.", "provenance_note_zh": "作者报告应用最初由自己手写，之后使用并审查 Claude 生成的实现。同一篇第一人称帖子链接 com.meberko.curate，将开发贡献声明对应到已发布应用，而非仅说明应用具有 AI 功能。", "license_en": "No public application source or reusable code license was established in the reviewed sources; the verified artifact is the store distribution.", "license_zh": "所查来源未确立公开应用源码或可复用代码许可；已核实的产物是商店分发版本。", "runtime_note_en": "Google Play Android release; current store update is March 14, 2026. Local use and optional account sync are documented by the developer. The minimum Android version was not established from the inspected public listing.", "runtime_note_zh": "Google Play 的 Android 发布，商店所列更新日期为 2026 年 3 月 14 日。开发者记录了本地使用及可选账号同步，所查公开商店页未确立最低 Android 版本。", "reviewed_version": { "android_package": "com.meberko.curate", "store_updated_at": "2026-03-14" }, "provenance_chain": [ { "url": "https://www.reddit.com/r/ClaudeCode/comments/1rttpr1/sharing_my_claude_coded_media_ranking_app/", "relationship": "Creator explains handwritten beginnings and subsequent Claude Code work, then links the exact Play package." }, { "url": "https://play.google.com/store/apps/details?id=com.meberko.curate", "relationship": "Live listing identifies meberko and Michael Ethan Berkowitz and documents the matching pairwise-media-ranking product." } ], "evidence": [ { "url": "https://www.reddit.com/r/ClaudeCode/comments/1rttpr1/sharing_my_claude_coded_media_ranking_app/", "supports": "First-party scoped development account and exact distribution link." }, { "url": "https://play.google.com/store/apps/details?id=com.meberko.curate", "supports": "Current Android release, developer, external catalogs, optional sync and iOS preparation note." } ] }, { "id": "agent-world-model", "name": "Agent World Model", "content_type": "app", "platforms": [], "uses": [ "training", "evaluation" ], "implementation": "unknown", "resource_kind": "environment-framework", "primary_url": "https://github.com/Snowflake-Labs/agent-world-model", "repo_url": "https://github.com/Snowflake-Labs/agent-world-model", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "description_en": "A generation pipeline and released suite of synthetic, SQL-backed tool environments exposed through MCP for multi-turn agent research.", "description_zh": "用于多轮 Agent 研究的生成流水线及已发布合成工具环境套件，通过 MCP 提供接口并以 SQL 数据库维护状态。", "paper_url": "https://arxiv.org/abs/2602.10090", "enables_en": "Train tool-use agents in generated scenarios and check resulting database state using either code verifiers or code-augmented model judges.", "enables_zh": "在生成场景中训练工具使用 Agent，并通过代码评分器或结合代码的模型评审检查数据库结果状态。", "limits_en": "These are synthetic SQLite/MCP scenarios, not real production services. The release distinguishes 1,000 environments from 10,000 tasks. Pure-code and SQL-plus-model verification produce different evidence and must be identified. OpenEnv is an integration, not another task set; its done action does not run verification. No explicit pipeline-code license was located in inspected project metadata.", "limits_zh": "这些是合成 SQLite／MCP 场景，并非真实生产服务；发布资料区分 1,000 个环境和 10,000 项任务。纯代码与 SQL 加模型评审提供不同证据，应注明模式。OpenEnv 属于集成，不是另一套任务；其 done 动作不会自动评测。所查项目元数据未明确流水线代码许可。", "runtime_note_en": "Python >=3.12, uv-managed dependencies, generated databases and MCP servers. Synthesis, agents and SQL-mode judges require configured models; pure-code verification does not need a judge model. The OpenEnv integration executes supplied scenario code and relies on container isolation. No environment code was executed here.", "runtime_note_zh": "需 Python >=3.12、uv 管理的依赖、生成数据库及 MCP 服务器。合成、Agent 和 SQL 模式评审需配置模型；纯代码评测不需要评审模型。OpenEnv 集成会执行所提供的场景代码，并依赖容器隔离。本次未执行环境代码。", "license_en": "AgentWorldModel-1K declares CC BY 4.0. No standalone license or license field was found in the inspected pipeline repository root, README or pyproject. The OpenEnv wrapper's license must not be substituted for the upstream pipeline's missing declaration.", "license_zh": "AgentWorldModel-1K 声明 CC BY 4.0；所查流水线仓库根目录、README 及 pyproject 未提供独立许可或许可字段。不能用 OpenEnv 封装的许可替代上游流水线缺失的声明。", "license_url": "https://huggingface.co/datasets/Snowflake/AgentWorldModel-1K/blob/dde80a0283fe781bdc51656bce57063dc5650213/README.md", "provenance_note_en": "The authors report generating environment code with models. This does not establish that the synthesis toolkit itself was AI-built; the project-level implementation label remains unknown.", "provenance_note_zh": "作者说明使用模型生成环境代码，但这不能证明合成工具包本身由 AI 开发，因此项目级开发方式仍标为未知。", "evidence": [ { "url": "https://github.com/Snowflake-Labs/agent-world-model/blob/85e322f69279e3b3325b7377ec3bab788514e9cb/README.md", "supports": "Canonical pipeline, 1,000 environments, interaction CLI and separate verification modes." }, { "url": "https://github.com/Snowflake-Labs/agent-world-model/blob/85e322f69279e3b3325b7377ec3bab788514e9cb/awm/cli.py", "supports": "Public generation, environment, agent and verification command dispatch." }, { "url": "https://github.com/Snowflake-Labs/agent-world-model/blob/85e322f69279e3b3325b7377ec3bab788514e9cb/awm/core/agent.py", "supports": "Substantive MCP tool-use agent and local scenario-server interaction code." }, { "url": "https://github.com/Snowflake-Labs/agent-world-model/blob/85e322f69279e3b3325b7377ec3bab788514e9cb/awm/core/verify.py", "supports": "Code-mode verification versus SQL evidence plus an LLM judge." }, { "url": "https://github.com/Snowflake-Labs/agent-world-model/blob/85e322f69279e3b3325b7377ec3bab788514e9cb/pyproject.toml", "supports": "Python >=3.12 and package 0.1.0; no license field." }, { "url": "https://huggingface.co/datasets/Snowflake/AgentWorldModel-1K/blob/dde80a0283fe781bdc51656bce57063dc5650213/README.md", "supports": "CC BY 4.0; 1,000 scenario rows, ten tasks per scenario and two sets of 10K verifiers." }, { "url": "https://github.com/huggingface/OpenEnv/blob/55347f62e22f83a618b49ee984e2f684b69d8fbd/envs/agent_world_model_env/README.md", "supports": "Pinned integration explicitly identifies Snowflake upstream; done/verify distinction and runtime isolation boundary." }, { "url": "https://github.com/huggingface/OpenEnv/blob/55347f62e22f83a618b49ee984e2f684b69d8fbd/envs/agent_world_model_env/openenv.yaml", "supports": "Exact agent_world_model_env registry manifest identity." } ] }, { "id": "scienceagentbench", "name": "ScienceAgentBench", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "resource_kind": "environment-and-benchmark", "primary_url": "https://github.com/OSU-NLP-Group/ScienceAgentBench", "repo_url": "https://github.com/OSU-NLP-Group/ScienceAgentBench", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "description_en": "An executable benchmark for generating standalone Python programs that perform data-driven scientific analyses, with task assets and direct or containerized evaluation.", "description_zh": "面向独立 Python 科学分析程序生成的可执行基准，提供任务资产以及直接运行或容器化评测流程。", "paper_url": "https://arxiv.org/abs/2410.05080", "enables_en": "Evaluate whether generated programs produce task-specific scientific outputs, using execution checks and visual judging where required.", "enables_zh": "通过执行检查及必要的视觉评审，评估生成程序是否产生任务要求的科学分析输出。", "limits_en": "Use the verified split with matching benchmark_verified assets; it was announced after the frozen 102-task Harbor package. The public runner still defaults to validation, so select the release explicitly. Some outputs use model-based visual judging and results can depend on scientific-library versions or randomness. Full assets are separately hosted and were not downloaded; authors request no redistribution of unpacked files.", "limits_zh": "应使用 verified 划分及匹配的 benchmark_verified 资产；其发布时间晚于冻结的102项 Harbor 包。公开运行器仍默认 validation，因此需明确选择版本。部分输出采用模型视觉评审，结果会受科学库版本或随机性影响。完整资产另行托管，本次未下载；作者要求不要重新公开解压文件。", "runtime_note_en": "The documented setup uses Python 3.10 and Docker evaluation or per-task Conda environments. Provide the benchmark assets and configured judge/model access. Task-specific dependencies, compute and memory affect reproduction; no programs, images or scientific datasets were run.", "runtime_note_zh": "文档使用 Python 3.10，可通过 Docker 或各任务的 Conda 环境评测。需准备基准资产及评审／模型访问；各任务依赖、算力和内存会影响复现。本次未运行程序、镜像或科学数据集。", "license_en": "Repository code is MIT. Most task material is CC BY 4.0; rasterio- and matminer-derived tasks retain their original licenses. The README's unpacked-file redistribution request is recorded separately from those license declarations.", "license_zh": "仓库代码为 MIT，多数任务材料为 CC BY4.0；来自 rasterio 和 matminer 的任务保留原许可。README 对解压文件再分发的要求与这些许可声明分别记录。", "license_url": "https://github.com/OSU-NLP-Group/ScienceAgentBench/blob/c26e151ed601ba109dc4d35e057ff8e73fec469d/LICENSE", "evidence": [ { "url": "https://github.com/OSU-NLP-Group/ScienceAgentBench/blob/c26e151ed601ba109dc4d35e057ff8e73fec469d/README.md", "supports": "102 tasks, standalone program outputs, April 30 verified release, setup/assets and source-specific licenses." }, { "url": "https://github.com/OSU-NLP-Group/ScienceAgentBench/blob/c26e151ed601ba109dc4d35e057ff8e73fec469d/evaluation/harness/run_evaluation.py", "supports": "Released Docker evaluator; default split remains validation." }, { "url": "https://github.com/OSU-NLP-Group/ScienceAgentBench/blob/c26e151ed601ba109dc4d35e057ff8e73fec469d/gpt4_visual_judge.py", "supports": "Public visual-output judging code uses a configured GPT-4o service." }, { "url": "https://github.com/OSU-NLP-Group/ScienceAgentBench/blob/c26e151ed601ba109dc4d35e057ff8e73fec469d/LICENSE", "supports": "MIT repository license." }, { "url": "https://huggingface.co/api/datasets/osunlp/ScienceAgentBench", "supports": "Public ungated metadata declares 102 verified examples and a separate legacy validation config." }, { "url": "https://github.com/laude-institute/harbor/blob/3e9f14a7c1033896cc600d27de69d6948bcb3410/adapters/scienceagentbench/README.md", "supports": "Canonical upstream link; adapter distinguishes deterministic/visual checks and documents patched-harness comparisons, resource limits and stochastic cases." } ] }, { "id": "swe-bench-pro", "name": "SWE-bench Pro", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "resource_kind": "benchmark-suite", "primary_url": "https://github.com/scaleapi/SWE-bench_Pro-os", "repo_url": "https://github.com/scaleapi/SWE-bench_Pro-os", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "description_en": "Scale AI's long-horizon repository-repair benchmark, with released public task containers, issue specifications and patch-verification tooling.", "description_zh": "Scale AI 的长流程代码库修复基准，公开任务容器、问题说明及补丁验证工具。", "paper_url": "https://arxiv.org/abs/2509.16941", "enables_en": "Evaluate patches against task-specific failing and regression tests across Python, JavaScript/TypeScript and Go repositories.", "enables_zh": "在 Python、JavaScript／TypeScript 和 Go 代码库中，通过任务专用失败测试及回归测试评估补丁。", "limits_en": "Current public V2 has 642 tasks; HARD-51 is a subset, and legacy v1 has 731. The paper's broader collection includes nonpublic tracks. Frozen Harbor packages still advertise 731 and are not proven equivalent to V2. V2's reported protocol keeps the agent phase offline and re-grades captured patches in fresh containers. Do not combine versions, subsets or namespaces into an additive task count.", "limits_zh": "当前公开 V2 有642项任务，HARD-51 是其子集，旧 v1 有731项；论文更大的集合包含未公开赛道。冻结 Harbor 包仍标731，尚不能视为 V2 的等价发布。V2 协议让 Agent 阶段离线，并在全新容器中重评捕获的补丁。版本、子集和不同命名空间不能累加为任务总数。", "runtime_note_en": "V2 ships Harbor task directories and public linux/amd64 image references. The documented locked protocol uses Harbor with Modal's phase-specific network policy and a separate re-grader. Legacy local-Docker examples belong to v1. Container pulls, task execution and reference-patch claims were not independently tested.", "runtime_note_zh": "V2 提供 Harbor 任务目录及公开 linux/amd64 镜像引用。文档中的固定协议使用 Harbor、Modal 分阶段网络策略及独立重评分器；旧的本地 Docker 示例属于 v1。本次未拉取容器、执行任务或独立验证参考补丁结果。", "license_en": "Current harness/tooling is MIT. Task instructions, patches and tests retain the upstream repositories' licenses, including copyleft where applicable. The older Harbor adapter's Apache-2.0 statement does not establish uniform rights for the current release.", "license_zh": "当前框架／工具采用 MIT；任务指令、补丁及测试保留原代码库许可，其中可能包含 copyleft 条款。旧 Harbor 适配器的 Apache-2.0 说明不能确立当前发布材料统一适用该许可。", "license_url": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/66f92766bba642462d4bbe5479e83f91f9211862/LICENSE", "evidence": [ { "url": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/66f92766bba642462d4bbe5479e83f91f9211862/README.md", "supports": "Canonical public V2/v1 separation, task counts, public images and separate private leaderboard." }, { "url": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/66f92766bba642462d4bbe5479e83f91f9211862/v2/README.md", "supports": "642 tasks across 11 repositories, HARD-51, offline agent phase and fresh-container grading." }, { "url": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/66f92766bba642462d4bbe5479e83f91f9211862/v2/tooling/patch_replay.py", "supports": "Public implementation of separate patch replay grading; source inspection only." }, { "url": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/66f92766bba642462d4bbe5479e83f91f9211862/v2/hard51_ids.txt", "supports": "Exactly 51 unique listed instance IDs." }, { "url": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/66f92766bba642462d4bbe5479e83f91f9211862/LICENSE", "supports": "MIT current harness license." }, { "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro/blob/2d52cb3df914a3fcf80c7f66738b3a88ae37fc50/README.md", "supports": "V2 default 642, hard51, v1 731 and task-content source-specific licensing." }, { "url": "https://github.com/laude-institute/harbor/blob/3e9f14a7c1033896cc600d27de69d6948bcb3410/adapters/swebenchpro/README.md", "supports": "Legacy731-task adapter, changed test commands and an older incompatible blanket license statement." }, { "url": "https://hub.harborframework.com/datasets/cais/swebenchpro/latest", "supports": "CAIS describes a modified731-task SWE-bench Pro package; related family, not identical runtime." }, { "url": "https://hub.harborframework.com/datasets/scale-ai/swe-bench-pro/latest", "supports": "Package README identifies SWE-bench Pro and links the primary Scale paper/leaderboard." } ] }, { "id": "algotune", "name": "AlgoTune", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "resource_kind": "benchmark-suite", "primary_url": "https://algotune.io/", "repo_url": "https://github.com/oripress/AlgoTune", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "description_en": "A benchmark and execution harness for optimizing numerical and algorithmic programs while preserving valid outputs, measured against reference runtimes.", "description_zh": "面向数值及算法程序优化的基准和执行框架，在保持输出有效的前提下与参考运行时间比较。", "paper_url": "https://arxiv.org/abs/2507.15887", "enables_en": "Measure correctness-preserving solver speedups on generated held-out inputs, using task-specific generators, validators and reference solvers.", "enables_zh": "使用任务专用输入生成器、验证器和参考解法，在生成的保留输入上衡量保持正确性的求解器加速。", "limits_en": "Timing depends on hardware, library versions, calibration and parallel contention. The 154 task families each generate many inputs; input counts are not additional tasks. Original aggregate speedup, AlgoTune-Pass resolve rate and Harbor rewards are different measures. Harbor documents a 1.0 floor for invalid or slower solutions, so that reward alone does not prove correctness.", "limits_zh": "计时取决于硬件、库版本、校准及并行资源竞争。154个任务族各自生成多组输入，输入数量不能当作新增任务。原始综合加速比、AlgoTune-Pass 完成率与 Harbor 奖励属于不同指标。Harbor 对无效或更慢的解法也记录1.0下限，因此该奖励本身不能证明正确性。", "runtime_note_en": "Python >=3.10 and numerical dependencies; documented launchers support standalone, SLURM and optional AWS paths. AlgoTuner requires model configuration. Preserve train/test input seeds, problem-size calibration, CPU/thread limits and correctness traces. No solver, timing benchmark or cloud resource was run.", "runtime_note_zh": "需 Python >=3.10 及数值计算依赖；文档提供独立运行、SLURM 和可选 AWS 路径。AlgoTuner 需配置模型。应保留训练／测试输入种子、问题规模校准、CPU／线程限制及正确性记录。本次未运行解法、计时评测或云资源。", "license_en": "Repository benchmark/harness code is MIT. Numerical libraries and other dependencies retain their own licenses.", "license_zh": "仓库中的基准／框架代码采用 MIT，数值库及其他依赖保留各自许可。", "license_url": "https://github.com/oripress/AlgoTune/blob/dff9914c10800c7a031c9e8c3d4d1c8cd1b38906/LICENSE", "evidence": [ { "url": "https://algotune.io/", "supports": "Original metric is harmonic-mean speedup on validated held-out inputs; task API is described." }, { "url": "https://github.com/oripress/AlgoTune/blob/dff9914c10800c7a031c9e8c3d4d1c8cd1b38906/README.md", "supports": "154 function tasks and separate AlgoTuner agent; standalone/SLURM/AWS setup." }, { "url": "https://github.com/oripress/AlgoTune/blob/dff9914c10800c7a031c9e8c3d4d1c8cd1b38906/AlgoTuneTasks/base.py", "supports": "Public task generator/validator/reference-solver and calibrated train/test input machinery." }, { "url": "https://github.com/oripress/AlgoTune/blob/dff9914c10800c7a031c9e8c3d4d1c8cd1b38906/scripts/algotune.py", "supports": "Substantive launcher for timing and agent runs; source inspected only." }, { "url": "https://github.com/oripress/AlgoTune/blob/dff9914c10800c7a031c9e8c3d4d1c8cd1b38906/pyproject.toml", "supports": "Python >=3.10 package metadata." }, { "url": "https://github.com/oripress/AlgoTune/blob/dff9914c10800c7a031c9e8c3d4d1c8cd1b38906/LICENSE", "supports": "MIT license." }, { "url": "https://github.com/laude-institute/harbor/blob/3e9f14a7c1033896cc600d27de69d6948bcb3410/adapters/algotune/README.md", "supports": "Canonical upstream identity and explicit hardware, calibration, reward-floor and AlgoTune-Pass differences." } ] }, { "id": "ds-1000", "name": "DS-1000", "content_type": "app", "platforms": [], "uses": [ "evaluation" ], "implementation": "unknown", "resource_kind": "benchmark-suite", "primary_url": "https://github.com/xlang-ai/DS-1000", "repo_url": "https://github.com/xlang-ai/DS-1000", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "description_en": "An execution-tested data-science code-generation benchmark covering 1,000 Python problems across seven widely used libraries.", "description_zh": "覆盖七种常用库、包含1,000道 Python 问题的数据科学代码生成基准，通过执行测试判分。", "paper_url": "https://arxiv.org/abs/2211.11501", "enables_en": "Check generated code using functional test cases and task-specific code-form constraints, and report accuracy by library or problem type.", "enables_zh": "通过功能测试及任务专用代码形式约束检查生成代码，并按库或问题类型报告准确率。", "limits_en": "This evaluates scoped code solutions, not complete application development or GUI navigation. The simplified release preserves problem content but changes packaging and removes the original infilling mode. It executes generated code and needs appropriate isolation; separate processes also matter for stateful libraries. Pin dataset and dependency versions when comparing original and Harbor runs.", "limits_zh": "它评估限定范围的代码解法，而非完整应用开发或 GUI 导航。简化发布保留问题内容，但改变封装并移除原始填空模式。评测会执行生成代码，需要适当隔离；对有状态库，独立进程同样重要。比较原始及 Harbor 运行时应固定数据与依赖版本。", "runtime_note_en": "The current environment file pins Python 3.10 and scientific-library versions. The released runner launches per-problem execution checks, plus test_string where supplied. The original-format branch and simplified test split are two protocols for the same benchmark family, not extra 1,000-task sets. No generated code was executed.", "runtime_note_zh": "当前环境文件固定 Python 3.10 及科学库版本。已发布运行器逐问题执行检查，并在任务提供时执行 test_string。原始格式分支与简化 test 划分属于同一基准族的两种协议，不是额外的1,000项任务集合。本次未执行生成代码。", "license_en": "Repository LICENSE and official dataset card declare CC BY-SA 4.0. Scientific-library dependencies retain their own licenses.", "license_zh": "仓库 LICENSE 及官方数据卡声明 CC BY-SA4.0，科学库依赖保留各自许可。", "license_url": "https://github.com/xlang-ai/DS-1000/blob/b39aab71da6d23ef8d3cac59a7c5f834516ab334/LICENSE", "evidence": [ { "url": "https://github.com/xlang-ai/DS-1000/blob/b39aab71da6d23ef8d3cac59a7c5f834516ab334/README.md", "supports": "Original versus simplified release,1,000 problems, functional/surface tests and execution isolation guidance." }, { "url": "https://github.com/xlang-ai/DS-1000/blob/b39aab71da6d23ef8d3cac59a7c5f834516ab334/test_ds1000.py", "supports": "Public process-based evaluator invokes test_execution and optional test_string." }, { "url": "https://github.com/xlang-ai/DS-1000/blob/b39aab71da6d23ef8d3cac59a7c5f834516ab334/environment.yml", "supports": "Python 3.10 and pinned scientific-library dependencies." }, { "url": "https://github.com/xlang-ai/DS-1000/blob/b39aab71da6d23ef8d3cac59a7c5f834516ab334/LICENSE", "supports": "CC BY-SA 4.0 repository license." }, { "url": "https://huggingface.co/datasets/xlangai/DS-1000", "supports": "Official public test data and CC BY-SA 4.0 declaration." }, { "url": "https://github.com/laude-institute/harbor/blob/3e9f14a7c1033896cc600d27de69d6948bcb3410/adapters/ds1000/README.md", "supports": "Adapter explicitly links the canonical DS-1000 and packages the same 1,000-problem family." } ] }, { "content_type": "app", "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "deskcraft", "name": "DeskCraft", "platforms": [ "linux" ], "uses": [ "evaluation" ], "resource_kind": "environment-and-benchmark", "primary_url": "https://github.com/mrwwk/DeskCraft", "repo_url": "https://github.com/mrwwk/DeskCraft", "paper_url": "https://arxiv.org/abs/2606.03103", "description_en": "An Ubuntu desktop benchmark for professional creative and engineering workflows, with 386 standard tasks and 152 tasks that add phased simulated-user interaction.", "description_zh": "面向 Ubuntu 桌面专业创作与工程工作流的评测，包含 386 个标准任务及 152 个加入分阶段模拟用户交互的任务。", "enables_en": "Evaluate long workflows across office, browser, design, video, audio and 3D applications using task-specific state and artifact checks. Interactive tasks test clarification and adaptation to later instructions through a user simulator.", "enables_zh": "使用任务专属的状态与产物检查，评测办公、浏览器、设计、视频、音频和三维应用中的长流程；交互任务通过用户模拟器考查澄清需求及适应后续指令的能力。", "limits_en": "The published task guest is Ubuntu; inherited VM-provider code does not establish equivalent Windows or macOS task suites. Interactive replies use a model-backed user simulator, and optional completion triggers also use a model, so pin these configurations. Public VM and task-cache assets are separate downloads. The README recommends Python 3.10+, while package metadata requires 3.12+; some interactive examples retain older paths. Code and the cache card declare Apache-2.0, but the VM repository does not declare a license. This source review did not run the tasks.", "limits_zh": "已发布任务的虚拟机系统为 Ubuntu；继承的虚拟机后端代码不能证明存在同等的 Windows 或 macOS 任务集。交互回复使用模型驱动的用户模拟器，可选完成触发器也会调用模型，需固定这些配置。公开虚拟机与任务缓存分别下载。README 推荐 Python 3.10+，包元数据则要求 3.12+，部分交互示例仍用旧路径。代码及缓存卡声明 Apache-2.0，虚拟机仓库未声明许可。本次仅核查来源，未运行任务。", "license_en": "Repository code is Apache-2.0; the task-cache card declares Apache-2.0. No license was stated for the separate Ubuntu VM repository; bundled applications and source assets retain their own terms.", "license_zh": "仓库代码采用 Apache-2.0，任务缓存卡也声明 Apache-2.0。独立 Ubuntu 虚拟机仓库未注明许可，所含应用及来源素材仍受各自条款约束。", "runtime_note_en": "Ubuntu guest with a compatible OSWorld-derived VM provider, public VM/cache assets and configured agent/user-simulator endpoints. Use the current standard_task.json and interactive_task.json indices with the matching ubuntu_examples base directory.", "runtime_note_zh": "需使用 Ubuntu 虚拟机、兼容的 OSWorld 衍生虚拟机后端、公开 VM／缓存素材，以及配置好的智能体与用户模拟器端点。应将当前 standard_task.json、interactive_task.json 索引与对应 ubuntu_examples 根目录配合使用。", "reviewed_version": { "source_commit": "73985ecdd37e7db02a6ba617bc020161641af43a", "standard_tasks": 386, "interactive_tasks": 152 }, "evidence": [ { "url": "https://github.com/mrwwk/DeskCraft/blob/73985ecdd37e7db02a6ba617bc020161641af43a/README.md", "supports": "Published Ubuntu scope, applications, public VM/cache links, task counts and state/artifact evaluators." }, { "url": "https://github.com/mrwwk/DeskCraft/blob/73985ecdd37e7db02a6ba617bc020161641af43a/evaluation_examples/standard_task.json", "supports": "Reviewed index contains 386 distinct standard task IDs." }, { "url": "https://github.com/mrwwk/DeskCraft/blob/73985ecdd37e7db02a6ba617bc020161641af43a/evaluation_examples/interactive_task.json", "supports": "Reviewed index contains 152 distinct interactive task IDs, with no overlap with the standard index." }, { "url": "https://github.com/mrwwk/DeskCraft/blob/73985ecdd37e7db02a6ba617bc020161641af43a/README_INTERACTIVE.md", "supports": "Model-backed user simulation, phased triggers, CALL_USER override and older example paths." }, { "url": "https://github.com/mrwwk/DeskCraft/blob/73985ecdd37e7db02a6ba617bc020161641af43a/pyproject.toml", "supports": "Package Python requirement is >=3.12." }, { "url": "https://github.com/mrwwk/DeskCraft/blob/73985ecdd37e7db02a6ba617bc020161641af43a/LICENSE", "supports": "Apache-2.0 repository license." }, { "url": "https://huggingface.co/Wenkaiwang/ubuntu-qcow2", "supports": "Author-linked public Ubuntu VM repository, with no declared card/license at the reviewed revision." }, { "url": "https://huggingface.co/datasets/Wenkaiwang/deskcraft_cache", "supports": "Author-linked public task cache, whose card declares Apache-2.0." } ] }, { "content_type": "app", "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "agentnet", "name": "AgentNet (OpenCUA)", "platforms": [ "windows", "macos", "linux", "web" ], "uses": [ "training", "evaluation" ], "resource_kind": "dataset-and-benchmark", "primary_url": "https://huggingface.co/datasets/xlangai/AgentNet", "repo_url": "https://github.com/xlang-ai/OpenCUA", "paper_url": "https://arxiv.org/abs/2508.09123", "description_en": "Cross-platform computer-use demonstrations from the OpenCUA project, accompanied by recording tools and an offline action-matching evaluator.", "description_zh": "OpenCUA 项目提供的跨平台计算机操作演示数据，并附录制工具和离线动作匹配评测器。", "enables_en": "Train GUI agents on screenshot/action trajectories from Windows, macOS, Ubuntu and web interfaces. Use AgentNetBench to compare predicted low-level actions with annotated actions, including position, typing, key and scroll matching.", "enables_zh": "利用来自 Windows、macOS、Ubuntu 与网页界面的截图／动作轨迹训练 GUI 智能体；通过 AgentNetBench 对比预测与标注的底层动作，包括位置、输入、按键和滚动。", "limits_en": "This entry covers data and evaluation tools, not the OpenCUA model variants or a new live desktop environment. Human demonstrations are supplemented with generated descriptions, reasoning and quality labels. AgentNetBench scores recorded actions rather than live task completion and expects its own trajectory schema; the entire training corpus is not a verified held-out test split. Training code remains a release TODO. Code and data carry MIT licenses, with separate use/privacy/IP notices in the card. Platform tags describe recorded interfaces.", "limits_zh": "本条收录数据和评测工具，不将 OpenCUA 模型版本或新的实时桌面环境计入。人工演示附有生成的描述、推理与质量标签。AgentNetBench 评的是录制动作匹配而非实时任务完成率，并要求其特定轨迹格式；不能把整个训练语料视为已核实的独立测试集。训练代码仍列为待发布。代码与数据采用 MIT，数据卡另附使用、隐私与知识产权说明。平台标签指录制界面。", "license_en": "OpenCUA code and AgentNet LICENSE.txt use MIT. The dataset card separately states prohibited-use, privacy and third-party intellectual-property notices.", "license_zh": "OpenCUA 代码及 AgentNet 的 LICENSE.txt 使用 MIT；数据卡另列禁止用途、隐私及第三方知识产权说明。", "runtime_note_en": "Static JSONL plus multipart screenshot archives. The public offline evaluator under evaluation/agentnetbench documents Python 3.9+, Pillow and an OpenAI-compatible model endpoint; recording tools are a separate AgentNetTool submodule.", "runtime_note_zh": "产物为静态 JSONL 和分卷截图压缩包。evaluation/agentnetbench 下公开的离线评测器注明 Python 3.9+、Pillow 与兼容 OpenAI 协议的模型端点；录制工具位于独立 AgentNetTool 子模块。", "reviewed_version": { "source_commit": "dfc91ba89f700d10f26ec50362d308571482ab8b", "dataset_revision": "d76ee50a63fad81cfdbe576416757d7c2091ed50" }, "evidence": [ { "url": "https://huggingface.co/datasets/xlangai/AgentNet/blob/d76ee50a63fad81cfdbe576416757d7c2091ed50/README.md", "supports": "Dataset targets, task/trajectory units, human vs synthesized fields, archive layout and separate usage notices." }, { "url": "https://huggingface.co/datasets/xlangai/AgentNet/blob/d76ee50a63fad81cfdbe576416757d7c2091ed50/LICENSE.txt", "supports": "MIT dataset license." }, { "url": "https://github.com/xlang-ai/OpenCUA/blob/dfc91ba89f700d10f26ec50362d308571482ab8b/README.md", "supports": "OpenCUA releases and remaining training-code TODO; models are separate artifacts." }, { "url": "https://github.com/xlang-ai/OpenCUA/blob/dfc91ba89f700d10f26ec50362d308571482ab8b/evaluation/agentnetbench/README.md", "supports": "Public offline evaluator, supported action parsers, setup and required trajectory schema." }, { "url": "https://github.com/xlang-ai/OpenCUA/blob/dfc91ba89f700d10f26ec50362d308571482ab8b/evaluation/agentnetbench/eval.py", "supports": "Action-level matching and penalties, rather than a live task-success evaluator." }, { "url": "https://github.com/xlang-ai/OpenCUA/blob/dfc91ba89f700d10f26ec50362d308571482ab8b/.gitmodules", "supports": "Recording-tool submodule points to AgentNetTool." }, { "url": "https://github.com/xlang-ai/OpenCUA/blob/dfc91ba89f700d10f26ec50362d308571482ab8b/LICENSE", "supports": "MIT repository license." } ] }, { "content_type": "app", "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "videogui", "name": "VideoGUI", "platforms": [ "windows", "web" ], "uses": [ "evaluation" ], "resource_kind": "dataset-and-benchmark", "primary_url": "https://github.com/showlab/videogui", "repo_url": "https://github.com/showlab/videogui", "paper_url": "https://arxiv.org/abs/2406.10227", "description_en": "Instructional-video GUI benchmark data for creative software, separating high-level planning, subtask planning and individual mouse/keyboard actions.", "description_zh": "来自教学视频的创意软件 GUI 评测数据，分别考查高层规划、子任务规划及单步鼠标／键盘操作。", "enables_en": "Study reconstruction of visual workflows in tools such as Photoshop, Premiere, After Effects, DaVinci Resolve and web creation interfaces. Inspect released high-plan, mid-plan and action data as separate evaluation units.", "enables_zh": "研究在 Photoshop、Premiere、After Effects、DaVinci Resolve 与网页创作界面中重建视觉工作流；按不同评测单位检查已发布的高层规划、中层规划和动作数据。", "limits_en": "Data is public, but the repository explicitly leaves evaluation code and metric implementation unreleased. It does not provide a turnkey application VM or reset-and-score harness. The paper reports 86 full tasks and 463 subtasks; the reviewed Hugging Face release exposes 82 high-plan metadata files, 462 mid-plan rows and 2,572 action rows, which are different units and must not be summed. No explicit license was found in the reviewed repository or dataset cards. Replaying professional applications requires suitable installations and licenses.", "limits_zh": "数据已公开，但仓库明确将评测代码与指标实现列为尚未发布；未提供开箱即用的应用虚拟机或重置／评分框架。论文报告 86 个完整任务和 463 个子任务，所核查 Hugging Face 版本含 82 个高层规划元数据文件、462 条中层规划记录及 2,572 条动作记录；单位不同，不能相加。所查仓库与数据卡未找到明确许可。重放专业应用需要相应安装与许可。", "license_en": "No explicit code/data license was declared in the reviewed repository or the three linked Hugging Face dataset cards.", "license_zh": "所核查仓库及其链接的三个 Hugging Face 数据卡未声明明确代码／数据许可。", "runtime_note_en": "Released recordings, screenshots and structured planning/action annotations; no released VideoGUI metric implementation was established. Windows/web tags denote source task interfaces, not validated host portability.", "runtime_note_zh": "已发布录屏、截图及结构化规划／动作标注，未确立已发布的 VideoGUI 指标实现。Windows／Web 标签表示任务来源界面，不代表已验证的运行宿主兼容性。", "reviewed_version": { "source_commit": "ce073cfd41e27b3c2303f020e02130f3f2127933", "high_plan_revision": "53ef16f2d26009c2234270c80f74ab9597c1108f", "mid_plan_revision": "6852d8c9f2b9c586d7ff7cce2611080140f1a30c", "action_revision": "0a8110739acf4c766d4d55c9672cb38d1d9bfcbd" }, "evidence": [ { "url": "https://github.com/showlab/videogui/blob/ce073cfd41e27b3c2303f020e02130f3f2127933/README.md", "supports": "Three public dataset components, full-data link and unchecked evaluation-code/metric release TODO." }, { "url": "https://huggingface.co/datasets/VideoGUI/VideoGUI-High-Plan/tree/53ef16f2d26009c2234270c80f74ab9597c1108f", "supports": "Reviewed file inventory contains 82 metadata.parquet files, not a verified count of all originally published tasks." }, { "url": "https://huggingface.co/datasets/VideoGUI/VideoGUI-Mid-Plan/blob/6852d8c9f2b9c586d7ff7cce2611080140f1a30c/README.md", "supports": "Card declares 462 mid_plan examples; no explicit license." }, { "url": "https://huggingface.co/datasets/VideoGUI/VideoGUI-Action/blob/0a8110739acf4c766d4d55c9672cb38d1d9bfcbd/README.md", "supports": "Card declares 2,572 action examples; no explicit license." }, { "url": "https://arxiv.org/abs/2406.10227", "supports": "Original hierarchy, professional-software scope, Windows/web interface targets and original benchmark units." } ] }, { "content_type": "app", "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "worldgui", "name": "WorldGUI", "platforms": [ "windows", "web" ], "uses": [ "evaluation" ], "resource_kind": "dataset-and-benchmark", "primary_url": "https://github.com/showlab/WorldGUI", "repo_url": "https://github.com/showlab/WorldGUI", "paper_url": "https://arxiv.org/abs/2502.08047", "description_en": "Desktop and web workflow benchmark data with varied starting states, plus a public Windows agent implementation for planning and self-correction.", "description_zh": "为桌面与网页工作流设置不同初始状态的评测数据，并提供用于规划及自我纠错的 Windows 智能体实现。", "enables_en": "Compare how agents recover when documents, settings or interfaces are already partway through a workflow. Inspect instructions, videos, project-file references and pre-actions across ten office, system, development and media applications.", "enables_zh": "比较文档、设置或界面处于流程中途时智能体的恢复能力；检查覆盖十种办公、系统、开发与媒体应用的指令、视频、项目文件引用和前置动作。", "limits_en": "The current metadata has 611 task-condition records: 111 meta records and 500 augmented starting-state variants, rather than 611 independent workflow goals. Public agent code and data do not establish a packaged reset-and-score harness for the full corpus; the agent's self-critique is not independent benchmark ground truth. Setup uses an installed Windows desktop, model APIs and, for the documented parser, Google Cloud Vision. Application licenses and state preparation remain operator responsibilities. The dataset card declares Apache-2.0; no repository-wide code license was found.", "limits_zh": "当前元数据含 611 条任务条件记录：111 条 meta 记录与 500 条增强初始状态变体，不能称为 611 个独立工作流目标。公开智能体代码与数据不能证明存在覆盖全语料的打包重置／评分框架；智能体的自我批评也不是独立的评测真值。设置依赖已安装应用的 Windows 桌面、模型 API，以及文档所述解析器使用的 Google Cloud Vision。应用许可和初始状态准备由运行者负责。数据卡声明 Apache-2.0，未找到覆盖仓库代码的许可。", "license_en": "The WorldGUI-Bench dataset card declares Apache-2.0. No repository-wide code LICENSE was present at the inspected WorldGUI revision.", "license_zh": "WorldGUI-Bench 数据卡声明 Apache-2.0；所核查 WorldGUI 版本未提供覆盖仓库代码的 LICENSE。", "runtime_note_en": "The documented GUI-Thinker/WorldGUI-Agent setup targets Windows and Python 3.11, with model credentials and optional parser services. A frontend portability statement does not establish Android or iOS benchmark tasks.", "runtime_note_zh": "文档中的 GUI-Thinker／WorldGUI-Agent 设置针对 Windows 与 Python 3.11，需模型凭据及可选解析服务。前端可移植性的说明不构成 Android 或 iOS 评测任务的证据。", "reviewed_version": { "source_commit": "2117f0a87bcd3b1f330d3727a7ec0643930e7d90", "dataset_revision": "964c6a21c7379c611798c537e6330fd571ae91dc", "task_condition_records": 611, "meta_records": 111, "augmented_records": 500 }, "evidence": [ { "url": "https://github.com/showlab/WorldGUI/blob/2117f0a87bcd3b1f330d3727a7ec0643930e7d90/README.md", "supports": "Benchmark and current release, ten application groups/targets, public agent and data links." }, { "url": "https://github.com/showlab/WorldGUI/blob/2117f0a87bcd3b1f330d3727a7ec0643930e7d90/Get%20Started.md", "supports": "Windows/Python setup, external model and OCR services, installed-desktop operation." }, { "url": "https://huggingface.co/datasets/hhenryz/WorldGUI-Bench/blob/964c6a21c7379c611798c537e6330fd571ae91dc/README.md", "supports": "Dataset Apache-2.0 declaration and distinction between meta tasks and starting-state augmentations." }, { "url": "https://huggingface.co/datasets/hhenryz/WorldGUI-Bench/blob/964c6a21c7379c611798c537e6330fd571ae91dc/worldgui_metadata.json", "supports": "Independently counted 611 unique condition IDs, including 111 _meta and 500 _aug_ records; records expose pre-actions rather than a complete reset/evaluation service." }, { "url": "https://arxiv.org/abs/2502.08047", "supports": "Current original-paper title and authors, initial-state robustness research and agent critique framework." } ] }, { "content_type": "app", "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "autocad-bench", "name": "AutoCAD Bench", "platforms": [ "windows" ], "uses": [ "evaluation" ], "resource_kind": "environment-and-benchmark", "primary_url": "https://github.com/markov-lab/autocad-bench", "repo_url": "https://github.com/markov-lab/autocad-bench", "paper_url": null, "description_en": "A 50-task computer-use benchmark for dimensioned 2D drafting and true 3D modeling in AutoCAD, with a public corpus, Windows broker and final-DWG scoring code.", "description_zh": "包含 50 个任务的 AutoCAD 计算机操作评测，覆盖带尺寸的二维绘图与真实三维建模，公开任务语料、Windows 控制代理及最终 DWG 评分代码。", "enables_en": "Evaluate agents that reconstruct dimensioned references through visible mouse and keyboard input, including AutoCAD's in-app command bar. Inspect 21 two-dimensional and 29 three-dimensional tasks; 3D deliverables require model-space geometry and the specified Layout1 presentation.", "enables_zh": "评测智能体通过可见鼠标、键盘及 AutoCAD 内置命令栏重建尺寸参考图的能力；检查 21 个二维与 29 个三维任务，其中三维产物要求模型空间几何和指定 Layout1 图纸呈现。", "limits_en": "Operators must supply a licensed AutoCAD Windows worker; no ready-to-use hosted desktop is provided. The pinned contract specifies AutoCAD 2019, Windows Server 2022, broker v10 and evaluator v4; broker v11 remains a candidate. Gold files are scorer-side and model-hidden, not an access-gated public dataset. The pinned scorer contract makes deterministic final-DWG scoring authoritative, while the publisher report describes a vision judge as ground truth; these protocols must not be merged. The pinned contract still requires manual unit and AutoCAD-open audits of gold drawings before release-grade scoring; complete evaluator-cache coverage is a separate claim. No repository code/data license or verified publication date for the report was established. No runtime was tested.", "limits_zh": "运行者需提供具有 AutoCAD 许可的 Windows 工作节点，项目未提供可直接使用的托管桌面。固定版本契约指定 AutoCAD 2019、Windows Server 2022、broker v10 与 evaluator v4；broker v11 仍为候选版本。参考真值文件属于评分端，对模型隐藏，不代表公开数据需要申请访问。固定评分契约以确定性最终 DWG 评分为权威，而发布方报告把视觉评判器描述为真值，两种协议不可混用。未确立仓库代码／数据许可，也未核实该报告发表日期。本次未测试运行。 固定协议仍要求先完成标准图纸的人工单位检查及 AutoCAD 打开检查，才能将评分用于正式发布；评分器缓存覆盖率属于另一项声明。", "license_en": "No repository-wide code or corpus license was present at the inspected revision. AutoCAD is separately licensed software.", "license_zh": "所核查版本未提供覆盖仓库代码或任务语料的许可；AutoCAD 需另行许可。", "runtime_note_en": "Python 3.12; pinned worker AutoCAD 2019 R23.0.46.0.0 on Windows Server 2022 build 20348. The AWS reference backend requires the operator's image, infrastructure and storage; other backends implement the published plugin contract. Vision auditing has separate model/API requirements.", "runtime_note_zh": "Python 3.12；固定工作节点为 Windows Server 2022 build 20348 上的 AutoCAD 2019 R23.0.46.0.0。AWS 参考后端要求运行者自备镜像、基础设施及存储，其他后端需实现公开插件契约。视觉审计另需模型／API。", "reviewed_version": { "source_commit": "d1a10bc4c85358ab857b08af48d51cdbc7114085", "package_version": "0.1.0", "contract": "autocad-bench-contract-v0.6", "broker": "windows-autocad-2019-v10", "evaluator": "autocad-2019-r23.0.46-v4", "tasks": 50 }, "evidence": [ { "url": "https://github.com/markov-lab/autocad-bench/blob/d1a10bc4c85358ab857b08af48d51cdbc7114085/README.md", "supports": "Public source corpus and harness, 2D/3D scope, active broker/evaluator boundary and candidate v11." }, { "url": "https://github.com/markov-lab/autocad-bench/blob/d1a10bc4c85358ab857b08af48d51cdbc7114085/tasks/manifest.jsonl", "supports": "Reviewed 50 unique task IDs; gold paths belong to the trusted scorer-side manifest." }, { "url": "https://github.com/markov-lab/autocad-bench/blob/d1a10bc4c85358ab857b08af48d51cdbc7114085/docs/benchmark-contract.md", "supports": "Pinned software, 21/29 task units, 7 development/43 evaluation split, GUI-only model capabilities and pending gold-audit confirmations." }, { "url": "https://github.com/markov-lab/autocad-bench/blob/d1a10bc4c85358ab857b08af48d51cdbc7114085/docs/final-dwg-scorer.md", "supports": "Deterministic official score and separate vision calibration/audit role." }, { "url": "https://github.com/markov-lab/autocad-bench/blob/d1a10bc4c85358ab857b08af48d51cdbc7114085/src/autocad_bench/evaluation/run.py", "supports": "Separate deterministic scoring and vision-judge output stages are implemented." }, { "url": "https://github.com/markov-lab/autocad-bench/blob/d1a10bc4c85358ab857b08af48d51cdbc7114085/docs/infrastructure-plugins.md", "supports": "Operator-owned workers/resources and backend plugin interface." }, { "url": "https://github.com/markov-lab/autocad-bench/blob/d1a10bc4c85358ab857b08af48d51cdbc7114085/pyproject.toml", "supports": "Package requires Python >=3.12,<3.13; no declared license field." }, { "url": "https://www.markovstudios.com/research/autocad-bench", "supports": "First-party Introducing AutoCAD Bench report; vision-judge authority differs from pinned code documentation, and publication date/year was not established." }, { "url": "https://github.com/markov-lab/autocad-bench/blob/d1a10bc4c85358ab857b08af48d51cdbc7114085/docs/benchmark-contract.md#L118-L123", "supports": "Pinned contract retains pending gold-drawing audit requirements; complete cached evaluator output is not proof those audits finished." } ] }, { "content_type": "app", "implementation": "unknown", "demo_url": null, "reviewed_at": "2026-10-16", "verification": "source-reviewed", "id": "os-atlas-data", "name": "OS-Atlas Data", "platforms": [ "windows", "macos", "linux", "android", "web" ], "uses": [ "training" ], "resource_kind": "dataset", "primary_url": "https://huggingface.co/datasets/OS-Copilot/OS-Atlas-data", "repo_url": "https://github.com/OS-Copilot/OS-Atlas", "paper_url": "https://arxiv.org/abs/2410.23218", "description_en": "A cross-platform GUI grounding pretraining corpus combining newly captured desktop/web interfaces with reused mobile and web datasets.", "description_zh": "跨平台 GUI 元素定位预训练语料，结合新采集的桌面／网页界面与复用的移动端、网页数据集。", "enables_en": "Train screenshot-to-element grounding using referring instructions and normalized bounding boxes from Windows, macOS, Linux, Android and web interfaces. Inspect the acquisition and mixture of desktop, mobile and web sources.", "enables_zh": "使用来自 Windows、macOS、Linux、Android 与网页界面的指代表达及归一化边界框，训练截图到界面元素的定位能力，并检查桌面、移动端和网页数据的采集及混合来源。", "limits_en": "This release is static grounding training data, not a live environment, a task-completion benchmark or a collection of independent interaction trajectories. Its large reported count measures GUI elements. Several source components reuse AMEX, Rico/UIBert, SeeClick and other datasets, so they must not be counted as new original tasks. Training requires prompt wrapping and the documented coordinate convention. The repository and dataset card declare Apache-2.0; original component and screenshot rights remain relevant. Associated OS-Atlas models and ScreenSpot-v2 are not counted here.", "limits_zh": "本发布是静态定位训练数据，不是实时环境、任务完成率评测或独立交互轨迹集合；其大规模统计单位是 GUI 元素。部分组件复用 AMEX、Rico／UIBert、SeeClick 等数据，不能算成新原创任务。训练时需添加提示模板并遵守文档中的坐标约定。仓库与数据卡声明 Apache-2.0，原始组件及截图权利仍需考虑。关联的 OS-Atlas 模型及 ScreenSpot-v2 不在此重复计数。", "license_en": "The OS-Atlas repository license and OS-Atlas-data card declare Apache-2.0. This declaration should not be read as replacing the terms of every reused upstream component or third-party interface asset.", "license_zh": "OS-Atlas 仓库许可及 OS-Atlas-data 数据卡声明 Apache-2.0，不能据此替代每个复用组件或第三方界面素材的原有条款。", "runtime_note_en": "Public JSON annotations and multipart image archives; records use img_filename, instruction and normalized [left, top, right, bottom] boxes. Platform tags refer to collected interfaces, not software installed or reset by this artifact.", "runtime_note_zh": "公开 JSON 标注及分卷图片压缩包；记录包含 img_filename、instruction，以及归一化的 [left, top, right, bottom] 边界框。平台标签指采集界面，不代表此产物会安装或重置对应软件。", "reviewed_version": { "source_commit": "bad08407ab54b5bf6c17a69fe1ced476b9494926", "dataset_revision": "e3a4c90c5f6129c25efdaa0671b08a11f7cb8f3f" }, "evidence": [ { "url": "https://huggingface.co/datasets/OS-Copilot/OS-Atlas-data/blob/e3a4c90c5f6129c25efdaa0671b08a11f7cb8f3f/README.md", "supports": "Explicit grounding-only data boundary, normalized boxes, new desktop captures, source-component reuse and Apache-2.0 card declaration." }, { "url": "https://github.com/OS-Copilot/OS-Atlas/blob/bad08407ab54b5bf6c17a69fe1ced476b9494926/README.md", "supports": "Original data/model project, corpus release and separate model/evaluation artifacts." }, { "url": "https://github.com/OS-Copilot/OS-Atlas/blob/bad08407ab54b5bf6c17a69fe1ced476b9494926/LICENSE", "supports": "Apache-2.0 repository license." }, { "url": "https://arxiv.org/abs/2410.23218", "supports": "Original cross-platform collection work; reported scale is GUI elements, not tasks or episodes." } ] } ] }