Eval runs

Launch one run per target as a group (async)

Launches ONE run per target — attached project environments, or attached named hosts — under a single server-minted runGroupId, then detaches execution and responds 202 with a per-target receipt.

This is the only endpoint with grouped-launch semantics. POST /eval-runs also accepts a runGroupId, but purely as a display label: it groups sibling rows in the UI and gives N separate launches no group treatment. Here the server bounds the fan-out (max 10 targets), validates EVERY target before launching any of them (an unattached target, a heterogeneous target list, or a target whose host selects an unavailable harness returns 400 with zero runs started), and holds exactly ONE organization concurrency slot for the whole group — released only when the last sibling finishes.

Targets fan out along ONE axis: all environmentId or all namedHostId, never a mix. Duplicates are deduplicated by id, preserving order.

A per-target failure does NOT abort its siblings, so read outcome rather than treating the 202 as “everything started”: started means every target launched, partial means some did, failed means none did.

Send Idempotency-Key (or idempotencyKey) to make the launch replayable: the group id is DERIVED from the key, and each target carries a run key derived from that, so a retry after a crash mid-launch returns the original run ids instead of double-launching the targets that already started.

x-mcpjam-launcher and x-mcpjam-ci are read once and applied to EVERY sibling: one fan-out is one launch by one process, so the targets badge identically.

POST

projects

{projectId}

eval-run-groups

Launch one run per target as a group (async)

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "suiteId": "suite_def456",
  "targets": [
    {
      "namedHostId": "host_claude"
    },
    {
      "namedHostId": "host_chatgpt"
    }
  ],
  "iterationOverride": 3
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups"

payload = {
    "suiteId": "suite_def456",
    "targets": [{ "namedHostId": "host_claude" }, { "namedHostId": "host_chatgpt" }],
    "iterationOverride": 3
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    suiteId: 'suite_def456',
    targets: [{namedHostId: 'host_claude'}, {namedHostId: 'host_chatgpt'}],
    iterationOverride: 3
  })
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'suiteId' => 'suite_def456',
    'targets' => [
        [
                'namedHostId' => 'host_claude'
        ],
        [
                'namedHostId' => 'host_chatgpt'
        ]
    ],
    'iterationOverride' => 3
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups"

	payload := strings.NewReader("{\n  \"suiteId\": \"suite_def456\",\n  \"targets\": [\n    {\n      \"namedHostId\": \"host_claude\"\n    },\n    {\n      \"namedHostId\": \"host_chatgpt\"\n    }\n  ],\n  \"iterationOverride\": 3\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"suiteId\": \"suite_def456\",\n  \"targets\": [\n    {\n      \"namedHostId\": \"host_claude\"\n    },\n    {\n      \"namedHostId\": \"host_chatgpt\"\n    }\n  ],\n  \"iterationOverride\": 3\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"suiteId\": \"suite_def456\",\n  \"targets\": [\n    {\n      \"namedHostId\": \"host_claude\"\n    },\n    {\n      \"namedHostId\": \"host_chatgpt\"\n    }\n  ],\n  \"iterationOverride\": 3\n}"

response = http.request(request)
puts response.read_body

{
  "runGroupId": "6f1a2b3c4d5e6f708192a3b4c5d6e7f8",
  "suiteId": "suite_def456",
  "outcome": "partial",
  "startedCount": 1,
  "failedCount": 1,
  "targets": [
    {
      "target": {
        "namedHostId": "host_claude",
        "name": "Claude"
      },
      "status": "started",
      "runId": "run_abc123",
      "runStatus": "running"
    },
    {
      "target": {
        "namedHostId": "host_chatgpt",
        "name": "ChatGPT"
      },
      "status": "failed",
      "error": {
        "code": "VALIDATION_ERROR",
        "message": "Suite has no saved server selection to rerun against."
      }
    }
  ],
  "runId": "run_abc123",
  "status": "running"
}

Authorizations

Authorization

string

header

required

MCPJam API key (sk_…). Create one at Settings → API keys. Guest sessions cannot use the API, and API keys cannot manage other API keys.

Headers

x-mcpjam-launcher

string

The launching process's own DECLARED identity, as compact JSON: {"kind":"cli"|"mcp"|"github_action","client":"…","version":"…"}. Stored on the run as launcher and used only as a display label — source stays api and the audit trail reads the platform-minted attribution instead.

A header rather than a body field so that sending it never breaks against an older deployment: both eval-run bodies reject unknown properties, while an unknown header is ignored everywhere. For the same reason a malformed value, or a kind outside the three above, is DROPPED rather than rejected — a label must never fail a launch. Max 512 bytes.

x-mcpjam-ci

string

The CI job this run was launched from, as compact JSON. Recognized keys: provider, pipelineId (or runId), jobId (or job), runUrl, branch, commitSha. Stored on the run as ciMetadata, which is what makes a run resolvable by commit for baseline comparison. Unrecognized keys are dropped.

A header for the same compatibility reason as x-mcpjam-launcher, and dropped rather than rejected when malformed. Max 2048 bytes.

Path Parameters

projectId

string

required

ID of the hosted project that contains the server.

Body

application/json

Launch one run per target under a single server-minted group id.

targets fan out along ONE axis: every entry is an environmentId, or every entry is a namedHostId. A mixed list returns 400 with details.reason: "HETEROGENEOUS_TARGETS" — mixing them would have to answer whether a host runs inside an environment or beside it, and every answer to that is a cross product nobody asked for. Duplicates are deduplicated by id, preserving order.

The knobs below apply UNIFORMLY to every run in the group. serverIds and refreshSnapshot are deliberately absent: an environment supplies its own closed server set, a host derives one from its config, and last-writer-wins on a frozen suite snapshot is never what a fan-out means. Inline tests are absent for the same reason — a group runs a persisted suite.

suiteId

string

required

The suite every target runs.

targets

object[]

required

One entry per run. Every target must be ATTACHED to the suite; an unattached one returns 400 with zero runs started.

Required array length: 1 - 10 elements

Option 1
Option 2

Show child attributes

iterationOverride

integer

Override the per-case runs count for every run in the group.

Required range: 1 <= x <= 10

caseIds

string[]

Narrow every run in the group to these suite cases.

Minimum array length: 1

matchOptionsOverride

object

Match options for every run in the group. Same public-or-internal vocabulary as POST /eval-runs.

skillsOverride

enum<string>

Run every target as the "without skills" arm.

Available options:

exclude

notes

string

passCriteria

object

The floor a run must clear, as a PERCENT in [0, 100] — 80 means 80%. minimumPassRatePercent is the canonical spelling (the unit is in the name); minimumPassRate is the deprecated alias for it. Send exactly one. THE NAME DISAMBIGUATES THE UNIT: a value in (0, 1) is accepted on minimumPassRatePercent, where it unambiguously means a sub-1% floor the evaluator can act on, and REJECTED on the bare minimumPassRate, where 0.8 cannot be told apart from a fraction sent by mistake and would silently make the gate unfailable. Neither field ever reinterprets a value. A per-case passThreshold IS a fraction; this suite/run floor is not.

Option 1
Option 2

Show child attributes

idempotencyKey

string

Makes the whole launch replayable. The runGroupId is DERIVED from this key and each target's run key is derived from that, so a retry after a crash mid-launch returns the original run ids rather than double-launching the targets that already started. The Idempotency-Key header carries the same value and wins over this field.

Maximum string length: 256

ephemeralEnvironment

boolean

When true, each environment target may be a project-scoped, non-archived environment that is NOT attached to the suite. The launch never mutates the suite. Absent / false keeps the membership check. Probe GET /environments/capabilities (ephemeralEnvironmentLaunch) before sending — older servers reject the unknown field.

importApprovals

object[]

Approve approximated imported cases for THIS RUN ONLY. The caller supplies a hosted test-case id and a reason and nothing else: the approver's identity and the approval time are derived server-side and frozen into the run's snapshot, because a caller-supplied approver would file one person's approval under another's name and a caller-supplied timestamp could be backdated past the edit that invalidated the claim. Nothing persists on the case — the next run of the same approximation needs a new approval. On a grouped launch the SAME approvals go to every target: a case's approximation is approximated the same way on each of them, so one human decision covers the whole fan-out.

Minimum array length: 1

Show child attributes

Response

Group accepted. Each target's outcome is in targets; execution continues in the background.

One entry per target. A per-target failure does NOT abort its siblings, so read outcome rather than treating the 202 as "everything started".

runGroupId

string

required

Server-minted. Every sibling run carries it, and the hosted UI collapses them into one row.

suiteId

string

required

outcome

enum<string>

required

started — every target launched; partial — some did and some did not; failed — none did (still a 202: the group itself was valid, and the per-target reasons are in targets).

Available options:

started,

partial,

failed

startedCount

integer

required

failedCount

integer

required

targets

object[]

required

Discriminated on status, so a target that failed can never be mistaken for one that started with an unread runId.

Show child attributes

runId

string

deprecated

Mirror of the FIRST started run, so readers written against the single-run receipt keep working. Absent when nothing started. Read targets instead — this describes one run out of several.

status

string

deprecated

See runId.

servers

object[]

deprecated

See runId.

Show child attributes

environment

object · null · null

deprecated

See runId.

Show child attributes

caseUpsert

object

deprecated

See runId.

⌘I

Launch one run per target as a group (async)

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "suiteId": "suite_def456",
  "targets": [
    {
      "namedHostId": "host_claude"
    },
    {
      "namedHostId": "host_chatgpt"
    }
  ],
  "iterationOverride": 3
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups"

payload = {
    "suiteId": "suite_def456",
    "targets": [{ "namedHostId": "host_claude" }, { "namedHostId": "host_chatgpt" }],
    "iterationOverride": 3
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    suiteId: 'suite_def456',
    targets: [{namedHostId: 'host_claude'}, {namedHostId: 'host_chatgpt'}],
    iterationOverride: 3
  })
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'suiteId' => 'suite_def456',
    'targets' => [
        [
                'namedHostId' => 'host_claude'
        ],
        [
                'namedHostId' => 'host_chatgpt'
        ]
    ],
    'iterationOverride' => 3
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups"

	payload := strings.NewReader("{\n  \"suiteId\": \"suite_def456\",\n  \"targets\": [\n    {\n      \"namedHostId\": \"host_claude\"\n    },\n    {\n      \"namedHostId\": \"host_chatgpt\"\n    }\n  ],\n  \"iterationOverride\": 3\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"suiteId\": \"suite_def456\",\n  \"targets\": [\n    {\n      \"namedHostId\": \"host_claude\"\n    },\n    {\n      \"namedHostId\": \"host_chatgpt\"\n    }\n  ],\n  \"iterationOverride\": 3\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-run-groups")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"suiteId\": \"suite_def456\",\n  \"targets\": [\n    {\n      \"namedHostId\": \"host_claude\"\n    },\n    {\n      \"namedHostId\": \"host_chatgpt\"\n    }\n  ],\n  \"iterationOverride\": 3\n}"

response = http.request(request)
puts response.read_body

{
  "runGroupId": "6f1a2b3c4d5e6f708192a3b4c5d6e7f8",
  "suiteId": "suite_def456",
  "outcome": "partial",
  "startedCount": 1,
  "failedCount": 1,
  "targets": [
    {
      "target": {
        "namedHostId": "host_claude",
        "name": "Claude"
      },
      "status": "started",
      "runId": "run_abc123",
      "runStatus": "running"
    },
    {
      "target": {
        "namedHostId": "host_chatgpt",
        "name": "ChatGPT"
      },
      "status": "failed",
      "error": {
        "code": "VALIDATION_ERROR",
        "message": "Suite has no saved server selection to rerun against."
      }
    }
  ],
  "runId": "run_abc123",
  "status": "running"
}

API Guides

Endpoints

Launch one run per target as a group (async)

Authorizations

Headers

Path Parameters

Body

Response