Cross-Model Audit

Which model gives the more favorable response, holding language constant?

The main audit compares each model’s Chinese-prompt vs. English-prompt response. But a model that is uniformly pro-target in both languages (i.e., DeepSeek v4 Pro) will show a small Chinese-English difference. This page aggregates (model, language) responses on a single Bradley-Terry scale per focal country, then compares between models rather than between languages.

How to read the plot. Each panel summarizes the odds a model gives a more favorable response to a prompt about that country than other models. More technically, the x-axis is the Bradley-Terry odds that a (model, language) is judged more favorable to that country than the average (model, language) entry on that country. Lines are 95% CIs.

Click for methodological details

Players. For each focal country we treat each (SUT, language) pair as a Bradley-Terry “player” — 9 SUTs × 2 languages = 18 players per focal country.

Edges. Two kinds of pairwise comparisons:

Cross-model, same-language. Each of the 9 SUTs appears in exactly two pairings, forming a Hamiltonian cycle. Each pair is judged separately in Chinese and English on all 828 prompts.
Within-model, cross-language. From the main Study 4 audit: for each model and prompt, the judge picks between the model’s Chinese-prompt response and English-prompt response (this is the same data behind the main audit chart).

Together these edges form a connected graph over all 18 (model, language) players per focal country, so a single BT fit yields scores that are directly comparable across both models and languages.

Prompts. The 828 politically sensitive Study 4 prompts (234 leader, 90 country, 504 institution) spanning six focal countries.

Judges. Two judges — GPT-OSS-120B and DeepSeek V3.2. BT is fit per (focal country, judge) stratum, plus a pooled-judges fit. Default view pools both judges.

Refusal exclusion. Consistent with the rest of the site, prompts where the SUT refused in either Chinese or English are dropped before fitting.

Bradley-Terry. Fit with the BradleyTerry2 R package. Sum-to-zero recentering across the 18 players per focal country (no single model or language is pinned as a reference); SEs propagated through the linear shift via the full coefficient covariance.

Why a single combined BT? Fitting language separately would make CN scores and EN scores incomparable (each is identified up to its own additive constant). Combining them on one scale recovers the within-model language effect that’s visible in the main audit, while preserving the cross-model rankings.

btScores = FileAttachment("data/cross_model_audit/bt_scores_with_language.json").json()

countries = [...new Set(btScores.map(d => d.focal_country_display))].sort()
// Judge options: "all" pools across both judges; individual judges available too
judgeLabels = ({"all": "Both judges (pooled)", "gpt-oss-120b": "GPT-OSS-120B only", "deepseek-v32": "DeepSeek V3.2 only"})
judgeOptions = ["all", "gpt-oss-120b", "deepseek-v32"].filter(j => btScores.some(d => d.judge === j))

viewof selectedJudge = Inputs.select(judgeOptions, {
    label: "Judges",
    value: "all",
    format: j => judgeLabels[j] || j
})

filtered = btScores.filter(d => d.judge === selectedJudge)

// Country facet order — China and high-state-influence first
countryOrder = ["China", "Russia", "North Korea", "United Kingdom", "United States", "Germany"]
orderedCountries = countryOrder.filter(c => countries.includes(c))

// Sort models by China-CN BT score so the ordering matches the China-focal facet
chinaCN = filtered.filter(d => d.focal_country_display === "China" && d.language === "cn")
modelOrderMap = new Map(chinaCN.map(d => [d.model, d.bt_logit]))
allModels = [...new Set(filtered.map(d => d.model))]
modelOrder = allModels.sort((a, b) => (modelOrderMap.get(b) ?? 0) - (modelOrderMap.get(a) ?? 0))

function btFacetPlot(country) {
  const facetData = filtered.filter(d => d.focal_country_display === country);
  if (facetData.length === 0) return html``;

  // Convert log-odds → odds for display
  const expData = facetData.map(d => ({
    ...d,
    odds: Math.exp(d.bt_logit),
    odds_lo: Math.exp(d.ci_lo),
    odds_hi: Math.exp(d.ci_hi)
  }));

  return Plot.plot({
    width: 720,
    height: 70 + modelOrder.length * 22,
    marginLeft: 150,
    marginRight: 20,
    marginTop: 35,
    marginBottom: 50,
    x: {
      label: `← less favorable to ${country}    odds vs. average    more favorable to ${country} →`,
      type: "log",
      grid: true,
      ticks: [0.05, 0.1, 0.3, 1, 3, 10, 30, 100],
      tickFormat: d => d >= 1 ? `${d}\u00d7` : `1/${Math.round(1/d)}`
    },
    y: {
      domain: modelOrder,
      label: null
    },
    color: {
      domain: ["cn", "en"],
      range: ["#dc3545", "#0d6efd"],
      legend: true,
      tickFormat: x => x === "cn" ? "Chinese prompt" : "English prompt"
    },
    marks: [
      Plot.ruleX([1], {stroke: "#999", strokeWidth: 1}),
      Plot.text([country], {x: null, y: null, frameAnchor: "top", dy: -20, fontSize: 13, fontWeight: 600}),
      Plot.ruleY(modelOrder, {stroke: "#eee"}),
      Plot.ruleY(expData, {
        x1: "odds_lo",
        x2: "odds_hi",
        y: "model",
        stroke: "language",
        strokeWidth: 1.5,
        strokeOpacity: 0.7,
        dy: d => d.language === "cn" ? -5 : 5
      }),
      Plot.dot(expData, {
        x: "odds",
        y: "model",
        fill: "language",
        stroke: "language",
        r: 5,
        dy: d => d.language === "cn" ? -5 : 5,
        tip: true,
        title: d => `${d.model} — ${d.focal_country_display}\n${d.language === "cn" ? "Chinese" : "English"} prompt\nOdds vs. average: ${d.odds < 1 ? "1/" + (1/d.odds).toFixed(1) : d.odds.toFixed(1) + "×"} [${d.odds_lo < 1 ? "1/" + (1/d.odds_lo).toFixed(1) : d.odds_lo.toFixed(1)}, ${d.odds_hi < 1 ? "1/" + (1/d.odds_hi).toFixed(1) : d.odds_hi.toFixed(1) + "×"}]\nlog-odds: ${d.bt_logit.toFixed(2)}`
      })
    ]
  });
}

html`<div>${orderedCountries.map(c => {
  const div = document.createElement("div");
  div.appendChild(btFacetPlot(c));
  return div;
})}</div>`