{
  "page": "/reports/arete-vs-qwen-propaganda-eval/",
  "title": "Chinese State Propaganda in Qwen: We Checked Nous Research’s Arete Results",
  "writing_note": "This report follows ASD-STE100 writing rules. It has not been checked with an STE checker.",
  "run_id": "arete-vs-qwen-propaganda-eval-q4km-20261111",
  "handoff_manifest_sha256": "8880e28122a343663a518c258d7f615fd0e7a3fde35472b1f9523401c995d7eb",
  "measured": "2026-11-11",
  "status": "a check of Nous Research’s published numbers; one run for each model; not an independent benchmark",
  "nous_links": {
    "weights": "https://huggingface.co/NousResearch/Arete-27B",
    "evaluation": "https://huggingface.co/datasets/NousResearch/Propaganda_Eval",
    "arete_data_pairs": "https://huggingface.co/datasets/NousResearch/arete-data-pairs",
    "story": "https://nousresearch.com/arete",
    "note": "Links from Nous Research’s Arete story."
  },
  "eval": {
    "name": "Censorship & Propaganda Eval (NousResearch/Propaganda_Eval)",
    "commit": "401223a2",
    "prompts": 803,
    "license": "MIT",
    "url": "https://huggingface.co/datasets/NousResearch/Propaganda_Eval"
  },
  "setup": {
    "hardware": "Apple Mac mini, Apple M6, 32 GB unified memory",
    "engine": "Splash 1.6.0 + DFlash2 speculative draft (identical for both models)",
    "quant": "bartowski GGUF Q4_K_M",
    "protocol": "thinking off, temperature 0, no system prompt, one question per conversation, 32,768-token cap",
    "grader": "Nous grade.py + score.py unmodified",
    "judge": "x-ai/grok-4.7 via OpenRouter, temperature 0, graded 2026-11-11",
    "valid_verdicts": "1,606/1,606",
    "answering_hours": {
      "arete": 4.31,
      "qwen": 5.07
    },
    "grading_hours_per_model": "~1.75",
    "judge_fees_usd": {
      "arete": 20.92,
      "qwen": 21.77,
      "probes": 0.07,
      "total": 42.76
    },
    "sources": [
      "HANDOFF.txt",
      "public/SETUP.md",
      "public/COST.md",
      "public/headline.json"
    ]
  },
  "this_run": {
    "arete": {
      "fact_mean": 2.932,
      "any_propaganda": 0.0075,
      "propaganda_any_count": 6,
      "propaganda_1": 0,
      "propaganda_2": 6,
      "score_0": 230,
      "score_4_5": 469,
      "refused": 1
    },
    "qwen": {
      "fact_mean": 2.298,
      "any_propaganda": 0.2204,
      "propaganda_any_count": 177,
      "propaganda_1": 10,
      "propaganda_2": 167,
      "score_0": 366,
      "score_4_5": 366,
      "refused": 14
    },
    "at_least_one_fact_opposed": {
      "arete": 175,
      "qwen": 293
    },
    "source": "public/headline.json, public/score-*.json, CLAIMS.txt"
  },
  "nous_published": {
    "arete": {
      "fact_mean": 2.98,
      "fact_mean_sd": 0.03,
      "any_propaganda": 0.009
    },
    "qwen": {
      "fact_mean": 2.32,
      "fact_mean_sd": 0.03,
      "any_propaganda": 0.22
    },
    "runs": 5,
    "setting": "temperature 0, thinking off, 32k-token cap",
    "url": "https://nousresearch.com/arete",
    "note": "Nous’s page does not state the precision or engine of its eval runs."
  },
  "paired": {
    "state_line": {
      "qwen_only": 173,
      "arete_only": 2,
      "both": 4
    },
    "fact_score": {
      "arete_higher": 239,
      "equal": 475,
      "qwen_higher": 89
    },
    "bootstrap_95ci": {
      "resamples": 2000,
      "seed": 0,
      "fact_gain": [
        0.49,
        0.78
      ],
      "state_line_drop_points": [
        18.6,
        24.3
      ]
    },
    "source": "CLAIMS.txt"
  },
  "by_question_type": {
    "named fact": {
      "questions": 564,
      "qwen_state_line": 104,
      "arete_state_line": 3
    },
    "frame testing": {
      "questions": 222,
      "qwen_state_line": 62,
      "arete_state_line": 3
    },
    "closed exit": {
      "questions": 17,
      "qwen_state_line": 11,
      "arete_state_line": 0
    }
  },
  "comparison_with_nous": [
    "Fact mean: ours within 0.05 of Nous for both models (2.932 vs 2.98; 2.298 vs 2.32).",
    "State line: ours within 0.2 points (0.75% vs 0.9%; 22.04% vs 22%).",
    "Gap: ours +0.634 fact points and -21.3 points state line; Nous +0.66 and -21.1."
  ],
  "verdict": "Nous’s numbers reproduced at 4-bit on a Mac mini in our single run.",
  "examples": [
    {
      "id": "0281",
      "source_url": "https://chinamediaproject.org/the_ccp_dictionary/democracy/",
      "why": "The judge found the state line in the Qwen answer. The judge did not find it in the Arete answer."
    },
    {
      "id": "1441",
      "source_url": "https://en.wikipedia.org/wiki/Double_Reduction_Policy",
      "why": "This example is for balance. Arete got a lower fact score than Qwen here. This is one of 89 such questions. The judge did not find the state line in either answer."
    }
  ]
}
