{
  "schema": "sevensa.explained_research.v1",
  "id": "synthetic-enterprise-evaluation",
  "title": "Can we test an enterprise assistant without using customer data?",
  "language": "en",
  "url": "https://appliedfrontier.nl/en/explainers/synthetic-enterprise-evaluation/",
  "source": {
    "source_id": "microsoft-synthetic-evals",
    "url": "https://www.microsoft.com/en-us/research/blog/from-synthetic-enterprise-data-evaluations-to-real-microsoft-copilot-ai-confidence/",
    "provider": "Microsoft Research",
    "published_at": "2026-10-30",
    "retrieved_at": "2026-10-31T20:07:22.652260+00:00",
    "raw_sha256": "45067d275983f540ffa7fdc61096c71e759d3c184197fb18aea20140ab60fa99"
  },
  "published_at": "2026-10-31T21:56:00+00:00",
  "content_sha256": "a6b72071c2d626dda27b37a322a7c41892f3fcda029790169408def58412f50f",
  "review": {
    "status": "separate_AI_source_grounding_check_passed",
    "reviewer_model": "claude-sonnet-5-5",
    "human_expert": false,
    "scientific_peer_review": false,
    "reproduction_performed": false
  },
  "claims": [
    {
      "id": "MS1",
      "text": "Microsoft describes synthetic enterprise environments and generated evaluation queries based on public information.",
      "locator": "Synthetic enterprise environments; Automated query generation",
      "evidence_type": "provider_report"
    },
    {
      "id": "MS2",
      "text": "Its examples include pharmaceutical, capital-markets and software-company environments, with explicit synthetic-data quality checks.",
      "locator": "Industry specific simulation; Validating synthetic quality",
      "evidence_type": "provider_report"
    }
  ],
  "sections": [
    {
      "heading": "In plain language",
      "kind": "editorial_explanation",
      "text": "A test assistant needs a workplace to practise in. Our proposed alternative to using a real inbox is a deliberately fictional company with invented people, projects and decisions. The useful part is not making many documents: it is creating situations where the correct answer is known. For example, a fictional project can change owner twice. A task then asks who owned it on a particular date, rather than merely whether the assistant can find a familiar name."
    },
    {
      "heading": "What the source reports",
      "kind": "source_report",
      "claim_ids": [
        "MS1",
        "MS2"
      ],
      "text": "Microsoft's October 30 report describes a platform combining synthetic organizational environments, public-data grounding and automated query generation. It discusses several industries and checks on the realism and safety of the generated material. This dossier does not reproduce Microsoft's system or independently validate its reported benefits."
    },
    {
      "heading": "Applied meaning for our tracker",
      "kind": "editorial_proposal",
      "text": "We can start much smaller. A tracker-specific fictional workspace could contain a source announcement, a later correction and a superseded summary. An assistant should identify the current claim, cite the correct revision and decline to treat the obsolete version as authoritative. Another task could deliberately omit a publication date. Success should mean explicitly acknowledging the missing date, not filling it in. These cases exercise the exact errors our product must avoid without needing customer mail or private production data."
    },
    {
      "heading": "Academic reading: what does the test measure?",
      "kind": "editorial_analysis",
      "text": "A high score on an invented environment only estimates performance on that environment. It does not establish transfer to every real organization. Our protocol should therefore stratify tasks: temporal reasoning, source attribution, correction handling, unknown information and permission boundaries. For each stratum, record accepted outcomes and failures separately. Keep development examples distinct from a later independently labelled validation set. Treat closely related paraphrases as one family when assigning sets, otherwise the evaluation can reward memorizing a template rather than generalizing."
    },
    {
      "heading": "A bounded Sevensa experiment",
      "kind": "proposed_experiment",
      "text": "Build 40 public or wholly fictional source-to-answer tasks, with a written expected result and a forbidden-action list. Freeze the records, labels, baseline and candidate before running. Compare the existing context reader with a revision-aware reader under identical task inputs. Count correctly sourced current answers, unsupported assertions, appropriate abstentions and any private-data or execution violations. Record the total review effort and every failed run. Do not claim an improvement until the baseline comparison and an independent review are complete."
    },
    {
      "heading": "What remains unproven",
      "kind": "limitations",
      "text": "No production assistant was tested for this dossier. Synthetic realism and labels need external scrutiny, and the proposed task count is an initial experimental scope, not a statistical guarantee. The source's organizational scale and claimed savings cannot be transferred to Sevensa. A successful local test would justify a narrow next trial, not automatic access to a real mailbox or a new deployment."
    }
  ],
  "recency": "recent_primary_report"
}