{ "schemaVersion": 1, "suiteId": "aie-transcription", "caseId": "CEvIs9y1uog", "measuredAt": "2026-08-21", "status": "diagnostic_no_human_gold", "sourceSha256": "af8050fdab9c8c214410ded106da8509ea51d44901928f2b3dc2480a05d943ab", "thesis": "Podhood's product structure is worth copying, but the examined transcript is not trustworthy enough to become a knowledge-graph source without independent ASR, timing, diarization, and correction provenance.", "recommendations": { "immediate": "ElevenLabs Scribe v2 with native word timestamps, explicitly enabled diarization, speaker labels, a technical glossary, and immutable correction overlays.", "qualityFirst": "Run Scribe v2 and AssemblyAI Universal-3.5 Pro in parallel; use pyannote Precision-2 for panels; review transcript, entity, speaker, and timestamp disagreements.", "selfHosted": "Benchmark faster-whisper large-v3 plus pyannote Community-1 on NVIDIA hardware. Keep Parakeet as a draft/search lane until technical entity quality passes human-gold gates.", "speedAndProvenance": "Deepgram Nova-3 was the fastest completed managed run and returned the strongest model/version metadata." }, "podhood": { "page": { "transcriptSegments": 57, "medianSegmentStartGapSeconds": 17, "p90SegmentStartGapSeconds": 23, "maxSegmentStartGapSeconds": 30, "speakerCount": 2, "speakerTurns": 4, "chapterCount": 10, "keyMomentCount": 9, "searchObserved": { "query": "Cloud Code", "matchPosition": 1, "matchCount": 4 } }, "observedFailure": { "expectedEntityCandidate": "Claude", "publishedEntity": "Cloud", "publishedClaudeCount": 0, "publishedCloudCount": 26, "highlightedQuoteLabelSeconds": 864, "correspondingSpeechApproxSeconds": 851, "semanticLinkErrorApproxSeconds": 13, "explanation": "The 14:24 key moment points to the next thought; the cited day-30 quote occurs in the segment beginning around 14:11." }, "apiObserved": { "segmentTiming": "millisecond", "wordTiming": "millisecond", "speakerLabels": true, "searchCitationUrls": true, "formatVersion": 1, "provenanceUnavailable": [ "ASR provider and immutable model revision", "diarization model and revision", "audio and transcript checksums", "per-segment review status", "correction history", "native versus forced-alignment timing method" ] } }, "technicalEntityDiagnostic": [ { "system": "ElevenLabs Scribe v2", "claude": 26, "cloud": 0, "class": "managed" }, { "system": "AssemblyAI Universal-3.5 Pro", "claude": 26, "cloud": 0, "class": "managed" }, { "system": "Deepgram Nova-3", "claude": 25, "cloud": 1, "class": "managed" }, { "system": "Speechmatics Enhanced", "claude": 21, "cloud": 5, "class": "managed" }, { "system": "faster-whisper large-v3", "claude": 20, "cloud": 6, "class": "self-hosted" }, { "system": "Parakeet-TDT 0.6B v2", "claude": 11, "cloud": 15, "class": "self-hosted" }, { "system": "Parakeet-TDT 0.6B v3", "claude": 7, "cloud": 19, "class": "self-hosted" }, { "system": "Podhood published transcript", "claude": 0, "cloud": 26, "class": "published" } ], "silverReferenceComparison": { "status": "machine_reference_not_human_gold", "caseId": "CEvIs9y1uog", "referenceSystem": "ElevenLabs Scribe v2", "referenceRun": "recovered production campaign run with technical keyterms on the same public video", "referenceWords": 2728, "normalizedReferenceSha256": "701d9f18aa46aaac54d9c2de02f9a1c13314b13a260caefd1510de450ac9a11c", "normalization": "Unicode NFKC; lowercase; replace Unicode punctuation and symbols with spaces; split on whitespace", "formula": "(substitutions + insertions + deletions) / normalized silver-reference word count", "audioCaveat": "The recovered Scribe run and original local runs use the same YouTube recording but independently extracted source encodings, not byte-identical audio.", "goldCaveat": "This is disagreement with a machine transcript, not human-gold word error rate. The reference system scores zero against itself by definition.", "results": [ { "id": "elevenlabs-scribe-v2", "system": "ElevenLabs Scribe v2", "class": "managed", "status": "reference_by_definition", "referenceWords": 2728, "candidateWords": 2728, "substitutions": 0, "insertions": 0, "deletions": 0, "errors": 0, "disagreementWer": 0, "normalizedCandidateSha256": "701d9f18aa46aaac54d9c2de02f9a1c13314b13a260caefd1510de450ac9a11c" }, { "id": "assemblyai-universal-3-5-pro", "system": "AssemblyAI Universal-3.5 Pro", "class": "managed", "status": "unavailable_transcript_not_retained", "disagreementWer": null }, { "id": "deepgram-nova-3", "system": "Deepgram Nova-3", "class": "managed", "status": "unavailable_transcript_not_retained", "disagreementWer": null }, { "id": "speechmatics-enhanced", "system": "Speechmatics Enhanced", "class": "managed", "status": "unavailable_transcript_not_retained", "disagreementWer": null }, { "id": "faster-whisper-large-v3", "system": "faster-whisper large-v3", "class": "self-hosted", "status": "scored_against_silver_reference", "referenceWords": 2728, "candidateWords": 2679, "substitutions": 42, "insertions": 13, "deletions": 62, "errors": 117, "disagreementWer": 0.04288856304985337, "normalizedCandidateSha256": "6bc724b2add0098bb814d0d1c42194d54d9b53529fe197244de7652ce82c3e1f" }, { "id": "parakeet-tdt-0-6b-v2-mlx", "system": "Parakeet-TDT 0.6B v2", "class": "self-hosted", "status": "scored_against_silver_reference", "referenceWords": 2728, "candidateWords": 2685, "substitutions": 44, "insertions": 10, "deletions": 53, "errors": 107, "disagreementWer": 0.039222873900293255, "normalizedCandidateSha256": "eb71c0a0cbb89bc0749933f7c427eb79049fadd67f1d8379a5adf56e787c2c20" }, { "id": "parakeet-tdt-0-6b-v3-mlx", "system": "Parakeet-TDT 0.6B v3", "class": "self-hosted", "status": "scored_against_silver_reference", "referenceWords": 2728, "candidateWords": 2693, "substitutions": 62, "insertions": 5, "deletions": 40, "errors": 107, "disagreementWer": 0.039222873900293255, "normalizedCandidateSha256": "9b23547be6df0a5598d0b2c0d97438d6c64ffb9b6c2113f5010f8e418a883d7d" } ] }, "managed": [ { "system": "ElevenLabs Scribe v2", "status": "complete_recovered_observation", "runtimeSeconds": 17.047, "realtimeMultiple": 57.573844, "speakerCount": 2, "wordRecords": 2655, "timing": "word", "usdPerAudioHour": 0.22, "usdPerAudioHourWithKeyterms": 0.27 }, { "system": "AssemblyAI Universal-3.5 Pro", "status": "complete_recovered_observation", "runtimeSecondsUpperBound": 29, "speakerCount": 2, "wordRecords": 2639, "utterances": 5, "timing": "word", "usdPerAudioHourBase": 0.21 }, { "system": "Deepgram Nova-3", "status": "complete_recovered_observation", "runtimeSeconds": 6.688, "realtimeMultiple": 146.749, "speakerCount": 2, "wordRecords": 2650, "utterances": 156, "timing": "word", "modelVersion": "2025-07-31.0", "usdPerAudioHourApproxBeforeAddOns": 0.462 }, { "system": "Speechmatics Enhanced", "status": "complete_recovered_observation", "serviceMetadataSeconds": 12, "clientWallTimeSeconds": null, "speakerCount": 2, "wordRecords": 2640, "timing": "word", "usdPerAudioHour": 0.40 }, { "system": "Groq Whisper large-v3", "status": "partial_result_lost", "runtimeSeconds": 10.899, "timing": "word_and_segment", "usdPerAudioHour": 0.111 }, { "system": "OpenAI gpt-4o-transcribe-diarize", "status": "result_lost_not_rerun", "timing": "speaker_segment_only" }, { "system": "Soniox stt-async-v5", "status": "unavailable_no_funded_project", "timing": "token" } ], "productionCandidate": { "provider": "ElevenLabs", "model": "scribe_v2", "apiDocumentation": "https://elevenlabs.io/docs/api-reference/speech-to-text/convert", "pricingDocumentation": "https://elevenlabs.io/pricing/api", "requiredRequest": { "model_id": "scribe_v2", "diarize": true, "timestamps_granularity": "word" }, "features": { "publicYoutubeSourceUrl": "documented_but_two_public_video_requests_failed_http_400", "wordStartEndSeconds": "observed_and_documented", "characterTimestamps": "documented_not_benchmarked", "speakerDiarization": "observed_when_explicitly_enabled", "speakerIdentity": "anonymous_per_recording_unless_speaker_library_is_configured", "speakerLimit": 32, "speakerConfidence": "unavailable", "simultaneousOverlapAttribution": "unverified", "technicalKeyterms": "observed_paid_add_on_with_98_campaign_terms", "maximumKeyterms": 1000, "audioEvents": "documented_enabled_by_default", "languages": "90_plus_documented", "asyncWebhooks": "documented_not_benchmarked", "maximumStandardDurationHours": 10, "maximumMultichannelDurationHours": 1, "conservativeMaximumFileGigabytes": 3, "fileLimitDocumentationConflict": "Overview says 3 GB; endpoint says 5 GB; deprecated cloud-storage URL says 2 GB.", "defaultDataRetention": "enabled", "zeroRetention": "enterprise_only", "immutableModelRevision": "unavailable", "exactTimedWordCoverage": "complete_for_all_2013_words_in_preserved_production_pilot", "absoluteTimestampAccuracy": "unavailable_no_human_gold" }, "observedBenchmark": { "audioSeconds": 981.461313, "clientWallSeconds": 17.047, "realTimeFactor": 0.017369, "realtimeMultiple": 57.573844, "wordRecords": 2655, "speakerClusters": 2, "speakerChangeSeconds": [299.66, 800.284, 942.544], "claudeCount": 26, "cloudCount": 0, "rawProviderResponse": "lost_after_workstation_crash", "humanGold": "unavailable" }, "observedProductionPilot": { "videoId": "HvMyYLTfvhg", "audioSeconds": 802, "clientWallSeconds": 12.83, "realtimeMultiple": 62.51, "audioFormat": "native_m4a", "audioBytes": 12975349, "audioSha256Prefix": "2e590", "audioFilenamePattern": "--.m4a", "keytermCount": 98, "diarizationEnabled": true, "wordRecords": 2013, "timedWordRecords": 2013, "speakerAttributedWordRecords": 2013, "speakerCount": 1, "segmentCount": 44, "observedMaximumConcurrentRequests": 20, "estimatedUsdWithKeyterms": 0.0602, "humanGold": "unavailable" }, "observedIngestion": { "directYoutubeSourceUrl": { "attemptedPublicVideos": 2, "httpStatus": 400, "failure": "provider upstream source download failed for public unrestricted embeddable videos", "result": "direct YouTube ingestion not operational for the tested videos" }, "downloadAndUpload": { "workingYtDlpVersion": "2026.8.19", "previousSystemYtDlpVersion": "2026.03.17", "previousSystemFailure": "HTTP 403", "retainedNativeAudio": true, "audioChecksum": "sha256", "backfilledAudioFiles": 75, "backfillChecksumsMatched": 75 } }, "pricingUsdPerAudioHour": { "base": 0.22, "keytermsAddOn": 0.05, "baseWithKeyterms": 0.27, "entityDetectionAddOn": 0.07, "taxesIncluded": false, "meteringCaveat": "Public pricing lists rounded add-ons; endpoint describes 20% and 30% surcharges. Verify actual invoice metering." }, "planLimits": [ { "plan": "Free / PAYG", "monthlyUsd": 0, "includedAudioHours": 4.5, "speechToTextConcurrency": 8, "caveat": "General PAYG documentation separately describes a three-request generic concurrency limit; verify actual response headers." }, { "plan": "Starter", "monthlyUsd": 6, "includedAudioHours": 27, "speechToTextConcurrency": 12 }, { "plan": "Creator", "monthlyUsd": 22, "firstMonthUsd": 11, "includedAudioHours": 100, "speechToTextConcurrency": 20 }, { "plan": "Pro", "monthlyUsd": 99, "includedAudioHours": 450, "speechToTextConcurrency": 40 }, { "plan": "Scale", "monthlyUsd": 299, "includedAudioHours": 1359, "speechToTextConcurrency": 60 }, { "plan": "Business", "monthlyUsd": 990, "includedAudioHours": 4500, "speechToTextConcurrency": 60 } ], "concurrencyCaveats": [ "Files longer than eight minutes are internally sharded across up to four workers.", "Documentation does not establish whether internal workers consume separate account-concurrency slots.", "Free/PAYG generic and speech-specific published limits conflict.", "Measure current-concurrent-requests and maximum-concurrent-requests before bulk fanout." ] }, "publicArchiveInventory": { "channelHandle": "@aiDotEngineer", "channelId": "UCLKPca3kwwd-B59HNr-_lvA", "inventoryMethod": "live public YouTube upload enumeration by video ID; livestream tab inspected only to document excluded scope", "publicUploads": { "videos": 1049, "audioHours": 449.8939 }, "excludedShortUploads": { "videos": 7, "audioHours": 0.4231, "maximumDurationMinutes": 5 }, "standaloneConferenceLength": { "videos": 1042, "audioHours": 449.4708 }, "excludedLivestreams": { "videos": 32, "audioHours": 205.8289, "reason": "explicitly excluded from the approved standalone-talk transcription campaign" }, "campaignScope": "public_standalone_conference_length_only", "outsidePublicInventory": "Private and unlisted recordings are not included.", "independentStudioSnapshot": { "asOf": "2026-08-18", "publishedSince": "2025-06-01", "publicConferenceLength": { "videos": 811, "audioHours": 352.2911 }, "privateConferenceLength": { "videos": 105, "audioHours": 44.3236 } } }, "bulkCostScenarios": [ { "scope": "public_standalone_conference_length", "videos": 1042, "audioHours": 449.4708, "baseUsd": 98.88, "withKeytermsUsd": 121.36 }, { "scope": "planning_100_audio_hours", "audioHours": 100, "baseUsd": 22, "withKeytermsUsd": 27 }, { "scope": "planning_500_audio_hours", "audioHours": 500, "baseUsd": 110, "withKeytermsUsd": 135 }, { "scope": "planning_1000_audio_hours", "audioHours": 1000, "baseUsd": 220, "withKeytermsUsd": 270 } ], "bulkLatencyProjection": { "basis": "single 981.461313-second recovered benchmark observation; optimistic service-only linear extrapolation", "observedRealTimeFactor": 0.017369, "standaloneArchiveHours": 449.4708, "serialServiceHours": 7.806858, "ideal12ConcurrentMinutes": 39.034, "ideal20ConcurrentMinutes": 23.421, "ideal40ConcurrentMinutes": 11.710, "excludes": ["YouTube retrieval", "network transfer", "queue saturation", "account rate limits", "retries", "webhook delivery", "normalization", "speaker review"], "recommendedPilotVideoCount": [25, 50], "recommendedOperationalBudget": "a few hours, calibrated against an initial pilot" }, "selfHosted": [ { "system": "Parakeet-TDT 0.6B v3 on MLX", "status": "complete", "runtimeSeconds": 24.29, "rtf": 0.02475, "timedCoverage": 1 }, { "system": "Parakeet-TDT 0.6B v2 on MLX", "status": "complete", "runtimeSeconds": 23.78, "rtf": 0.02423, "timedCoverage": 1 }, { "system": "faster-whisper large-v3 CPU int8", "status": "complete", "runtimeSeconds": 1288.350862, "rtf": 1.313, "timedCoverage": 1 }, { "system": "Canary 1B v2", "status": "unsupported_host", "runtimeSeconds": null, "rtf": null, "timedCoverage": null } ], "alignment": { "status": "complete_relative_disagreement_only", "nativeSourceTokens": 2625, "whisperxEmittedTokens": 2609, "whisperxEmittedTimedCoverage": 1, "sourceTokensWithExactTimedAlignment": 2595, "sourceTokenExactTimedCoverage": 0.9885714285714285, "sourceTokensWithoutExactTimedAlignment": 30, "midpointAbsoluteDriftSeconds": { "median": 0.15899999999999181, "p90": 0.3027999999999792, "p95": 0.3528000000000131, "p99": 0.4930399999998469, "max": 3.789500000000089 }, "coveragePenalizedMidpointSeconds": { "missingTokenPenalty": 1, "median": 0.15999999999996817, "p95": 0.372500000000013, "p99": 1, "max": 3.789500000000089 }, "mergedTokenPatterns": ["production-ready", "20-25", "self-documenting", "skill.md", "third-party", "domain-specific", "professional-quality", "open-source", "team-specific", "code-style", "day-to-day", "in-context"], "goldLimitation": "No human word-boundary gold exists, so drift measures disagreement rather than correctness." }, "diarization": [ { "system": "pyannote Community-1", "status": "complete", "runtimeSeconds": 1244.160978, "rtf": 1.267662, "speakerCount": 2, "turnCount": 118, "proxyChangeSeconds": [297.031, 800.277, 942.533], "observedFailure": "One 16.9 ms false-speaker blip caused two extra changes." }, { "system": "NeMo multiscale spectral clustering", "status": "complete", "runtimeSeconds": 1870.630698, "rtf": 1.905965, "speakerCount": 2, "turnCount": 163, "proxyChangeSeconds": [299.5, 800.22, 942.46], "observedFailure": "Four isolated 270 ms false-speaker assignments caused eight extra changes; the clustering output cannot represent overlap.", "limitation": "The published five-scale recipe used here is telephone-tuned and the run used an oracle speaker count of two, so treat it as diagnostic rather than conference-optimized." }, { "system": "pyannote Precision-2", "status": "not_run", "runtimeSeconds": null, "rtf": null, "speakerCount": null, "turnCount": null } ], "unavailableGoldMetrics": { "wer": { "value": null, "status": "unavailable_no_gold", "goldRequired": true }, "technicalNamedEntityErrorRate": { "value": null, "status": "unavailable_no_gold", "goldRequired": true }, "der": { "value": null, "status": "unavailable_no_gold", "goldRequired": true }, "jer": { "value": null, "status": "unavailable_no_gold", "goldRequired": true }, "absoluteWordTimestampError": { "value": null, "status": "unavailable_no_gold", "goldRequired": true } }, "priceSnapshot": { "asOf": "2026-08-21", "prices": [ { "system": "ElevenLabs Scribe v2", "label": "$0.22/audio hour base; $0.27/audio hour with keyterm prompting", "url": "https://elevenlabs.io/pricing/api" }, { "system": "AssemblyAI Universal-3.5 Pro", "label": "$0.21/hour base", "url": "https://www.assemblyai.com/pricing" }, { "system": "Speechmatics Batch Enhanced", "label": "$0.40/hour", "url": "https://www.speechmatics.com/pricing" }, { "system": "Deepgram Nova-3 prerecorded mono", "label": "about $0.462/hour before add-ons", "url": "https://deepgram.com/pricing" }, { "system": "OpenAI GPT-Transcribe", "label": "$0.0045/minute ($0.27/hour)", "url": "https://openai.com/api/pricing/" }, { "system": "Groq Whisper large-v3", "label": "$0.111/hour", "url": "https://console.groq.com/docs/speech-to-text" }, { "system": "pyannote Precision-2 Developer", "label": "€0.112/hour", "url": "https://www.pyannote.ai/pricing" } ] }, "artifactReceipts": [ { "id": "alignment-scores", "path": "evidence/alignment-scores.json", "sha256": "cf10afeae2c4bb8f71cc54a82fa51812426d9d34462a934a85cedc54edd76a34" }, { "id": "alignment-details", "path": "evidence/alignment-details.json", "sha256": "f7eb89120cc28d7fda5c807dac44be539aa94f5705f93eeb081e74b1fd8dd8b8" }, { "id": "community-1-rttm", "path": "evidence/community1.rttm", "sitePath": "evidence/community1.rttm.txt", "sha256": "35e84eb0f6b0115036da463901d3f3acb5441968087523a23f9335433e964371" }, { "id": "nemo-rttm", "path": "evidence/nemo.rttm", "sitePath": "evidence/nemo.rttm.txt", "sha256": "475fd73ffa1823c52fcc45702f066e394b2ff349f0013c8c11c2fd3ea76db28c" } ], "limitations": [ "This is one clean two-speaker conference talk, not a representative leaderboard.", "Managed raw provider responses were not preserved across the workstation crash, so managed findings are recovered session observations.", "No human gold exists; machine-reference disagreement is not correctness and human-gold WER, technical entity error rate, DER, and JER remain unavailable.", "Mac CPU/MLX runtimes are not directly comparable with managed GPU services.", "Public prices are dated labels, not guarantees." ] }