#!/usr/bin/env python3
"""Reproduce fixed 628-record prompt-text cue statistics from a reader-supplied authorized internal snapshot matching both SHA-256 values. Full input is not distributed; current public case pages are not an immutable equivalent. No API/video/network calls."""
from pathlib import Path
from collections import Counter, defaultdict
import argparse, csv, datetime, hashlib, html, json, re, sys, unicodedata, urllib.parse
FULL_SHA = '7dbaed6887c829fa421fd9d7b20a27b96f958a93c459915efbbfa8fcdee61fc4'
SUBSET_SHA = '340970a0cb1fb7197bf5b63271ec7055433b62a663ce5baa7f9760b0af9e625e'
SOURCE_REVISION = '28f0a1e20a782f6a40ade97e7c9c2741e9e880cd'
SOURCE_PATH = 'src/shared/content/minimax-h3-prompts.json'
SOURCE_ACCESS = {'snapshotType': 'fixed-internal-repository-catalog', 'fullPromptCorpusIncluded': False, 'publicFrozenInputDownloadAvailable': False, 'currentPublicCatalogUrl': 'https://minimaxh3.studio/prompts', 'currentSourcePageNotImmutable': True, 'fullRerunRequires': 'An authorized local input file matching the recorded full-source and subset SHA-256 values. Current public pages do not supply an equivalent immutable input.', 'immediatelyRunnablePublicScope': 'Standard-library codebook tests use synthetic text; derived rows, rules, hashes and recorded results are included.'}

def sha(data): return hashlib.sha256(data).hexdigest()
def canonical(data): return json.dumps(data, ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode('utf-8')
def normalize(text): return re.sub(r'\s+', ' ', unicodedata.normalize('NFKC', text).casefold()).strip()
def percentage(n,d): return round(n*100/d,2) if d else None

def code_text(text, codebook):
    normalized = normalize(text)
    negative = re.compile(codebook['negativePrefixRegex'])
    labels, matches, suppressed = [], [], []
    for rule in codebook['labels']:
        positive = []
        for match in re.finditer(rule['regex'], normalized):
            prefix=normalized[max(0,match.start()-codebook['negativePrefixWindowChars']):match.start()]
            item={'label':rule['id'],'cue':match.group(0),'start':match.start(),'end':match.end()}
            clause=re.split(codebook.get('negativeClauseBreakRegex',r'[.;!?。！？；，,:]'),prefix)[-1]
            if negative.search(clause): suppressed.append(item)
            else: positive.append(item)
        if positive:
            labels.append(rule['id'])
            matches.extend(positive)
    return normalized, labels, matches, suppressed

def csv_file(path, rows, fields):
    with path.open('w',encoding='utf-8',newline='') as output:
        writer=csv.DictWriter(output,fieldnames=fields)
        writer.writeheader(); writer.writerows(rows)

def json_file(path,value): path.write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
def histogram(values, denominator):
    return [{'value':key,'count':count,'denominator':denominator,'sharePercent':percentage(count,denominator)} for key,count in sorted(Counter(values).items(),key=lambda item:(-item[1],item[0]))]

def bar_chart(path, items, locale, title, subtitle, footnotes):
    width,height=1520,210+len(items)*84+160
    chart_x,chart_width=485,770
    font='Arial, Noto Sans CJK SC, PingFang SC, sans-serif'
    parts=[f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" viewBox="0 0 {width} {height}" role="img" aria-labelledby="title desc"><title id="title">{html.escape(title)}</title><desc id="desc">{html.escape(subtitle+" "+" ".join(footnotes))}</desc><rect width="100%" height="100%" fill="#ffffff"/>',f'<text x="64" y="70" font-family="{font}" font-size="34" font-weight="700" fill="#182e3c">{html.escape(title)}</text>',f'<text x="64" y="114" font-family="{font}" font-size="21" fill="#536773">{html.escape(subtitle)}</text>']
    for value in [0,100,200,300,400,500,628]:
        x=chart_x+chart_width*value/628
        parts.append(f'<path d="M{x:.2f} 155V{176+len(items)*84}" stroke="#d9e1e5" stroke-width="1"/><text x="{x:.2f}" y="148" text-anchor="middle" font-family="{font}" font-size="17" fill="#63717a">{value}</text>')
    for index,item in enumerate(items):
        y=184+index*84
        color='#ac7731' if item.get('unknownMode') else '#63747e' if item.get('unmatched') else '#25638a'
        size=chart_width*item['count']/628
        label=item['label'][locale]
        parts.append(f'<text x="64" y="{y+32}" font-family="{font}" font-size="22" fill="#253b48">{html.escape(label)}</text><rect x="{chart_x}" y="{y}" width="{size:.2f}" height="44" fill="{color}" stroke="#163a51" stroke-width="1"/><text x="1280" y="{y+30}" font-family="{font}" font-size="21" fill="#253b48">{item["count"]} / 628 · {percentage(item["count"],628):.2f}%</text>')
    for index,note in enumerate(footnotes):
        parts.append(f'<text x="64" y="{210+len(items)*84+index*31}" font-family="{font}" font-size="18" fill="#596d79">{html.escape(note)}</text>')
    parts.append('</svg>')
    path.write_text(''.join(parts)+'\n',encoding='utf-8')

def make_study(catalog_path, rules_path, output):
    started=datetime.datetime.now(datetime.timezone.utc).isoformat()
    content=catalog_path.read_bytes()
    if sha(content)!=FULL_SHA: raise ValueError('Source bytes differ from the fixed revision; fork the study rather than silently replacing its denominator.')
    catalog=json.loads(content)
    if not isinstance(catalog,list) or len(catalog)!=3784: raise ValueError('Expected the fixed 3,784-record parent catalog.')
    selected=[row for row in catalog if row.get('modelId')=='MiniMax-H3']
    if len(selected)!=628 or sha(canonical(selected))!=SUBSET_SHA: raise ValueError('Fixed H3 selection changed.')
    codebook=json.loads(rules_path.read_text(encoding='utf-8'))
    if codebook['codingVersion']!='cue-codebook-v1.1' or len(codebook['labels'])!=8: raise ValueError('Unexpected codebook identity.')
    ids=[row.get('video',{}).get('sourceId') for row in selected]
    slugs=[row.get('seoSlug') for row in selected]
    if not all(isinstance(value,str) and value for value in ids+slugs) or len(set(ids))!=628 or len(set(slugs))!=628: raise ValueError('Catalog record IDs/canonical slugs are not unique and complete.')
    records=[]
    for row in selected:
        text=row.get('prompt') or ''
        if not isinstance(text,str): raise ValueError('Prompt is not text.')
        normalized,labels,matches,suppressed=code_text(text,codebook)
        rawmode=row.get('originalMode')
        unknown=rawmode is None or str(rawmode).strip().casefold() in ['', 'unspecified', 'unknown', 'not specified', 'n/a']
        source=row.get('source',{})
        video=row.get('video',{}).get('url') or ''
        records.append({'sampleId':row['video']['sourceId'],'canonicalUrl':'https://minimaxh3.studio/prompts/'+row['seoSlug'],'titleForLookup':row.get('title',{}).get('en',''),'modelId':row['modelId'],'rawOriginalMode':rawmode,'modeUnknown':unknown,'siteMappedMode':row.get('mode'),'promptSha256':sha(text.encode('utf-8')),'normalizedPromptSha256':sha(normalized.encode('utf-8')) if normalized else None,'promptCharacterCount':len(text),'emptyPrompt':not normalized,'sourceName':source.get('name',''),'sourceUrl':source.get('url',''),'sourceDomain':urllib.parse.urlsplit(source.get('url','')).netloc.casefold(),'sourceKind':source.get('kind',''),'sourceLicense':source.get('license',''),'sourceLicenseUrl':source.get('licenseUrl',''),'videoUrl':video,'labels':labels,'labelCount':len(labels),'matches':matches,'suppressedMatches':suppressed,'codingScope':'prompt text only; no video interpretation'})
    denominator=len(records)
    unmatched=[r for r in records if not r['labels']]
    counts=Counter(label for row in records for label in row['labels'])
    purpose=[{'labelId':rule['id'],'label':rule['label'],'count':counts[rule['id']],'denominator':denominator,'sharePercent':percentage(counts[rule['id']],denominator),'unit':'records with one or more positive lexical matches in this family'} for rule in codebook['labels']]
    prompt_groups=defaultdict(list);video_groups=defaultdict(list)
    for row in records:
        if row['normalizedPromptSha256']:prompt_groups[row['normalizedPromptSha256']].append(row)
        if row['videoUrl']:video_groups[row['videoUrl']].append(row)
    # Preserve each empty-prompt / missing-video row as its own unit; do not merge all missing values.
    prompt_unique=[sorted(group,key=lambda r:r['sampleId'])[0] for group in prompt_groups.values()]+[r for r in records if r['emptyPrompt']]
    video_unique=[sorted(group,key=lambda r:r['sampleId'])[0] for group in video_groups.values()]+[r for r in records if not r['videoUrl']]
    sensitivities=[]
    for unit,subset in [('catalog_records',records),('one_per_identical_normalized_nonempty_prompt',prompt_unique),('one_per_identical_nonempty_video_url',video_unique)]:
        c=Counter(label for r in subset for label in r['labels'])
        for rule in codebook['labels']:
            sensitivities.append({'denominatorPolicy':unit,'labelId':rule['id'],'count':c[rule['id']],'denominator':len(subset),'sharePercent':percentage(c[rule['id']],len(subset))})
    duplicate_groups={'normalizedPrompt':[{'normalizedPromptSha256':key,'recordCount':len(group),'sampleIds':sorted(r['sampleId'] for r in group)} for key,group in sorted(prompt_groups.items()) if len(group)>1],'exactVideoUrl':[{'videoUrl':key,'recordCount':len(group),'sampleIds':sorted(r['sampleId'] for r in group)} for key,group in sorted(video_groups.items()) if len(group)>1]}
    group_note='Sensitivity keeps the lexicographically smallest opaque sampleId per identical key. Equal prompts can have different outputs; equal URLs are not perceptual duplicate detection. No fuzzy/video-quality clustering.'
    source_modes=histogram([r['rawOriginalMode'] if r['rawOriginalMode'] is not None else '(missing)' for r in records],denominator)
    summary={'schemaVersion':1,'studyId':'blueprint-31','snapshotDate':'2026-10-14','sourceRevision':SOURCE_REVISION,'fullCatalogSha256':FULL_SHA,'h3CanonicalSubsetSha256':SUBSET_SHA,'codebookVersion':codebook['codingVersion'],'codebookSha256':sha(rules_path.read_bytes()),'frame':{'fullCatalogRecords':3784,'excludedOtherModelRecords':3784-628,'h3CatalogRecords':628,'primaryDenominator':628,'emptyPromptRecords':sum(r['emptyPrompt'] for r in records),'missingSourceUrlRecords':sum(not r['sourceUrl'] for r in records),'missingVideoUrlRecords':sum(not r['videoUrl'] for r in records),'allStatusesAreSourceExamples':all(row.get('status')=='source-example' for row in selected),'sourceTypesAreNotRandomOrAllPublicCases':True},'mode':{'unknownOriginalModeCount':sum(r['modeUnknown'] for r in records),'unknownDenominator':628,'unknownSharePercent':percentage(sum(r['modeUnknown'] for r in records),628),'rawOriginalModeCounts':source_modes,'siteMappedModeCounts':histogram([r['siteMappedMode'] or '(missing)' for r in records],628),'notProviderAcceptedParameters':True},'textCueLabels':purpose,'coverage':{'matchedRecords':denominator-len(unmatched),'unmatchedRecords':len(unmatched),'multipleLabelRecords':sum(r['labelCount']>1 for r in records),'labelAssignments':sum(r['labelCount'] for r in records),'assignmentSharesSumPercent':round(sum(item['sharePercent'] for item in purpose),2),'suppressedNegativeCueMatches':sum(len(r['suppressedMatches']) for r in records),'unmatchedMeans':'No positive codebook phrase matched, not no purpose.'},'sourceDomains':histogram([r['sourceDomain'] or '(missing)' for r in records],628),'sourceKinds':histogram([r['sourceKind'] or '(missing)' for r in records],628),'deduplication':{'primaryRecordsRemoved':0,'normalizedPromptDuplicateGroups':len(duplicate_groups['normalizedPrompt']),'recordsInNormalizedPromptDuplicateGroups':sum(x['recordCount'] for x in duplicate_groups['normalizedPrompt']),'promptSensitivityDenominator':len(prompt_unique),'exactVideoUrlDuplicateGroups':len(duplicate_groups['exactVideoUrl']),'recordsInExactVideoUrlDuplicateGroups':sum(x['recordCount'] for x in duplicate_groups['exactVideoUrl']),'videoUrlSensitivityDenominator':len(video_unique),'selectionRule':group_note},'sourceAccess':SOURCE_ACCESS,'visualScope':'These are text-cue and source-field statistics on 628 records. No video, audio, performance or success-rate inference is performed by this script.'}
    json_file(output/'summary.json',summary);json_file(output/'coded-records.json',{'schemaVersion':1,'primaryDenominator':628,'sourceRevision':SOURCE_REVISION,'rulesSha256':summary['codebookSha256'],'records':records})
    json_file(output/'duplicate-groups.json',duplicate_groups)
    rows=[]
    for r in records:
        rows.append({key:r.get(key) for key in ['sampleId','canonicalUrl','titleForLookup','rawOriginalMode','modeUnknown','siteMappedMode','promptSha256','normalizedPromptSha256','promptCharacterCount','sourceUrl','sourceDomain','sourceKind','videoUrl']}|{'labels':'|'.join(r['labels']),'labelCount':r['labelCount'],'matchedCues':json.dumps(r['matches'],ensure_ascii=False,separators=(',',':')),'suppressedCues':json.dumps(r['suppressedMatches'],ensure_ascii=False,separators=(',',':'))})
    csv_file(output/'coded-records.csv',rows,list(rows[0]))
    csv_file(output/'purpose-counts.csv',[{'label_id':r['labelId'],'label_en':r['label']['en'],'label_zh':r['label']['zh'],'count':r['count'],'denominator':r['denominator'],'share_percent':r['sharePercent'],'multi_label':'true'} for r in purpose],['label_id','label_en','label_zh','count','denominator','share_percent','multi_label'])
    csv_file(output/'mode-counts.csv',[{'raw_original_mode':r['value'],'count':r['count'],'denominator':r['denominator'],'share_percent':r['sharePercent']} for r in source_modes],['raw_original_mode','count','denominator','share_percent'])
    csv_file(output/'dedup-sensitivity.csv',sensitivities,list(sensitivities[0]))
    csv_file(output/'source-attributions.csv',[{key:r[key] for key in ['sampleId','canonicalUrl','sourceName','sourceUrl','sourceKind','sourceLicense','sourceLicenseUrl','promptSha256']} for r in records],['sampleId','canonicalUrl','sourceName','sourceUrl','sourceKind','sourceLicense','sourceLicenseUrl','promptSha256'])
    csv_file(output/'unmatched-records.csv',[{'sample_id':r['sampleId'],'canonical_url':r['canonicalUrl'],'prompt_sha256':r['promptSha256'],'prompt_characters':r['promptCharacterCount'],'raw_original_mode':r['rawOriginalMode']} for r in unmatched],['sample_id','canonical_url','prompt_sha256','prompt_characters','raw_original_mode'])
    chart_items=sorted(purpose,key=lambda item:(-item['count'],item['labelId']))+[{'label':{'en':'No matching codebook cue','zh':'未命中词典线索'},'count':len(unmatched),'unmatched':True}]
    for language in ['en','zh']:
        title='Prompt-text cue families' if language=='en' else '提示词文本用途线索分布'
        subtitle='Fixed 2026-10-14 catalog · n = 628 records · multiple labels allowed' if language=='en' else '2026-10-14 固定目录 · 分母 628 条记录 · 每条可有多个标签'
        notes=['Binary lexical membership per family; shares can sum above 100%.','No video quality, success rate, traffic or market-demand inference.'] if language=='en' else ['各类按是否命中记一次，多标签占比相加可能超过 100%。','不推断视频质量、成功率、流量或市场需求。']
        bar_chart(output/f'purpose-cues.{language}.svg',chart_items,language,title,subtitle,notes)
        mode_items=[{'label':{'en':r['value']+' (unreported)' if r['value']=='unspecified' else r['value'],'zh':r['value']+'（未披露）' if r['value']=='unspecified' else r['value']},'count':r['count'],'unknownMode':r['value']=='unspecified'} for r in source_modes]
        title='Original-mode catalog fields' if language=='en' else '目录原始模式字段'
        subtitle='n = 628 catalog records · unknown modes are not filled from the site route' if language=='en' else '分母 628 条目录记录 · 未披露模式不由站点路由回填'
        notes=['Metadata fields are not proof of accepted provider request parameters.'] if language=='en' else ['目录字段不证明提供商曾接受对应请求参数。']
        bar_chart(output/f'mode-fields.{language}.svg',mode_items,language,title,subtitle,notes)
    deterministic_outputs=[p for p in sorted(output.iterdir()) if p.is_file()]
    receipt={'schemaVersion':1,'studyId':'blueprint-31','executionType':'local-standard-library-text-coding','executedAtUtc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'startedAtUtc':started,'pythonVersion':sys.version.split()[0],'sourceRevision':SOURCE_REVISION,'sourcePath':SOURCE_PATH,'sourceAccess':SOURCE_ACCESS,'sourceSha256':FULL_SHA,'subsetSha256':SUBSET_SHA,'scriptSha256':sha(Path(__file__).read_bytes()),'codebookSha256':summary['codebookSha256'],'primaryDenominator':628,'visualMediaInspectedByScript':False,'networkCalls':0,'productionTaskRun':False,'outputs':{p.name:{'sha256':sha(p.read_bytes()),'bytes':p.stat().st_size} for p in deterministic_outputs},'checks':{'fixedFullCatalogIdentity':True,'fixed628SubsetIdentity':True,'unique628SourceIdsAndCanonicalSlugs':True,'all628RowsRetainedInPrimaryCounts':len(records)==628,'unknownModesNotFilled':sum(r['modeUnknown'] for r in records)==323,'eachLabelNumeratorAtMostDenominator':all(0<=r['count']<=r['denominator'] for r in purpose),'originalModeCountsSum628':sum(r['count'] for r in source_modes)==628,'primaryDeduplicationRemovesNoRecords':summary['deduplication']['primaryRecordsRemoved']==0,'sourceCodeAndRulesHashed':True}}
    json_file(output/'execution-receipt.json',receipt)
    return summary

def main():
    parser=argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--catalog',type=Path,required=True,help='Reader-supplied authorized local catalog matching the two fixed hashes; no full-source download is provided.')
    parser.add_argument('--rules',type=Path,default=Path(__file__).resolve().parent/'coding-rules.json')
    parser.add_argument('--out-dir',type=Path,required=True)
    args=parser.parse_args()
    if args.out_dir.exists(): parser.error('Output directory exists; use a fresh directory to preserve prior evidence.')
    args.out_dir.mkdir(parents=True)
    try:
        summary=make_study(args.catalog,args.rules,args.out_dir)
        print(json.dumps({'records':628,'unknownOriginalMode':summary['mode']['unknownOriginalModeCount'],'purposeCounts':{r['labelId']:r['count'] for r in summary['textCueLabels']},'coverage':summary['coverage'],'deduplication':summary['deduplication']},ensure_ascii=False))
    except Exception as error:
        print('Study stopped: '+str(error),file=sys.stderr);return 1
    return 0
if __name__=='__main__': sys.exit(main())