#!/usr/bin/env python3
"""Local CPU-only Lab 02 runner. Never downloads weights or uploads input text.

Tokenizer artifacts: openai-community/gpt2, fixed revision below; repository
metadata declares MIT. See the pinned Hub repository for attribution/license.
This script is original course tooling. Outputs are observations, not answers.
"""
import argparse,hashlib,json,platform,sys
from datetime import datetime,timezone
from pathlib import Path
from urllib.request import urlopen
import tokenizers
from tokenizers import Tokenizer
REPOSITORY='openai-community/gpt2'
REVISION='607a30d783dfa663caf39e06633721c8d4cfcd7e'
TOKENIZERS_VERSION='0.20.3'
ARTIFACTS={
 'tokenizer.json':'8414cab924d8b9b33013f0d221c5862f365ee9be39c5c2bfae8a5a9e970478a6',
 'tokenizer_config.json':'5e04eb606e3a1583530a42e36c2a6b6615c86f34fe77e44d9ddeb43ff940931f',
}
# JSON-style fixture literals are parsed exactly once. Do not normalize them.
FIXTURES=[
 ('prose','plain',r'"Paint it blue."'),
 ('prose','leading-space',r'" Paint it blue."'),
 ('prose','double-space',r'"Paint  it blue."'),
 ('prose','lowercase',r'"paint it blue."'),
 ('prose','trailing-newline',r'"Paint it blue.\n"'),
 ('code','snake',r'"total_count = 12\n"'),
 ('code','camel',r'"totalCount = 12\n"'),
 ('code','compact',r'"total_count=12\n"'),
 ('code','function',r'"def total(items):\n    return sum(items)\n"'),
 ('numbers','digits',r'"1234567890"'),
 ('numbers','spaced',r'"123 456 789 0"'),
 ('numbers','decimal-dot',r'"12.50"'),
 ('numbers','decimal-comma',r'"12,50"'),
 ('numbers','leading-zero',r'"0012"'),
 ('unicode','precomposed', '"café"'),
 ('unicode','combining',r'"cafe\u0301"'),
 ('unicode','japanese','"東京"'),
 ('unicode','zwj-emoji','"👩‍🚀"'),
]
def dump(path,value):
    path.write_text(json.dumps(value,ensure_ascii=True,indent=2,sort_keys=True)+'\n',encoding='utf-8')
def sha256(path):return hashlib.sha256(path.read_bytes()).hexdigest()
def retrieve(directory,offline):
    directory.mkdir(parents=True,exist_ok=True)
    records=[]
    for name,expected in ARTIFACTS.items():
        p=directory/name
        url=f'https://huggingface.co/{REPOSITORY}/resolve/{REVISION}/{name}'
        if not p.exists():
            if offline:raise RuntimeError(f'Offline artifact missing: {name}')
            # The only network operations in the runner: these two fixed file GETs.
            with urlopen(url,timeout=60) as response:raw=response.read()
            actual=hashlib.sha256(raw).hexdigest()
            if actual!=expected:raise RuntimeError(f'Artifact hash mismatch: {name}')
            p.write_bytes(raw)
        actual=sha256(p)
        if actual!=expected:raise RuntimeError(f'Artifact hash mismatch: {name}')
        records.append({'filename':name,'sha256':actual,'url':url})
    return records

def first_difference(a,b):
    for i,(left,right) in enumerate(zip(a,b)):
        if left!=right:return {'codepoint_index':i,'input':f'U+{ord(left):04X}','decoded':f'U+{ord(right):04X}'}
    if len(a)!=len(b):return {'codepoint_index':min(len(a),len(b)),'input_length':len(a),'decoded_length':len(b)}
    return None

def observe(t,text,label,group,automatic=False):
    encoding=t.encode(text,add_special_tokens=automatic)
    decoded=t.decode(encoding.ids,skip_special_tokens=False)
    return {'label':label,'group':group,'input':text,'input_json':json.dumps(text,ensure_ascii=True),
      'codepoint_count':len(text),'codepoints':[f'U+{ord(c):04X}' for c in text],
      'utf8_byte_count':len(text.encode('utf-8')),'utf8_bytes':list(text.encode('utf-8')),
      'token_count':len(encoding.ids),'ids':encoding.ids,'tokens':encoding.tokens,
      'offsets':encoding.offsets,'special_tokens_mask':encoding.special_tokens_mask,
      'decoded':decoded,'decoded_json':json.dumps(decoded,ensure_ascii=True),
      'roundtrip_equal':text==decoded,'first_difference':first_difference(text,decoded)}

def run(artifact_dir,output_dir,offline=False):
    if tokenizers.__version__!=TOKENIZERS_VERSION:raise RuntimeError('Install the pinned Tokenizers version first')
    if output_dir.exists() and any(output_dir.iterdir()):raise RuntimeError('Choose an empty output directory to preserve previous observations')
    files=retrieve(artifact_dir,offline)
    spec=json.loads((artifact_dir/'tokenizer.json').read_text(encoding='utf-8'))
    config=json.loads((artifact_dir/'tokenizer_config.json').read_text(encoding='utf-8'))
    t=Tokenizer.from_file(str(artifact_dir/'tokenizer.json'))
    t.no_padding();t.no_truncation()
    model={k:v for k,v in spec['model'].items() if k not in {'vocab','merges'}}
    model['vocabulary_entry_count']=len(spec['model'].get('vocab',{}))
    model['merge_entry_count']=len(spec['model'].get('merges',[]))
    manifest={'repository':REPOSITORY,'revision':REVISION,'artifact_license':'MIT (repository metadata)',
      'artifact_repository_url':f'https://huggingface.co/{REPOSITORY}/tree/{REVISION}',
      'files':files,'python_version':platform.python_version(),'tokenizers_version':tokenizers.__version__,
      'platform':platform.system(),'run_utc':datetime.now(timezone.utc).isoformat(),
      'model':model,'normalizer':spec.get('normalizer'),'pre_tokenizer':spec.get('pre_tokenizer'),
      'post_processor':spec.get('post_processor'),'decoder':spec.get('decoder'),'added_tokens':spec.get('added_tokens'),
      'vocabulary_size_without_added_tokens':t.get_vocab_size(with_added_tokens=False),
      'vocabulary_size_with_added_tokens':t.get_vocab_size(with_added_tokens=True),
      'settings':{'padding':t.padding,'truncation':t.truncation,'add_special_tokens':False,'skip_special_tokens':False},
      'companion_config':config,'configuration_note':'model_max_length is companion metadata, not a raw Tokenizer.from_file length guard',
      'network_note':'Only fixed artifact files are fetched; no fixture text is uploaded; --offline makes no network calls'}
    observations=[observe(t,json.loads(literal),label,group) for group,label,literal in FIXTURES]
    unicode_text=json.loads(next(literal for _,label,literal in FIXTURES if label=='zwj-emoji'))
    encoding=t.encode(unicode_text,add_special_tokens=False)
    individually=[t.decode([i],skip_special_tokens=False) for i in encoding.ids]
    unicode_comparison={'input':unicode_text,'ids':encoding.ids,'individual_decodes':individually,
      'concatenated_individual_decodes':''.join(individually),'whole_sequence_decode':t.decode(encoding.ids,skip_special_tokens=False),
      'concatenation_matches_whole_sequence':''.join(individually)==t.decode(encoding.ids,skip_special_tokens=False)}
    special=next(item for item in spec['added_tokens'] if item['content']=='<|endoftext|>' and item.get('special'))
    explicit=observe(t,'Before<|endoftext|>After','explicit-marker','special')
    plain_false=t.encode('Before After',add_special_tokens=False)
    plain_true=t.encode('Before After',add_special_tokens=True)
    specials={'declared_marker':special,'tokenizer_marker_id':t.token_to_id(special['content']),
      'explicit_marker':explicit,'recognized_explicit_marker':special['id'] in explicit['ids'],
      'decoded_with_markers':t.decode(explicit['ids'],skip_special_tokens=False),
      'decoded_skipping_markers':t.decode(explicit['ids'],skip_special_tokens=True),
      'automatic_insertion_comparison':{'input':'Before After','disabled_ids':plain_false.ids,'enabled_ids':plain_true.ids,'ids_changed':plain_false.ids!=plain_true.ids},
      'recognition_note':'Compare declared ID and token strings; special_tokens_mask alone is not proof of recognition'}
    long_text=' hello'*1100
    long=observe(t,long_text,'over-companion-context','length')
    if long['token_count']<=1024:raise RuntimeError('Long fixture did not exceed 1024 tokens')
    results={'fixtures':observations,'per_id_unicode_decode':unicode_comparison,'special_tokens':specials,'over_1024_tokens':long}
    repeated=[observe(t,json.loads(literal),label,group) for group,label,literal in FIXTURES]
    results['same_process_repeatability']=observations==repeated
    output_dir.mkdir(parents=True,exist_ok=True)
    dump(output_dir/'manifest.json',manifest);dump(output_dir/'results.json',results)
    mismatches=[item['label'] for item in observations if not item['roundtrip_equal']]
    dump(output_dir/'checks.json',{'fixture_count':len(observations),'roundtrip_mismatch_labels':mismatches,
       'repeatability':results['same_process_repeatability'],'long_token_count':long['token_count'],'long_roundtrip_equal':long['roundtrip_equal']})
    print(f'Saved 18 fixture observations, manifest, special-token/Unicode/length checks. Round-trip mismatches: {len(mismatches)}. See local output files.')
    return results

def main():
    parser=argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--artifacts',type=Path,default=Path('tokenizer-artifacts'))
    parser.add_argument('--output',type=Path,default=Path('tokenizer-results'))
    parser.add_argument('--offline',action='store_true')
    args=parser.parse_args()
    if args.artifacts.resolve()==args.output.resolve():parser.error('Artifact and output directories must differ')
    try:run(args.artifacts,args.output,args.offline)
    except (RuntimeError,ValueError) as error:raise SystemExit(str(error))
if __name__=='__main__':main()