"""IMM0008 stored-XFA regression tests; Reader validation remains manual.""" import hashlib import json import os from pathlib import Path import re import subprocess import sys import unittest from uuid import uuid4 from unittest.mock import patch import pikepdf from lxml import etree as E from prefill_imm_xfa_pdf import fill BASE = Path(__file__).resolve().parents[2] def packets(path): with pikepdf.open(path) as pdf: x = pdf.Root.AcroForm.XFA return {str(x[i]): x[i+1].read_bytes() for i in range(0,len(x),2)} class IMM0008WriterTest(unittest.TestCase): def test_phone_saved_state_matches_adobe_dataset_binding_and_requirements(self): # Observed in IMM0008_wait.pdf after Reader's official Canada/US click: # only selected checkbox has value override; digits bind to datasets; # all required validators exist, including on initially hidden widgets. from imm0008_pdf import parse, resolve for language in ('en', 'fr'): for group in ('Phone', 'AltPhone'): prefix = 'form1.Page1.contactInformation.q3-4.' + group + '.' fields = {prefix+'CanadaUS':'1', prefix+'Other':'0', prefix+'Type':'02', prefix+'NumberCountry':'1', prefix+'NANumber.AreaCode':'416', prefix+'NANumber.FirstThree':'555', prefix+'NANumber.LastFive':'0100'} saved = packets(self.write({'form':'IMM0008','language':language,'fields':fields})) form = parse(saved['form']).getroot() self.assertEqual(form.xpath('//*[local-name()="script"]'), []) data = parse(saved['datasets']).getroot().find('{http://www.xfa.org/schema/xfa-data/1.0/}data') for leaf in ('CanadaUS','Type','NANumber.AreaCode','NANumber.FirstThree','NANumber.LastFive'): widget = resolve(form, prefix+leaf, named=True) self.assertEqual(widget.xpath('./*[local-name()="validate"]/@nullTest'), ['error'], leaf) for child in widget: if isinstance(child.tag, str): self.assertEqual(E.QName(child).namespace, E.QName(form).namespace) for leaf, value in fields.items(): widget = resolve(form, leaf, named=True) self.assertEqual(widget.xpath('./*[local-name()="value"]/*'), [], leaf) self.assertEqual(resolve(data, leaf).text, value) self.assertIsNone(resolve(data, leaf).get('{http://www.xfa.org/schema/xfa-data/1.0/}dataNode')) self.assertNotIn(b'IMM0008_PHONE_RESTORE', saved['form']) self.assertNotIn((prefix+'NANumber').encode(), saved['form'].split(b'IMM0008_PREFILL_STATE')[-1]) def test_phone_missing_selection_is_not_guessed_and_other_branch_is_required(self): from imm0008_pdf import parse, resolve prefix = 'form1.Page1.contactInformation.q3-4.Phone.' for language in ('en', 'fr'): for fields in ({}, {prefix+'CanadaUS':'0', prefix+'Other':'1', prefix+'Type':'02', prefix+'NumberCountry':'94', prefix+'IntlNumber.IntlNumber':'770000001'}): before = dict(fields) saved = packets(self.write({'form':'IMM0008','language':language,'fields':fields})) state = parse(saved['form']).getroot() for leaf in ('CanadaUS','Other'): node = resolve(state, prefix+leaf, named=True) self.assertEqual(node.xpath('./*[local-name()="value"]/@override'), ['1'] if fields.get(prefix+leaf)=='1' else []) for leaf in ('Type','NumberCountry','IntlNumber.IntlNumber'): node = resolve(state, prefix+leaf, named=True) self.assertEqual(node.xpath('./*[local-name()="validate"]/@nullTest'), ['error'] if fields else ['disabled']) self.assertEqual(fields, before) def setUp(self): self.work = BASE/'storage/app/imm/tmp'/('0008-test-'+uuid4().hex) self.work.mkdir(parents=True) def contract(self, lang): return json.loads((BASE/f'resources/imm/imm0008-contract.{lang}.json').read_text(encoding='utf-8')) def blank(self, lang): return BASE/'storage/app/imm'/('imm0008e.pdf' if lang=='en' else 'imm0008f.pdf') def write(self, payload, blank=None): source=self.work/(uuid4().hex+'.json') source.write_text(json.dumps(payload),encoding='utf-8') output=source.with_suffix('.pdf') fill(source,output,blank or self.blank(payload.get('language','en'))) return output def test_reject_unknown_before_output(self): with self.assertRaisesRegex(ValueError,'Unallowlisted IMM0008'): self.write({'form':'IMM0008','fields':{'form1.Page1.fake':'bad'}}) self.assertFalse(list(self.work.glob('*.pdf'))) def test_phone_selectors_and_digits_bind_to_dataset_without_scalar_overrides(self): from imm0008_pdf import parse, resolve for language in ('en', 'fr'): prefix = 'form1.Page1.contactInformation.q3-4.Phone.' fields = {prefix + 'CanadaUS': '1', prefix + 'Other': '0', prefix + 'Type': '02', prefix + 'NumberCountry': '1', prefix + 'NANumber.AreaCode': '416', prefix + 'NANumber.FirstThree': '555', prefix + 'NANumber.LastFive': '0100'} output = self.write({'form': 'IMM0008', 'language': language, 'fields': fields}) state = parse(packets(output)['form']).getroot() data = parse(packets(output)['datasets']).getroot().find('{http://www.xfa.org/schema/xfa-data/1.0/}data') contract = self.contract(language) for path, value in fields.items(): meta = contract['fields'][path] node = resolve(state, meta.get('state_path', path), named=True) if path.endswith('.CanadaUS'): self.assertEqual(node.xpath('./*[local-name()="value"]/@override'), ['1']) self.assertEqual(node.xpath('./*[local-name()="value"]/*'), []) self.assertEqual(len(node.xpath('./*[local-name()="ui"]/*[local-name()="checkButton"]')), 1) else: self.assertEqual(node.xpath('./*[local-name()="value"]'), [], path) leaf = resolve(data, meta.get('data_path', path)) self.assertEqual(leaf.text, value, path) self.assertIsNone(leaf.get('{http://www.xfa.org/schema/xfa-data/1.0/}dataNode'), path) self.assertIsNone(resolve(data, prefix + 'NANumber').get('{http://www.xfa.org/schema/xfa-data/1.0/}dataNode')) def test_no_dependants_hides_default_personal_details_in_saved_form(self): for language in ('en', 'fr'): output = self.write({'form': 'IMM0008', 'language': language, 'fields': {}, 'imm0008_dependent_count': 0}) state = E.fromstring(packets(output)['form']) for name in ('depPrime', 'depInfoHeader', 'depHeader'): self.assertEqual(state.xpath('//*[@name="Page1"]/*[@name="'+name+'"]/@presence'), ['hidden'], name) def test_saved_form_does_not_override_signed_document_ready_script(self): for language in ('en', 'fr'): prefix = 'form1.Page1.contactInformation.q3-4.Phone.' output = self.write({'form': 'IMM0008', 'language': language, 'fields': {prefix+'CanadaUS':'1', prefix+'Other':'0'}}) saved = packets(output) state = E.fromstring(saved['form']) self.assertEqual(state.xpath('//*[local-name()="script"]'), []) self.assertEqual(saved['template'], packets(self.blank(language))['template']) def test_saved_form_checksum_matches_changed_dataset(self): from imm0008_xfa_checksum import form_checksum prefix = 'form1.Page1.contactInformation.q3-4.Phone.' for language in ('en', 'fr'): output = self.write({'form': 'IMM0008', 'language': language, 'fields': {prefix + 'CanadaUS': '1', prefix + 'NANumber.AreaCode': '416'}}) generated = packets(output) actual = E.fromstring(generated['form']).get('checksum') self.assertNotEqual(actual, E.fromstring(packets(self.blank(language))['form']).get('checksum')) self.assertEqual(actual, form_checksum(generated['template'], generated['datasets'])) @unittest.skipUnless(os.environ.get('IMM_XFA_RUNTIME'), 'Optional native V8/XFA rendering runtime') def test_native_open_preserves_phone_dataset_values(self): from imm0008_pdf import resolve for language in ('en', 'fr'): primary = 'form1.Page1.contactInformation.q3-4.Phone.' alternate = 'form1.Page1.contactInformation.q3-4.AltPhone.' fields = {primary + 'CanadaUS': '1', primary + 'Other': '0', primary + 'Type': '02', primary + 'NumberCountry': '1', primary + 'NANumber.AreaCode': '416', primary + 'NANumber.FirstThree': '555', primary + 'NANumber.LastFive': '0100', alternate + 'CanadaUS': '0', alternate + 'Other': '1', alternate + 'Type': '02', alternate + 'NumberCountry': '94', alternate + 'IntlNumber.IntlNumber': '770000001'} output = self.write({'form': 'IMM0008', 'language': language, 'fields': fields}) native = output.with_name(output.stem + '-opened.pdf') # Adobe checks this before restoring saved state; PDFium omits that # check. Gate the runtime test on an interoperable checksum first. from imm0008_xfa_checksum import form_checksum generated = packets(output) self.assertEqual(E.fromstring(generated['form']).get('checksum'), form_checksum(generated['template'], generated['datasets'])) probe = '''import sys sys.path.insert(0, sys.argv[1]) import pypdfium2_raw.bindings as raw raw.FPDF_ERR_XFALOAD = 7 raw.FPDF_ERR_XFALAYOUT = 8 import pypdfium2 as pdfium doc = pdfium.PdfDocument(sys.argv[2]) doc.init_forms() for page in doc: page.render(scale=1, may_draw_forms=True) doc.save(sys.argv[3]) doc.close() ''' result = subprocess.run([sys.executable, '-c', probe, os.environ['IMM_XFA_RUNTIME'], str(output), str(native)], capture_output=True, text=True, timeout=120) self.assertEqual(result.returncode, 0, result.stderr) state = E.fromstring(packets(native)['form']) # This runtime also drops branch deltas in the actual Reader-saved # reference. It proves data merge only, not Reader state restoration. for path, expected in fields.items(): actual = resolve(state, path, named=True).xpath('./*[local-name()="value"]/*/text()') self.assertEqual(''.join(actual).strip(), expected, path) @unittest.skipUnless(os.name == 'nt', 'Windows web-server environment regression') def test_web_server_child_without_systemroot_generates_downloadable_pdf(self): environment = {key: value for key, value in os.environ.items() if key.upper() != 'SYSTEMROOT'} self.assertTrue(environment.get('WINDIR')) for language in ('en', 'fr'): # Exercise both inherited WINDIR and the native OS-directory fallback. if language == 'fr': environment = {key: value for key, value in environment.items() if key.upper() != 'WINDIR'} source = self.work / (language + '-web.json') output = source.with_suffix('.pdf') source.write_text(json.dumps({'form': 'IMM0008', 'language': language, 'fields': {}}), encoding='utf-8') result = subprocess.run([sys.executable, str(BASE / 'resources/imm/prefill_imm_xfa_pdf.py'), str(source), str(output), str(self.blank(language))], env=environment, capture_output=True, text=True, timeout=60) self.assertEqual(result.returncode, 0, result.stderr) self.assertTrue(output.read_bytes().startswith(b'%PDF')) with pikepdf.open(output) as pdf: rights = [str(right) for right in pdf.Root.Perms.UR3.Reference[0].TransformParams.Form] self.assertIn('/BarcodePlaintext', rights) def test_signed_original_and_barcode_rights_survive_incremental_fill(self): from asn1crypto import cms for language in ('en', 'fr'): blank = self.blank(language) fields = {'form1.Page1.contactInformation.q1.AddressRow2.Country.Country': '511'} output = self.write({'form': 'IMM0008', 'language': language, 'fields': fields}) self.assertTrue(output.read_bytes().startswith(blank.read_bytes())) with pikepdf.open(blank) as original, pikepdf.open(output) as result: self.assertTrue(result.is_encrypted) self.assertEqual(str(original.trailer.Encrypt), str(result.trailer.Encrypt)) for permission in ('/UR3', '/DocMDP'): before = original.Root.Perms[permission] after = result.Root.Perms[permission] self.assertEqual(list(before.ByteRange), list(after.ByteRange)) self.assertEqual(bytes(before.Contents), bytes(after.Contents)) ranges = list(after.ByteRange) signed = lambda raw: b''.join(raw[ranges[i]:ranges[i]+ranges[i+1]] for i in range(0, len(ranges), 2)) self.assertEqual(signed(blank.read_bytes()), signed(output.read_bytes())) signer = cms.ContentInfo.load(bytes(after.Contents))['content']['signer_infos'][0] algorithm = signer['digest_algorithm']['algorithm'].native digest = next(attr['values'][0].native for attr in signer['signed_attrs'] if attr['type'].native == 'message_digest') self.assertEqual(digest, hashlib.new(algorithm, signed(output.read_bytes())).digest()) rights = result.Root.Perms.UR3.Reference[0].TransformParams.Form self.assertIn('/BarcodePlaintext', [str(right) for right in rights]) before_packets, after_packets = packets(blank), packets(output) for name in before_packets.keys() - {'datasets', 'form'}: self.assertEqual(before_packets[name], after_packets[name], name) def test_canada_us_district_cleared_in_dataset_and_saved_state(self): from imm0008_pdf import resolve, parse prefix = 'form1.Page1.contactInformation.' for language in ('en', 'fr'): for country in ('511', '461', '241'): fields = {prefix + 'q2.SameAsMailingIndicator': 'N'} for base in ('q1.AddressRow2.', 'q2.ResidentialAddressRow2.'): fields[prefix + base + 'Country.Country'] = country fields[prefix + base + 'District'] = 'Colombo' output = self.write({'form': 'IMM0008', 'language': language, 'fields': fields}) raw = packets(output) data = parse(raw['datasets']).getroot().find('{http://www.xfa.org/schema/xfa-data/1.0/}data') state = parse(raw['form']).getroot() contract = self.contract(language) for base in ('q1.AddressRow2.', 'q2.ResidentialAddressRow2.'): path = prefix + base + 'District' meta = contract['fields'][path] expected = 'Colombo' if country == '241' else '' self.assertEqual(resolve(data, meta.get('data_path', path)).text or '', expected) saved = resolve(state, meta.get('state_path', path), named=True) self.assertEqual(saved.xpath('string(./*[local-name()="value"]/*)'), expected) original = packets(self.blank(language)) pattern = rb']*/>)[^>]*>.*?' self.assertEqual(re.findall(pattern, original['template'], re.S), re.findall(pattern, raw['template'], re.S)) def test_manual_targets_rejected(self): for path in ('form1.SignatureField1','form1.Page1.declaration.Consent1.TextField2', 'form1.Page1.declaration.C1CertificateIssueDate','form1.Page1.PersonalDetails.ApplicationValidatedFlag'): with self.subTest(path=path), self.assertRaisesRegex(ValueError,'Unallowlisted IMM0008'): self.write({'form':'IMM0008','fields':{path:'stale'}}) def test_empty_php_fields_array_downloads_blank_draft_without_invented_values(self): for language in ('en', 'fr'): output = self.write({'form': 'IMM0008', 'language': language, 'fields': [], 'imm0008_dependent_count': 0}) data = E.fromstring(packets(output)['datasets']) page = data.xpath('//*[local-name()="data"]/*[local-name()="form1"]/*[local-name()="Page1"]')[0] self.assertEqual([], page.findall('depPrime')) self.assertEqual([], page.findall('dependants')) with self.assertRaisesRegex(ValueError, 'fields must be an object'): self.write({'form': 'IMM0008', 'fields': ['invalid']}) def test_unresolved_contract_target_rejected(self): contract=self.contract('en') path='form1.Page1.genDetails.q2.NumberFamilyMembers' path=next(iter(contract['fields'])) contract['fields'][path]['template_path']='form1.Page1.fake' with patch('imm0008_pdf.load_contract',return_value=contract): with self.assertRaisesRegex(ValueError,'Unresolved IMM0008'): self.write({'form':'IMM0008','fields':{path:''}}) self.assertFalse(list(self.work.glob('*.pdf'))) def test_never_overwrite_original_blank(self): source=self.work/'payload.json' source.write_text(json.dumps({'form':'IMM0008','fields':{}})) blank=self.blank('en') before=hashlib.sha256(blank.read_bytes()).hexdigest() with self.assertRaisesRegex(ValueError,'must not overwrite'): fill(source,blank,blank) self.assertEqual(before,hashlib.sha256(blank.read_bytes()).hexdigest()) def test_bad_metadata_and_hash_rejected(self): for count in (-1,6,'2',True): with self.subTest(count=count), self.assertRaisesRegex(ValueError,'dependent count'): self.write({'form':'IMM0008','fields':{},'imm0008_dependent_count':count}) blank=self.work/'altered.pdf' blank.write_bytes(self.blank('en').read_bytes()+b'\n') with self.assertRaisesRegex(ValueError,'template requires review'): self.write({'form':'IMM0008','fields':{}},blank) def test_exact_dependent_instances_and_stable_invalid_middle(self): for lang in ('en','fr'): contract=self.contract(lang) names=[p for p in contract['fields'] if p.endswith('.q1.FamilyName') and ('.depPrime.' in p or '.dependants[' in p)] self.assertEqual(5,len(names)) for count in (0,1,2,5): with self.subTest(language=lang,count=count): fields={p:'Person'+str(i) for i,p in enumerate(names[:count]) if i!=1} output=self.write({'form':'IMM0008','language':lang,'fields':fields,'imm0008_dependent_count':count}) raw=packets(output) data=E.fromstring(raw['datasets']) page=data.xpath('//*[local-name()="data"]/*[local-name()="form1"]/*[local-name()="Page1"]')[0] self.assertEqual(int(count>0),len(page.findall('depPrime'))) self.assertEqual(max(0,count-1),len(page.findall('dependants'))) if count==5: self.assertEqual('Person2',page.findall('dependants')[1].find('q1/FamilyName').text) saved=E.fromstring(raw['form']) saved_page=saved.xpath('//*[@name="form1"]/*[@name="Page1"]')[0] self.assertEqual(max(0,count-1),len(saved_page.xpath('./*[@name="dependants"]'))) original=packets(self.blank(lang)) lov_pattern=rb'' self.assertEqual(re.search(lov_pattern,original['datasets'],re.S)[0],re.search(lov_pattern,raw['datasets'],re.S)[0]) for name in original.keys()-{'template','datasets','form'}: self.assertEqual(original[name],raw[name],name) # Every official script remains byte-identical, including line endings. pattern=rb']*/>)[^>]*>.*?' self.assertEqual(re.findall(pattern,original['template'],re.S),re.findall(pattern,raw['template'],re.S)) def test_full_contract_dataset_and_saved_values(self): from imm0008_pdf import resolve, parse from imm0008_pdf_state import normalize_fields for lang in ('en','fr'): contract=self.contract(lang) fields={} for i,(path,meta) in enumerate(contract['fields'].items()): choices=meta.get('allowed_values') or meta.get('choices') value=next(iter(choices)) if choices else ('V'+str(i))[:meta.get('max_chars',2000)] fields[path]=value fields={p:v for p,v in normalize_fields(fields).items() if p in contract['fields']} output=self.write({'form':'IMM0008','language':lang,'fields':fields,'imm0008_dependent_count':5}) raw=packets(output) data=parse(raw['datasets']).getroot().find('{http://www.xfa.org/schema/xfa-data/1.0/}data') form=parse(raw['form']).getroot() for path,value in fields.items(): meta=contract['fields'][path] with self.subTest(language=lang,path=path): self.assertEqual(value,resolve(data,meta.get('data_path',path)).text or '') node=resolve(form,meta.get('state_path',path),named=True) values=node.xpath('./*[local-name()="value"]/*') if ('.q3-4.' in path or '.q5-6.FaxPhone.' in path) and path.rsplit('.', 1)[-1] in ('CanadaUS', 'Other') and value == '1': self.assertEqual(node.xpath('./*[local-name()="value"]/@override'), ['1']) self.assertEqual(values, []) continue if path.startswith(('form1.Page1.contactInformation.q3-4.Phone.', 'form1.Page1.contactInformation.q3-4.AltPhone.', 'form1.Page1.contactInformation.q5-6.FaxPhone.')): self.assertEqual(values, []) continue self.assertEqual(value,values[0].text or '') if __name__=='__main__': unittest.main()