"""Extract independent IMM5009 contracts and create its supplemental intake. Existing intake seeds and existing form assets are never written here. """ from collections import defaultdict import hashlib import json from pathlib import Path import pikepdf from lxml import etree as E BASE = Path(__file__).resolve().parents[2] PREFIX = 'immigration_document_request_hub.imm5009.' def save(path, value): path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') def extract(blank): with pikepdf.open(blank) as pdf: x = pdf.Root.AcroForm.XFA packets = {str(x[i]): x[i+1].read_bytes() for i in range(0, len(x), 2)} template = E.fromstring(packets['template']) data = E.fromstring(packets['datasets'])[0][0] fields = {} def children(node): for c in node: if not isinstance(c.tag, str) or E.QName(c).localname not in ('subform', 'field', 'exclGroup'): continue if c.get('name'): yield c else: yield from children(c) def bind(t, d, prefix): counters = defaultdict(int) for n in children(t): if n.xpath('./*[local-name()="bind" and @match="none"]'): continue name = n.get('name') matches = [c for c in d if isinstance(c.tag, str) and E.QName(c).localname == name] index = counters[name] counters[name] += 1 if index >= len(matches): raise ValueError(f'Unresolved IMM5009 binding: {prefix}.{name}[{index}]') path = prefix + '.' + name + (f'[{index}]' if len(matches) > 1 else '') if E.QName(n).localname == 'subform': bind(n, matches[index], path) continue choices = {} if E.QName(n).localname == 'exclGroup': choices = {c.get('name'): c.xpath('./*[local-name()="items"]/*/text()')[0] for c in n.xpath('./*[local-name()="field"]')} elif n.xpath('./*[local-name()="items"]'): choices = {str(i): v for i, v in enumerate(n.xpath('./*[local-name()="items"]/*/text()'))} limits = n.xpath('./*[local-name()="value"]/*/@maxChars') values = n.xpath('./*[local-name()="value"]/*') fields[path] = {'choices': choices, 'max_chars': int(limits[0]) if limits else 2000, 'value_type': E.QName(values[0]).localname if values else 'text', 'numeric': bool(n.xpath('./*[local-name()="ui"]/*[local-name()="numericEdit"]')), 'pictures': n.xpath('./*[local-name()="format"]/*[local-name()="picture"]/text()'), 'caption': ' '.join(n.xpath('./*[local-name()="caption"]//text()')).strip(), 'tooltip': ' '.join(n.xpath('./*[local-name()="assist"]//text()')).strip()} root = template.xpath('./*[local-name()="subform" and @name="IMM_5009"]')[0] bind(root, data, 'IMM_5009') return fields def build(): contracts = {} for lang, suffix in [('en', 'e'), ('fr', 'f')]: blank = BASE / f'storage/app/imm/imm5009{suffix}.pdf' fields = extract(blank) fr = lang == 'fr' a = 'IMM_5009.Page1.partA.' if fr else 'IMM_5009.page1.' b = 'IMM_5009.Page1.partB.' if fr else 'IMM_5009.page1.PartB.' p2 = 'IMM_5009.Page2.' if fr else 'IMM_5009.page2.' p3 = 'IMM_5009.Page3.' if fr else 'IMM_5009.page3.' targets = { 'uci': a + ('clientIDno' if fr else 'header.clientIDno'), 'entry_family_name': a + ('familyName' if fr else 'Surname'), 'entry_given_names': a + ('givenName' if fr else 'Given_name'), 'current_family_name': a + ('otherName[0]' if fr else 'Current_surname'), 'current_given_names': a + ('otherName[1]' if fr else 'Current_given_name'), 'dob': a+'dateBox[0].theDate', 'birth_place': a+('pob[0]' if fr else 'Place_of_birth'), 'citizenship': a+('pob[1]' if fr else 'Citizenship'), 'passport_number': a+'passportNo', 'passport_kind': a+'passportGroup', 'passport_issue': a+'dateBox[1].theDate', 'passport_expiry': a+'dateBox[2].theDate', 'entry_marital_status': a+'MaritalGroup', 'entry_spouse_canadian': a+'NoYesGroup', 'language': a+'languageGroup', 'mailing_po_box': a+'P.O.box', 'mailing_unit': a+'apt[0]', 'mailing_street_number': a+'street_no[0]', 'mailing_street_name': a+'Street_name[0]', 'mailing_city': a+('city[0]' if fr else 'City-Town'), 'mailing_country': a+('TextField3' if fr else 'TextField5'), 'mailing_province': a+('TextField2' if fr else 'TextField4'), 'mailing_postal_code': a+('pc[0]' if fr else 'Postal_code'), 'mailing_district': a+('pc[1]' if fr else 'District[0]'), 'same_address': a+'RadioButtonList[0]', 'residential_unit': a+'apt[1]', 'residential_street_number': a+'street_no[1]', 'residential_street_name': a+'Street_name[1]', 'residential_city': a+('city[1]' if fr else 'city'), 'residential_country': a+('TextField4' if fr else 'TextField2'), 'residential_province': a+('TextField5' if fr else 'TextField3'), 'residential_postal_code': a+('pc[2]' if fr else 'pc'), 'residential_district': a+('pc[3]' if fr else 'District[1]'), 'email': a+('email' if fr else 'Email_address'), 'request_vos': b+('sub1B.' if fr else '')+'vos', 'request_replacement': b+('sub1B.' if fr else '')+'replacement', 'first_document_issue': b+'dateBox.theDate', 'first_document_place': b+'CndPOE-Place_issue', 'document_condition': p2+'sub3B.RadioButtonList', 'document_explanation': p2+'sub3B.TextField1', 'prior_vos': p2+'sub4B.NoYesGroup', 'prior_vos_date': p2+('sub4B.dateBox.theDate' if fr else 'sub4B.theDate'), 'convicted_since_pr': p2+'sub5B.NoYesGroup', 'canadian_citizen': p2+'sub1C.RadioButtonList', 'first_status': p2+'sub3C.RadioButtonList', 'parent_born_canadian': p2+'sub6C.RadioButtonList', 'mother_occupation': p3+'sub8C.mother', 'father_occupation': p3+'sub8C.father', 'military_at_entry': p3+'sub9C.RadioButtonList', } for i, key in enumerate(['phone', 'alternate_phone']): for column, path in [('region',f'RadioButtonList[{i+1}]'),('type',f'DropDownList4[{i}]'), ('country_code',f'TextField1[{i*2}]'),('number',f'PhoneNum[{i}]'),('extension',f'TextField1[{i*2+1}]')]: targets[key+'_'+column] = a+path doc_names = {'copr':'copr','exclusion_order':'study','departure_order':'certDeparture','authorization_return':'visitor', 'temporary_resident_permit':'work','protected_person':'exclusion','work_permit':'deportation', 'study_permit':'departure','visitor_record':'toCome'} for key, name in doc_names.items(): targets['document_'+key] = b+name for i,key in enumerate(['work_permit','study_permit','visitor_record']): for col,path in [('number',f'street_no[{i}]'),('issue',f'theDate[{i*2}]'),('expiry',f'theDate[{i*2+1}]')]: targets[key+'_'+col] = b+path groups = {} for key, section, count, dob in [('other_names','sub2C',4,False),('companions','sub5C',4,True),('parents','sub7C',2,True)]: groups[key] = [] for i in range(count): row = {col:p2+section+'.'+name+f'[{i}]' for col,name in [('family_name','surname'),('given_names','given'),('other_names','other')]} if dob: row['dob']=p2+section+f'.dateBox[{i}].theDate' groups[key].append(row) mapped = set(targets.values()) | {p for rows in groups.values() for row in rows for p in row.values()} manual = {p for p in fields if '.subDeclaration.' in p or '.CheckBox1[' in p} assert mapped | manual == set(fields), sorted(set(fields)-mapped-manual) assert not mapped & manual choices = {} def exports(key, mapping): choices[key] = {meaning:fields[targets[key]]['choices'][name] for meaning,name in mapping.items()} exports('passport_kind',{'entry':'entryButton','current':'currentButton'}) exports('entry_marital_status',dict(zip(['never_married','married','widowed','separated','divorced','common_law'],['NeverButton','MarriedButton','WidowedButton','SeparatedButton','DivorcedButton','PartnerButton']))) exports('entry_spouse_canadian',{'yes':'YesButton','no':'NoButton'}) exports('language',{'en':'englishButton','fr':'frenchButton'}) exports('same_address',{'yes':'Yes' if fr else 'fourCyl','no':'No' if fr else 'sixCyl'}) exports('document_condition',{k:k for k in ['lost','stolen','destroyed','other']}) for k in ['prior_vos','convicted_since_pr']: exports(k,{'yes':'YesButton','no':'NoButton'}) for k in ['canadian_citizen','parent_born_canadian','military_at_entry']: exports(k,{'yes':'yes','no':'no'}) exports('first_status',{'visitor':'visitor','student':'student','worker':'worker','pr':'pr','protected_person':'pp'}) for key in ['phone','alternate_phone']: # French alternate group reverses child order. Its actual captions # identify sixCyl=Canada/US, fourCyl=Other; never infer from order. mapping = {'canada_us':'sixCyl','other':'fourCyl'} if fr and key == 'alternate_phone' else {'canada_us':'Canada-US','other':'Other'} exports(key+'_region',mapping) vals=list(fields[targets[key+'_type']]['choices'].values()) choices[key+'_type']=dict(zip(['residence','cellular','business'],vals)) contracts[lang]={'form':'IMM5009','language':lang,'sha256':hashlib.sha256(blank.read_bytes()).hexdigest(), 'revision':'03-2024','targets':targets,'groups':groups,'choices':choices, 'fields':{p:fields[p] for p in sorted(mapped)},'manual':sorted(manual), 'limits':'Explicit maxChars, otherwise 2000-character draft ceiling; visual fit needs Adobe acceptance.'} save(BASE/f'resources/imm/imm5009-contract.{lang}.json',contracts[lang]) save(BASE/('resources/imm/maps/imm5009'+('.fr' if fr else '')+'.json'),{'form':'IMM5009','fields':[{'xfa':p,'source':PREFIX+k} for k,p in targets.items()]+[{'xfa':p,'source':PREFIX+k} for k,rows in groups.items() for row in rows for p in row.values()]}) intake(contracts['en']) version_path=BASE/'resources/imm/template_versions.json' versions=json.loads(version_path.read_text(encoding='utf-8')) versions['IMM5009']={lang:{'sha256':c['sha256'],'pdf':f'imm5009{suffix}.pdf'} for (lang,suffix),c in zip([('en','e'),('fr','f')],contracts.values())} version_path.write_text(json.dumps(versions,ensure_ascii=False,indent=4)+'\n',encoding='utf-8') def intake(contract): sections=[] for key,en,fr in [('request','Your document request','Votre demande de document'),('entry','Your details on entry to Canada','Vos renseignements à votre entrée au Canada'),('contact','Missing contact details','Coordonnées complémentaires'),('history','Your document history','Historique de vos documents'),('before1973','Permanent resident entry before 1973','Entrée comme résident permanent avant 1973')]: sections.append({'source_key':PREFIX+'section_'+key,'slug':'imm5009-'+key,'title':en,'title_fr':fr,'display_order':len(sections)+1,'questions':[]}) def q(section,key,en,fr,kind='short_text',options=None,when=None,columns=None,required=True,help_en=None,help_fr=None): rule={'all':([{'field':PREFIX+'confirmation','op':'in','value':['yes','Yes']}] if key!='confirmation' else [])} rule['all'] += [{'field':PREFIX+k,'op':'in','value':v+[str(a).capitalize() for a in v]} for k,v in (when or {}).items()] item={'source_key':PREFIX+key,'label':en,'label_fr':fr,'question_type':'structured_fields' if kind == 'record' else kind,'required_default':required, 'display_order':len(sections[section]['questions'])+1,'visibility_rule':rule if rule['all'] else None} if options:item['options']=[{'value':v,'label':e,'label_fr':f} for v,e,f in options] if columns:item['field_schema']={'kind':'repeatable_table' if kind=='repeatable_table' else 'structured_fields','columns' if kind=='repeatable_table' else 'fields':columns} if help_en:item.update(help_text=help_en,help_text_fr=help_fr) sections[section]['questions'].append(item) def col(id,en,fr,type='short_text'):return {'id':id,'label':en,'label_fr':fr,'type':type} yn=lambda sec,key,en,fr,**kw:q(sec,key,en,fr,'yes_no',**kw) yn(0,'confirmation','Has your consultant asked you to request verification of your status or replace an immigration document?','Votre consultant vous a-t-il demandé de faire vérifier votre statut ou de remplacer un document d’immigration?',help_en='IMM5009. Answer only for the named applicant. Master details are reused. This does not sign or submit your application.',help_fr='IMM5009. Répondez uniquement pour la personne nommée. Les renseignements du questionnaire maître sont réutilisés. Ceci ne signe ni ne soumet votre demande.') q(0,'request_type','What do you need?','De quoi avez-vous besoin?','single_select',[('vos','Verification of Status','Vérification du statut'),('replacement','Replacement of a valid temporary resident document','Remplacement d’un document de résident temporaire valide')]) doc_en=['Immigrant Visa / Record of Landing / COPR','Exclusion Order','Departure Order','Authorization to Return to Canada','Temporary Resident Permit','Protected Person','Work Permit','Study Permit','Visitor Record'] doc_fr=['Visa d’immigrant / fiche relative au droit d’établissement / CRP','Mesure d’exclusion','Mesure de départ','Autorisation de revenir au Canada','Permis de séjour temporaire','Personne protégée','Permis de travail','Permis d’études','Fiche du visiteur'] docs=[k.removeprefix('document_') for k in contract['targets'] if k.startswith('document_') and k not in ['document_condition','document_explanation']] q(0,'requested_document','Which immigration document is this request about?','Quel document d’immigration est visé?','single_select',list(zip(docs,doc_en,doc_fr))) q(0,'document_record_number','Document number to select from your Master records, if more than one applies','Numéro du document à sélectionner dans vos dossiers maîtres, si plusieurs correspondent',required=False) q(0,'document_details','Requested document details — only if missing from Master','Document demandé — seulement si absent du questionnaire maître','record',columns=[col('number','Document number, if known','Numéro du document, si connu'),col('issue_date','Issue date','Date de délivrance','date'),col('expiry_date','Expiry date','Date d’expiration','date')],required=False) yn(1,'entry_names_same','Were your names on entry to Canada the same as your current Master names?','Vos noms à votre entrée au Canada étaient-ils les mêmes que vos noms actuels du questionnaire maître?') q(1,'entry_names','Names used on entry','Noms utilisés à l’entrée','record',when={'entry_names_same':['no']},columns=[col('family_name','Family name','Nom de famille'),col('given_names','Given names','Prénoms')]) q(1,'birth_province','Province or state of birth, if applicable','Province ou État de naissance, le cas échéant',required=False) q(1,'passport_source','Which passport or travel document should be used?','Quel passeport ou titre de voyage faut-il utiliser?','single_select',[('master_entry','Master passport used on entry','Passeport du questionnaire maître utilisé à l’entrée'),('master_current','Current Master passport; entry copy unavailable','Passeport actuel du questionnaire maître; copie à l’entrée indisponible'),('provide_entry','Provide missing entry passport details','Fournir le passeport à l’entrée manquant'),('provide_current','Provide current passport; entry copy unavailable','Fournir le passeport actuel; copie à l’entrée indisponible')]) q(1,'passport_record_number','Passport number to select from Master, if more than one applies','Numéro du passeport à sélectionner dans le questionnaire maître, si plusieurs correspondent',required=False) q(1,'passport_details','Missing passport details','Renseignements manquants sur le passeport','record',when={'passport_source':['provide_entry','provide_current']},columns=[col('number','Number','Numéro'),col('issuing_country_authority','Issuing country','Pays de délivrance'),col('issue_date','Issue date','Date de délivrance','date'),col('expiry_date','Expiry date','Date d’expiration','date')]) q(1,'citizenship_for_entry','Citizenship to use if you have multiple current citizenships','Citoyenneté à utiliser si vous avez plusieurs citoyennetés actuelles',required=False,help_en='Choose the country that issued the passport used to enter Canada. Do not repeat this if Master has one current citizenship.',help_fr='Choisissez le pays ayant délivré le passeport utilisé pour entrer au Canada. Inutile si le questionnaire maître indique une seule citoyenneté actuelle.') q(1,'entry_marital_status','What was your marital status on your first entry to Canada?','Quel était votre état matrimonial lors de votre première entrée au Canada?','single_select',list(zip(['never_married','married','widowed','separated','divorced','common_law'],['Never married','Married','Widowed','Separated','Divorced','Common-law partner'],['Jamais marié(e)','Marié(e)','Veuf ou veuve','Séparé(e)','Divorcé(e)','Conjoint de fait']))) yn(1,'entry_spouse_canadian','If married on first entry, was your spouse a Canadian citizen or permanent resident?','Si vous étiez marié(e) à la première entrée, votre conjoint était-il citoyen canadien ou résident permanent?',when={'entry_marital_status':['married']}) q(1,'first_entry_date','Date you first entered Canada','Date de votre première entrée au Canada','date') q(1,'first_status','What was your immigration status on your first entry to Canada?','Quel était votre statut d’immigration lors de votre première entrée au Canada?','single_select',list(zip(['visitor','student','worker','pr','protected_person'],['Visitor','Student','Worker','Permanent resident','Protected person'],['Visiteur','Étudiant','Travailleur','Résident permanent','Personne protégée']))) q(1,'first_document_issue','Issue date of your immigration document on first entry','Date de délivrance de votre document d’immigration à la première entrée','date') q(1,'first_document_place','Canadian port of entry or place where the document/status was issued','Point d’entrée canadien ou lieu de délivrance du document ou du statut') address_cols=[col(k,en,fr) for k,en,fr in [('po_box','P.O. box','Case postale'),('unit','Apartment/unit','Appartement/unité'),('street_number','Street number','Numéro de rue'),('street_name','Street name','Nom de rue'),('city','City/town','Ville'),('country','Country','Pays'),('province','Province/state','Province/État'),('postal_code','Postal code','Code postal'),('district','District','District')]] for key,en,fr in [('mailing','mailing','postale'),('residential','residential','résidentielle')]: q(2,key+'_details',f'Missing {en} address components — provide only details missing or unclear in Master',f'Éléments manquants de l’adresse {fr} — seulement les éléments absents ou ambigus du questionnaire maître','record',columns=address_cols if key=='mailing' else address_cols[1:],required=False) for key,en,fr in [('phone','Primary phone','Téléphone principal'),('alternate_phone','Alternate phone','Autre téléphone')]: q(2,key+'_details',en+' — missing details only',fr+' — renseignements manquants seulement','record',columns=[dict(col('region','Number region','Région du numéro','single_select'),options=[{'value':'canada_us','label':'Canada/US','label_fr':'Canada/É.-U.'},{'value':'other','label':'Other','label_fr':'Autre'}]),dict(col('type','Type','Type','single_select'),options=[{'value':v,'label':e,'label_fr':f} for v,e,f in [('residence','Residence','Résidence'),('cellular','Cellular','Cellulaire'),('business','Business','Affaires')]]),col('country_code','Country calling code','Indicatif de pays'),col('number','Number including area code, without country calling code','Numéro avec indicatif régional, sans indicatif de pays'),col('extension','Extension, if applicable','Poste, le cas échéant')],required=False) yn(2,'email_authorized','May IRCC send correspondence, including personal and file information, to your Master email address?','IRCC peut-il envoyer la correspondance, y compris vos renseignements personnels et de dossier, à votre adresse courriel du questionnaire maître?',required=False) q(3,'document_condition','What happened to your original immigration document?','Qu’est-il arrivé à votre document d’immigration original?','single_select',list(zip(['lost','stolen','destroyed','other'],['Lost','Stolen','Destroyed','Other'],['Perdu','Volé','Détruit','Autre']))) q(3,'document_explanation','Explain what happened, if applicable','Expliquez ce qui est arrivé, le cas échéant','long_text',required=False) q(3,'police_report','Police report number, or explain why you do not have one','Numéro du rapport de police, ou raison pour laquelle vous n’en avez pas','long_text',when={'request_type':['replacement'],'document_condition':['lost','stolen']}) yn(3,'prior_vos','Have you previously applied for VOS of an Immigrant Visa / Record of Landing / COPR?','Avez-vous déjà demandé une vérification du statut pour un visa d’immigrant, une fiche relative au droit d’établissement ou une CRP?') q(3,'prior_vos_date','When did you apply for that VOS?','Quand avez-vous demandé cette vérification du statut?','date',when={'prior_vos':['yes']}) yn(3,'canadian_citizen','Are you currently a Canadian citizen?','Êtes-vous actuellement citoyen canadien?') yn(3,'convicted_since_pr','Since becoming a permanent resident, have you been convicted of a crime or offence in Canada or elsewhere?','Depuis votre admission comme résident permanent, avez-vous été reconnu coupable d’un crime ou d’une infraction au Canada ou ailleurs?',when={'requested_document':['copr'],'canadian_citizen':['no']},help_en='Answer the exact declaration. If Yes, relevant court documents must be attached manually.',help_fr='Répondez à cette déclaration précise. Si oui, joignez manuellement les documents judiciaires pertinents.') q(3,'other_names_source','Which other names did you use at landing or entry to Canada?','Quels autres noms utilisiez-vous à l’établissement ou à l’entrée au Canada?','single_select',[('none','No other names','Aucun autre nom'),('master','All applicable names are recorded in Master','Tous les noms pertinents figurent au questionnaire maître'),('provide','Provide the complete applicable list here','Fournir ici la liste complète')]) names=[col('family_name','Family name','Nom de famille'),col('given_names','Given names','Prénoms'),col('other_names','Other names/initials','Autres noms/initiales')] q(3,'other_names','All applicable other names','Tous les autres noms pertinents','repeatable_table',when={'other_names_source':['provide']},columns=names) yn(4,'before1973','Did you first enter Canada as a permanent resident before 1973?','Êtes-vous entré au Canada pour la première fois comme résident permanent avant 1973?') old={'before1973':['yes']} yn(4,'has_companions','Did anyone enter Canada with you on that arrival?','Quelqu’un est-il entré au Canada avec vous à cette arrivée?',when=old) q(4,'companions','People who entered Canada with you on that arrival','Personnes entrées au Canada avec vous à cette arrivée','repeatable_table',when=old|{'has_companions':['yes']},columns=names+[col('dob','Date of birth','Date de naissance','date')]) yn(4,'parent_born_canadian','Was either parent born a Canadian citizen?','L’un de vos parents est-il né citoyen canadien?',when=old) yn(4,'parents_master_confirmed','Are your Master parent names and dates of birth correct for this historical record?','Les noms et dates de naissance des parents du questionnaire maître sont-ils exacts pour ce dossier historique?',when=old) q(4,'parents','Parent names and dates of birth — only if Master is missing or incorrect','Noms et dates de naissance des parents — seulement si absents ou inexacts dans le questionnaire maître','repeatable_table',when=old|{'parents_master_confirmed':['no']},columns=names+[col('dob','Date of birth','Date de naissance','date')]) for key,en,fr in [('mother','mother','mère'),('father','father','père')]: yn(4,key+'_occupation_applicable',f'Is your {en}’s occupation at entry applicable?',f'La profession de votre {fr} à l’entrée est-elle applicable?',when=old,help_en='No means not applicable, not unknown. If unknown, leave unanswered and ask your consultant.',help_fr='Non signifie sans objet, et non inconnu. Si vous l’ignorez, laissez sans réponse et consultez votre consultant.') q(4,key+'_occupation',f'Your {en}’s occupation at the time you entered Canada',f'Profession de votre {fr} au moment de votre entrée au Canada',when=old|{key+'_occupation_applicable':['yes']}) yn(4,'military_at_entry','At the time you entered Canada, had you or your parents ever been part of the military?','Au moment de votre entrée au Canada, vous-même ou vos parents aviez-vous déjà fait partie des forces armées?',when=old) save(BASE/'database/data/immigration_document_request_hub_questionnaire.json',{'questionnaire_key':'immigration_document_request_hub','version':'1.0.0','section_count':len(sections),'question_count':sum(len(s['questions']) for s in sections),'sections':sections}) if __name__ == '__main__': build()