diff --git a/mhr_api/src/mhr_api/models/db2/manuhome.py b/mhr_api/src/mhr_api/models/db2/manuhome.py index 2dc1bd722..61efb749a 100644 --- a/mhr_api/src/mhr_api/models/db2/manuhome.py +++ b/mhr_api/src/mhr_api/models/db2/manuhome.py @@ -80,22 +80,34 @@ def find_by_id(cls, id: int): manuhome = None if id and id > 0: try: + current_app.logger.debug('Db2Manuhome.find_by_id query.') manuhome = cls.query.get(id) except Exception as db_exception: # noqa: B902; return nicer error current_app.logger.error('Db2Manuhome.find_by_id exception: ' + str(db_exception)) raise DatabaseException(db_exception) if manuhome: - manuhome.reg_documents = Db2Document.find_by_mhr_number(manuhome.mhr_number) - manuhome.reg_owners = Db2Owner.find_by_manuhome_id(manuhome.id) + current_app.logger.debug('Db2Manuhome.find_by_id Db2Document query.') + documents = [] + doc = Db2Document.find_by_doc_id(manuhome.reg_document_id) + if doc: + documents.append(doc) + manuhome.reg_documents = documents + current_app.logger.debug('Db2Manuhome.find_by_id Db2Owner query.') + manuhome.reg_owners = Db2Owner.find_by_manuhome_id_registration(manuhome.id) + current_app.logger.debug('Db2Manuhome.find_by_id Db2Descript query.') manuhome.reg_descript = Db2Descript.find_by_manuhome_id_active(manuhome.id) + current_app.logger.debug('Db2Manuhome.find_by_id Db2Location query.') manuhome.reg_location = Db2Location.find_by_manuhome_id_active(manuhome.id) + current_app.logger.debug('Db2Manuhome.find_by_id Db2Mhomnote query.') manuhome.reg_notes = Db2Mhomnote.find_by_manuhome_id_active(manuhome.id) + current_app.logger.debug('Db2Manuhome.find_by_id completed.') return manuhome @classmethod def find_by_mhr_number(cls, mhr_number: str): """Return the MH registration matching the MHR number.""" manuhome = None + current_app.logger.debug(f'Db2Manuhome.find_by_mhr_number {mhr_number}.') if mhr_number: try: manuhome = cls.query.filter(Db2Manuhome.mhr_number == mhr_number).one_or_none() @@ -109,11 +121,17 @@ def find_by_mhr_number(cls, mhr_number: str): mhr_number=mhr_number), status_code=HTTPStatus.NOT_FOUND ) + current_app.logger.debug('Db2Manuhome.find_by_mhr_number Db2Document query.') manuhome.reg_documents = Db2Document.find_by_mhr_number(manuhome.mhr_number) + current_app.logger.debug('Db2Manuhome.find_by_mhr_number Db2Owner query.') manuhome.reg_owners = Db2Owner.find_by_manuhome_id(manuhome.id) + current_app.logger.debug('Db2Manuhome.find_by_mhr_number Db2Descript query.') manuhome.reg_descript = Db2Descript.find_by_manuhome_id_active(manuhome.id) + current_app.logger.debug('Db2Manuhome.find_by_mhr_number Db2Location query.') manuhome.reg_location = Db2Location.find_by_manuhome_id_active(manuhome.id) + current_app.logger.debug('Db2Manuhome.find_by_mhr_number Db2Mhomnote query.') manuhome.reg_notes = Db2Mhomnote.find_by_manuhome_id_active(manuhome.id) + current_app.logger.debug('Db2Manuhome.find_by_mhr_number completed.') return manuhome @property diff --git a/mhr_api/src/mhr_api/models/db2/owner.py b/mhr_api/src/mhr_api/models/db2/owner.py index 415aa2d37..c9b84bce8 100644 --- a/mhr_api/src/mhr_api/models/db2/owner.py +++ b/mhr_api/src/mhr_api/models/db2/owner.py @@ -15,6 +15,7 @@ from enum import Enum from flask import current_app +from sqlalchemy.sql import text from mhr_api.exceptions import DatabaseException from mhr_api.models import db, utils as model_utils @@ -22,6 +23,18 @@ from .owngroup import Db2Owngroup +OWNERS_QUERY = """ +select o.manhomid, o.owngrpid, o.ownerid, o.ownseqno, o.verified, o.ownrfone, o.ownrpoco, o.ownrname, o.ownrsuff, + o.ownraddr, og.copgrpid, og.grpseqno, og.status, og.pending, og.regdocid, og.candocid, og.tenytype, + og.lessee, og.lessor, og.interest, og.intnumer, og.tenyspec, o.ownrtype + from owner o, owngroup og + where o.manhomid = :query_value + and og.manhomid = o.manhomid + and o.owngrpid = og.owngrpid + and og.status in ('3', '4') +""" + + class Db2Owner(db.Model): """This class manages all of the legacy DB2 MHR owner nformation.""" @@ -86,6 +99,53 @@ def find_by_manuhome_id(cls, manuhome_id: int): owner.owngroup = Db2Owngroup.find_by_manuhome_id(owner.manuhome_id, owner.group_id) return owners + @classmethod + def find_by_manuhome_id_registration(cls, manuhome_id: int): + """Return the owners matching the manuhome id.""" + owners = None + rows = None + if manuhome_id and manuhome_id > 0: + try: + query = text(OWNERS_QUERY) + result = db.get_engine(current_app, 'db2').execute(query, {'query_value': manuhome_id}) + rows = result.fetchall() + except Exception as db_exception: # noqa: B902; return nicer error + current_app.logger.error('DB2Owner.find_by_manuhome_id exception: ' + str(db_exception)) + raise DatabaseException(db_exception) + if rows is not None: + owners = [] + for row in rows: + owner: Db2Owner = Db2Owner(manuhome_id=manuhome_id) + owngroup: Db2Owngroup = Db2Owngroup(manuhome_id=manuhome_id) + owner.group_id = int(row[1]) + owner.owner_id = int(row[2]) + owner.sequence_number = int(row[3]) + owner.verified_flag = str(row[4]) + owner.phone_number = str(row[5]) + owner.postal_code = str(row[6]) + owner.name = str(row[7]) + owner.suffix = str(row[8]) + owner.legacy_address = str(row[9]) + owngroup.group_id = owner.group_id + owngroup.copy_id = int(row[10]) + owngroup.sequence_number = int(row[11]) + owngroup.status = str(row[12]) + owngroup.pending_flag = str(row[13]) + owngroup.reg_document_id = str(row[14]) + owngroup.can_document_id = str(row[15]) + owngroup.tenancy_type = str(row[16]) + owngroup.lessee = str(row[17]) + owngroup.lessor = str(row[18]) + owngroup.interest = str(row[19]) + owngroup.interest_numerator = int(row[20]) + owngroup.tenancy_specified = str(row[21]) + owner.owner_type = str(row[22]) + owngroup.strip() + owner.strip() + owner.owngroup = owngroup + owners.append(owner) + return owners + @property def json(self): """Return a dict of this object, with keys in JSON format.""" diff --git a/mhr_api/src/mhr_api/models/db2/search_utils.py b/mhr_api/src/mhr_api/models/db2/search_utils.py index fabe3a8f9..c5da8ab6d 100644 --- a/mhr_api/src/mhr_api/models/db2/search_utils.py +++ b/mhr_api/src/mhr_api/models/db2/search_utils.py @@ -40,7 +40,7 @@ MHR_NUM_QUERY = """ SELECT mh.mhregnum, mh.mhstatus, mh.exemptfl, d.regidate, o.ownrtype, o.ownrname, l.towncity, de.sernumb1, de.yearmade, - de.makemodl + de.makemodl, mh.manhomid FROM manuhome mh, document d, owner o, location l, descript de WHERE mh.mhregnum = :query_value AND mh.mhregnum = d.mhregnum @@ -65,7 +65,7 @@ -- FROM owner o2 -- WHERE o2.manhomid = mh.manhomid) as owner_names, l.towncity, de.sernumb1, de.yearmade, - de.makemodl + de.makemodl, mh.manhomid FROM manuhome mh, document d, owner o, location l, descript de, cmpserno c WHERE mh.mhregnum = d.mhregnum AND mh.regdocid = d.documtid @@ -80,14 +80,13 @@ WHERE mh.manhomid = og2.manhomid AND og2.status IN ('3', '4')) AND mh.manhomid = c.manhomid - AND (c.serialno = :query_value OR - de.sernumb1 = :serial_value) + AND HEX(c.serialno) = :query_value ORDER BY d.regidate ASC """ OWNER_NAME_QUERY = """ SELECT DISTINCT mh.mhregnum, mh.mhstatus, mh.exemptfl, d.regidate, o.ownrtype, o.ownrname, l.towncity, de.sernumb1, - de.yearmade, de.makemodl + de.yearmade, de.makemodl, mh.manhomid FROM manuhome mh, document d, owner o, location l, descript de, owngroup og WHERE mh.mhregnum = d.mhregnum AND mh.regdocid = d.documtid @@ -100,13 +99,13 @@ AND o.compname LIKE :query_value || '%' AND o.manhomid = og.manhomid AND o.owngrpid = og.owngrpid - AND og.status IN ('3', '4') + AND og.status IN ('3', '4', '5') ORDER BY o.ownrname ASC, d.regidate ASC """ ORG_NAME_QUERY = """ SELECT DISTINCT mh.mhregnum, mh.mhstatus, mh.exemptfl, d.regidate, o.ownrtype, o.ownrname, l.towncity, de.sernumb1, - de.yearmade, de.makemodl + de.yearmade, de.makemodl, mh.manhomid FROM manuhome mh, document d, owner o, location l, descript de, owngroup og WHERE mh.mhregnum = d.mhregnum AND mh.regdocid = d.documtid @@ -119,10 +118,11 @@ AND o.compname LIKE :query_value || '%' AND o.manhomid = og.manhomid AND o.owngrpid = og.owngrpid - AND og.status IN ('3', '4') + AND og.status IN ('3', '4', '5') ORDER BY o.ownrname ASC, d.regidate ASC """ + def search_by_mhr_number(current_app, db, request_json): """Execute a DB2 search by mhr number query.""" mhr_num = request_json['criteria']['value'] @@ -169,13 +169,11 @@ def search_by_owner_name(current_app, db, request_json): def search_by_serial_number(current_app, db, request_json): """Execute a DB2 search by serial number query.""" serial_num:str = request_json['criteria']['value'] - serial_key = model_utils.get_serial_number_key(serial_num) # serial_num.upper().strip() + serial_key = model_utils.get_serial_number_key_hex(serial_num) # serial_num.upper().strip() current_app.logger.debug(f'DB2 search_by_serial_number search value={serial_num}, key={serial_key}.') try: query = text(SERIAL_NUM_QUERY) - result = db.get_engine(current_app, 'db2').execute(query, - {'query_value': serial_key, - 'serial_value': serial_num.strip()}) + result = db.get_engine(current_app, 'db2').execute(query, {'query_value': serial_key}) return result except Exception as db_exception: # noqa: B902; return nicer error current_app.logger.error('DB2 search_by_serial_number exception: ' + str(db_exception)) diff --git a/mhr_api/src/mhr_api/models/search_request.py b/mhr_api/src/mhr_api/models/search_request.py index fe86fdb4c..9056a1c49 100644 --- a/mhr_api/src/mhr_api/models/search_request.py +++ b/mhr_api/src/mhr_api/models/search_request.py @@ -106,8 +106,9 @@ def save(self): try: db.session.add(self) db.session.commit() + current_app.logger.debug('DB search_request.save completed') except Exception as db_exception: - current_app.logger.error('DB search_client save exception: ' + repr(db_exception)) + current_app.logger.error('DB search_request save exception: ' + str(db_exception)) raise DatabaseException(db_exception) def update_search_selection(self, search_json): @@ -128,7 +129,9 @@ def __build_search_result(cls, row): status = 'EXEMPT' else: status = 'HISTORIC' + # current_app.logger.info('Mapping timestamp') timestamp = row[3] + # current_app.logger.info('Timestamp mapped') value: str = str(row[8]) year = int(value) if value.isnumeric() else 0 result_json = { @@ -141,8 +144,10 @@ def __build_search_result(cls, row): 'year': year, 'make': str(row[9]).strip(), 'model': '' - } + }, + 'mhId': int(row[10]) } + # current_app.logger.info(result_json) owner_type = str(row[4]) owner_name = str(row[5]).strip() if owner_type != 'I': diff --git a/mhr_api/src/mhr_api/models/search_result.py b/mhr_api/src/mhr_api/models/search_result.py index b8cc2e14e..c02d05000 100644 --- a/mhr_api/src/mhr_api/models/search_result.py +++ b/mhr_api/src/mhr_api/models/search_result.py @@ -106,43 +106,62 @@ def update_selection(self, search_select, account_name: str = None, callback_url return self.set_search_selection(search_select) - results = self.search_response - new_results = [] - select_count = 0 - for select in self.search_select: - if 'selected' not in select or select['selected']: - mhr_num = select['mhrNumber'] - for result in results: - if mhr_num == result['mhrNumber']: - select_count += 1 - # Now if combined search add PPR MHR search financing statement info. - if select.get('includeLienInfo', False): - current_app.logger.info(f'Searching PPR for MHR num {mhr_num}.') - ppr_registrations = SearchResult.search_ppr_by_mhr_number(mhr_num) - result['pprRegistrations'] = ppr_registrations - new_results.append(result) - + detail_response['details'] = self.build_details() # current_app.logger.debug('saving updates') # Update summary information and save. + select_count = len(detail_response['details']) self.exact_match_count = select_count detail_response['totalResultsSize'] = select_count - detail_response['details'] = new_results self.search_response = detail_response if account_name: self.account_name = account_name if callback_url: self.callback_url = callback_url else: - results_length = len(json.dumps(new_results)) + results_length = len(json.dumps(detail_response['details'])) current_app.logger.debug(f'Search id= {self.search_id} results size={results_length}.') if results_length > current_app.config.get('MAX_SIZE_SEARCH_RT'): current_app.logger.info(f'Search id={self.search_id} size exceeds RT max, setting up async report.') self.callback_url = current_app.config.get('UI_SEARCH_CALLBACK_URL') self.save() + def build_details(self): + """Generate the search selection details.""" + new_results = [] + for select in self.search_select: + if 'selected' not in select or select['selected']: + mhr_num = select['mhrNumber'] + found = False + if new_results: # Check for duplicates. + for match in new_results: + if match['mhrNumber'] == mhr_num: + found = True + if not found: # No duplicates. + # Load registration details here. + current_app.logger.debug(f'Db2Manuhome.find_by_mhr_number {mhr_num} start') + mh_id = select.get('mhId', None) + record: Db2Manuhome = Db2Manuhome.find_by_id(mh_id) + current_app.logger.debug(f'Db2Manuhome.find_by_mhr_number {mhr_num} end') + result = record.registration_json + if select.get('includeLienInfo', False): + current_app.logger.info(f'Searching PPR for MHR num {mhr_num}.') + ppr_registrations = SearchResult.search_ppr_by_mhr_number(mhr_num) + result['pprRegistrations'] = ppr_registrations + new_results.append(result) + return new_results + def set_search_selection(self, update_select): """Sort the selection for the report TOC.""" - self.search_select = SearchResult.__sort_mhr_number(update_select) + original_results = self.search.search_response + final_selection = [] + for result in original_results: + for match in update_select: + if result['mhrNumber'] == match['mhrNumber']: + if match.get('includeLienInfo', False): + result['includeLienInfo'] = match.get('includeLienInfo') + final_selection.append(result) + break + self.search_select = SearchResult.__sort_mhr_number(final_selection) @classmethod def __sort_mhr_number(cls, update_select): @@ -214,21 +233,8 @@ def create_from_search_query(search_query): return SearchResult.create_from_search_query_no_results(search_query) search_result = SearchResult(search_id=search_query.id, exact_match_count=0, similar_match_count=0) - query_results = search_query.search_response + # query_results = search_query.search_response detail_results = [] - for result in query_results: - mhr_num = result['mhrNumber'] - found = False - if detail_results: # Check for duplicates. - for match in detail_results: - if match['mhrNumber'] == mhr_num: - found = True - if not found: # No duplicates. - record: Db2Manuhome = Db2Manuhome.find_by_mhr_number(mhr_num) - mhr_json = record.registration_json - # current_app.logger.debug(mhr_json) - detail_results.append(mhr_json) - search_result.search_response = detail_results return search_result @@ -239,13 +245,6 @@ def create_from_json(search_json, search_id: int): search.search_id = search_id search.search_select = search_json detail_results = [] - for result in search_json: - mhr_num = result['mhrNumber'] - record: Db2Manuhome = Db2Manuhome.find_by_mhr_number(mhr_num) - mhr_json = record.registration_json - current_app.logger.debug(mhr_json) - detail_results.append(mhr_json) - search.search_response = detail_results return search diff --git a/mhr_api/src/mhr_api/models/utils.py b/mhr_api/src/mhr_api/models/utils.py index 42c6a5cf9..0980b2b29 100644 --- a/mhr_api/src/mhr_api/models/utils.py +++ b/mhr_api/src/mhr_api/models/utils.py @@ -272,8 +272,11 @@ def format_ts(time_stamp): """Build a UTC ISO 8601 date and time string with no microseconds.""" formatted_ts = None if time_stamp: - formatted_ts = time_stamp.replace(tzinfo=timezone.utc).replace(microsecond=0).isoformat() - + try: + formatted_ts = time_stamp.replace(tzinfo=timezone.utc).replace(microsecond=0).isoformat() + except Exception as format_exception: # noqa: B902; return nicer error + current_app.logger.error('format_ts exception: ' + str(format_exception)) + formatted_ts = time_stamp.isoformat() return formatted_ts @@ -491,6 +494,50 @@ def get_compressed_key(name: str) -> str: return key +def get_serial_number_key_hex(serial_num: str) -> str: + """Get the compressed search serial number key for the MH serial number.""" + key: str = '' + + if not serial_num: + return key + key = serial_num.strip().upper() + # 1. Remove all non-alphanumberic characters. + key = re.sub('[^0-9A-Z]+', '', key) + # current_app.logger.debug(f'1: key={key}') + # 2. Add 6 zeroes to the start of the serial number. + key = '000000' + key + # current_app.logger.debug(f'2: key={key}') + # 3. Determine the value of I as last position in the serial number that contains a numeric value. + last_pos: int = 0 + for index, char in enumerate(key): + if char.isdigit(): + last_pos = index + # current_app.logger.debug(f'3: last_pos={last_pos}') + # 4. Replace alphas with the corresponding integers: + # 08600064100100000050000042 where A=0, B=8, C=6…Z=2 + key = key.replace('B', '8') + key = key.replace('C', '6') + key = key.replace('G', '6') + key = key.replace('H', '4') + key = key.replace('I', '1') + key = key.replace('L', '1') + key = key.replace('S', '5') + key = key.replace('Y', '4') + key = key.replace('Z', '2') + key = re.sub('[A-Z]', '0', key) + # current_app.logger.debug(f'4: key={key}') + # 5. Take 6 characters of the string beginning at position I – 5 and ending with the position determined by I + # in step 3. + start_pos = last_pos - 5 + key = key[start_pos:(last_pos + 1)] + # current_app.logger.debug(f'5: key={key}') + # 6. Convert it to bytes and return the last 3. + key_bytes: bytes = int(key).to_bytes(3, 'big') + key_hex = key_bytes.hex().upper() + current_app.logger.debug(f'key={key} last 3 bytes={key_bytes} hex={key_hex}') + return key_hex + + def get_serial_number_key(serial_num: str) -> str: """Get the compressed search serial number key for the MH serial number.""" key: str = '' diff --git a/mhr_api/src/mhr_api/resources/v1/search_results.py b/mhr_api/src/mhr_api/resources/v1/search_results.py index 4c40719f2..ba3483ab1 100644 --- a/mhr_api/src/mhr_api/resources/v1/search_results.py +++ b/mhr_api/src/mhr_api/resources/v1/search_results.py @@ -151,8 +151,10 @@ def post_search_results(search_id: str): # pylint: disable=too-many-branches, t try: # Save the search query selection and details that match the selection. account_name = resource_utils.get_account_name(jwt.get_token_auth_header(), account_id) + current_app.logger.debug('SearchResult.update_selection start') search_detail.update_selection(request_json, account_name, callback_url) query.save() + current_app.logger.debug('SearchResult.update_selection end') except Exception as db_exception: # noqa: B902; handle all db related errors. current_app.logger.error(SAVE_ERROR_MESSAGE.format(account_id, str(db_exception))) if invoice_id is not None: diff --git a/mhr_api/src/mhr_api/resources/v1/searches.py b/mhr_api/src/mhr_api/resources/v1/searches.py index 36f6e6f80..e28e3a1fa 100644 --- a/mhr_api/src/mhr_api/resources/v1/searches.py +++ b/mhr_api/src/mhr_api/resources/v1/searches.py @@ -58,7 +58,9 @@ def post_searches(): query: SearchRequest = SearchRequest.create_from_json(request_json, account_id, g.jwt_oidc_token_info.get('username', None)) # Execute the search query: treat no results as a success. + current_app.logger.debug('query.search() start') query.search() + current_app.logger.debug('query.search() end') # Now save the initial detail results in the search_result table with no # search selection criteria (the absence indicates an incomplete search). diff --git a/mhr_api/src/mhr_api/version.py b/mhr_api/src/mhr_api/version.py index 6f57debc9..40d5b8efb 100644 --- a/mhr_api/src/mhr_api/version.py +++ b/mhr_api/src/mhr_api/version.py @@ -22,4 +22,4 @@ Development release segment: .devN """ -__version__ = '1.0.0' # pylint: disable=invalid-name +__version__ = '1.0.1' # pylint: disable=invalid-name diff --git a/mhr_api/tests/unit/models/test_search_result.py b/mhr_api/tests/unit/models/test_search_result.py index aa04a48f3..7fb8f6975 100644 --- a/mhr_api/tests/unit/models/test_search_result.py +++ b/mhr_api/tests/unit/models/test_search_result.py @@ -213,12 +213,14 @@ def test_search_sort(session, client, jwt, mhr1, mhr2, mhr3, mhr4): # test search_result: SearchResult = SearchResult() + search_request: SearchRequest = SearchRequest(search_response=select_data) + search_result.search = search_request search_result.set_search_selection(select_data) sorted_data = search_result.search_select # check assert len(sorted_data) == 4 - assert select_data[0]['mhrNumber'] == '000199' - assert select_data[1]['mhrNumber'] == '001999' - assert select_data[2]['mhrNumber'] == '002200' - assert select_data[3]['mhrNumber'] == '022911' + assert sorted_data[0]['mhrNumber'] == '000199' + assert sorted_data[1]['mhrNumber'] == '001999' + assert sorted_data[2]['mhrNumber'] == '002200' + assert sorted_data[3]['mhrNumber'] == '022911' diff --git a/mhr_api/tests/unit/models/test_utils.py b/mhr_api/tests/unit/models/test_utils.py index 4b5c7154a..5c3122928 100644 --- a/mhr_api/tests/unit/models/test_utils.py +++ b/mhr_api/tests/unit/models/test_utils.py @@ -61,13 +61,15 @@ ] # testdata pattern is ({serial_num}, {hex_value}) TEST_DATA_SERIAL_KEY = [ - ('WIN14569401627', '0620db'), - ('A4492', '00118c'), - ('3E3947', '04a34b'), - ('6436252B10FK', '03db8a'), - ('I1724B', '002dcc'), - ('2427', '00097b'), - ('123', '00007b') + ('WIN14569401627', '0620DB'), + ('A4492', '00118C'), + ('3E3947', '04A34B'), + ('6436252B10FK', '03DB8A'), + ('I1724B', '002DCC'), + ('2427', '00097B'), + ('123', '00007B'), + ('12345', '003039'), + ('999999', '0F423F') ] @@ -88,10 +90,10 @@ def test_search_key_owner(name, key_value): @pytest.mark.parametrize('serial_num, hex_value', TEST_DATA_SERIAL_KEY) def test_search_key_serial(session, serial_num, hex_value): """Assert that computing a serial number search key works as expected.""" - value = model_utils.get_serial_number_key(serial_num) + value = model_utils.get_serial_number_key_hex(serial_num) # current_app.logger.info(f'Key={value}') - assert len(value) == 3 - assert value.hex() == hex_value + assert len(value) == 6 + assert value == hex_value @pytest.mark.parametrize('street1, street2, city, region, address', TEST_DB2_ADDRESS)