Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Import Flags from Opensrp 1 #335

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion importer/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,18 @@ The coverage report `coverage.html` will be at the working directory
- A separate List resource with references to all the Group and List resources generated is also created
- You can pass in a `list_resource_id` to be used as the identifier for the (reference) List resource, or you can leave it empty and a random uuid will be generated

### 12. Import JSON resources from file
### 12. Import flags from openSRP 1
- Run `python3 main.py --csv_file csv/import/flags.csv --setup flags --encounter_id 123 --practitioner_id 456 --visit_location_id 789 --log_level info`
- See example csv [here](/importer/csv/import/flags.csv)
- `encounter_id` (Required) is the id of the visit encounter that will be linked to all the resources that are imported when the command is run
- `practitioner_id` (Required) is the id of the practitioner that is linked to all the resources created during the import
- `visit_location_id` (Required) is the location linked to the visit encounter, ideally should be the root location
- This function creates a Visit Encounter using the provided encounter id (if one does not already exist)
- It then creates a Flag resource, an Observation resource and an encounter resource for every row on the csv, depending on its type
- It checks to confirm that the location provided in the csv exists, if it does not, it skips that row
- If the entityType is product, it also checks to confirm that the Group id exists in the server and skips the row if it does not

### 13. Import JSON resources from file
- Run `python3 main.py --bulk_import True --json_file tests/json/sample.json --chunk_size 500000 --sync sort --resources_count 100 --log_level info`
- This takes in a file with a JSON array, reads the resources from the array in the file and posts them to the FHIR server
- `bulk_import` (Required) must be set to True
Expand Down
5 changes: 5 additions & 0 deletions importer/csv/import/flags.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
event_id,event_date,id,entitytype,locationid,locationname,productid,productname,product_flag_problem,product_not_there,product_not_good,product_not_good_specify_other,product_not_there_specify_other,product_misuse,product_issue_details,product_service_point_good_order,product_service_point_not_good_order_reason,product_consult_beneficiaries_flag,product_consult_beneficiaries_issues_raised,product_required_action,product_required_action_yes
100,2022-11-06 21:00:00,5f513cfd-90fa-40a9-8d84-63eed687063d,service_point,661604af-d9f9-4be2-a500-4d127c4bfbed,Nakuru,,,,,,,,,,"[""no""]","[""Lack of infrastructure""]",,,,
101,2023-05-17 21:00:00,b0038380-7800-4771-b8e3-b97c413f8177,service_point,66cb98a3-6f0b-439d-95c1-7f562996f070,Isiolo,,,,,,,,,,,,"[""yes""]","[""Infrequent power""]",,
102,2024-10-11 21:00:00,f9a1bd74-23e8-433b-84b3-bbec68f70036,service_point,f0b8cfc6-eea7-4a54-8eec-057a14a4cc95,Kisumu,,,,,,,,,,,,,,"[""yes""]","[""Cleaning""]"
103,2024-10-13 21:00:00,128053da-639f-40ab-becc-374713fe60e2,product,1523b35a-1a23-4fc9-af8f-6fb5905236de,Malindi,46d7c50b-bfee-4715-a61b-141eb80a4a9d,Wheelbarrow,"[""Product is not there""]","[""never_received""]",,,,,,,,,,,
191 changes: 190 additions & 1 deletion importer/importer/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -679,7 +679,7 @@ def get_valid_resource_type(resource_type):

# This function gets the current resource version from the API
def get_resource(resource_id, resource_type):
if resource_type not in ["List", "Group"]:
if resource_type not in ["List", "Group", "Encounter", "Location"]:
resource_type = get_valid_resource_type(resource_type)
resource_url = "/".join([fhir_base_url, resource_type, resource_id])
response = handle_request("GET", "", resource_url)
Expand Down Expand Up @@ -1150,3 +1150,192 @@ def build_report(csv_file, response, error_details, fail_count, fail_all):
logging.info(string_report)
logging.info("============================================================")
logging.info("============================================================")


def build_single_resource(
resource_type, resource_id, practitioner_id, period_start, location_id, form_encounter,
visit_encounter, subject, value_string="", note="",
):
template_map = {
"visit": "visit_encounter_payload.json",
"flag": "flag_payload.json",
"encounter": "flag_encounter_payload.json",
"observation": "flag_observation_payload.json",
}
json_template = next(
(template for key, template in template_map.items() if key in resource_type),
None,
)

boolean_code = boolean_value = ""
if "product" in resource_type:
code = "PRODCHECK"
display = text = "Product Check"
c_code = "issue_details"
c_display = c_text = value_string
elif "service_point_check" in resource_type:
code = "SPCHECK"
display = text = "Service Point Check"
c_code = "34657579"
c_display = c_text = "Service Point Good Order Check"
boolean_code = "373067005"
boolean_value = "No (qualifier value)"
elif "consult_beneficiaries" in resource_type:
code = "CNBEN"
display = text = "Consult Beneficiaries Visit"
c_code = "77223346"
c_display = c_text = "Consult Beneficiaries"
boolean_code = "373066001"
boolean_value = "Yes (qualifier value)"
elif "warehouse_check" in resource_type:
code = "WHCHECK"
display = text = "Warehouse check Visit"
c_code = "68561322"
c_display = c_text = "Required action"
boolean_code = "373066001"
boolean_value = "Yes (qualifier value)"
else:
code = display = text = c_code = c_display = c_text = ""

with open(json_path + json_template) as json_file:
resource_payload = json_file.read()

visit_encounter_vars = {
"$id": resource_id,
"$version": "1",
"$category_code": code,
"$category_display": display,
"$category_text": text,
"$code_code": c_code,
"$code_display": c_display,
"$code_text": c_text,
"$practitioner_id": practitioner_id,
"$start": period_start.replace(" ", "T"),
"$end": period_start.replace(" ", "T"),
"$subject": subject,
"$location": location_id,
"$form_encounter": form_encounter,
"$visit_encounter": visit_encounter,
"$value_string": value_string,
"$boolean_code": boolean_code,
"$boolean_value": boolean_value,
"$note": note,
}
for var, value in visit_encounter_vars.items():
resource_payload = resource_payload.replace(var, value)

obj = json.loads(resource_payload)
if "product_observation" in resource_type:
del obj["resource"]["valueCodeableConcept"]
del obj["resource"]["note"]
if (
"service_point_check_encounter" in resource_type
or "consult_beneficiaries_encounter" in resource_type
):
del obj["resource"]["subject"]
if (
"service_point_check_observation" in resource_type
or "consult_beneficiaries_observation" in resource_type
):
del obj["resource"]["focus"]
del obj["resource"]["valueString"]
resource_payload = json.dumps(obj, indent=4)

return resource_payload


def build_resources(
resource_type, encounter_id, flag_id, observation_id, practitioner_id, period, location,
visit_encounter, subject, value_string="", note="",
):
encounter = build_single_resource(
resource_type + "_encounter", encounter_id, practitioner_id, period, location, "",
visit_encounter, subject,
)
flag = build_single_resource(
resource_type + "_flag", flag_id, practitioner_id, period, location, encounter_id,
"", subject,
)
observation = build_single_resource(
resource_type + "_observation", observation_id, practitioner_id, period, location, encounter_id,
"", subject, value_string, note,
)

resources = encounter + "," + flag + "," + observation + ","
return resources


def check_location(location_id, locations_list):
if location_id in locations_list:
return locations_list[location_id]

check = get_resource(location_id, "Location")
if check != "0":
locations_list[location_id] = True
return True
else:
locations_list[location_id] = False
logging.info("-- Skipping location, it does NOT EXIST " + location_id)
return False


def build_flag_payload(resources, practitioner_id, visit_encounter):
initial_string = """{"resourceType": "Bundle","type": "transaction","entry": [ """
final_string = ""
locations = {}
for resource in resources:
flag_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, "flag" + resource[2]))
observation_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, "observation" + resource[2]))

sub_list = []
valid_location = check_location(resource[4], locations)
if valid_location:
if resource[3] == "service_point":
if "no" in resource[15]:
note = (
resource[16].replace('"', "").replace("[", "").replace("]", "")
)
sub_list = build_resources(
"service_point_check", resource[2], flag_id, observation_id, practitioner_id,
resource[1], resource[4], visit_encounter, "Location/" + resource[4], "", note,
)
if "yes" in resource[17]:
note = (
resource[18].replace('"', "").replace("[", "").replace("]", "")
)
sub_list = build_resources(
"consult_beneficiaries", resource[2], flag_id, observation_id, practitioner_id,
resource[1], resource[4], visit_encounter, "Location/" + resource[4], "", note,
)
if "yes" in resource[19]:
note = (
resource[20].replace('"', "").replace("[", "").replace("]", "")
)
sub_list = build_resources(
"warehouse", resource[2], flag_id, observation_id, practitioner_id, resource[1],
resource[4], visit_encounter, "Location/" + resource[4], "", note,
)

elif resource[3] == "product":
if resource[8]:
product_info = [
resource[8], resource[9], resource[10], resource[11], resource[12], resource[13], resource[14],
]
value_string = " | ".join(filter(None, product_info))
value_string = value_string.replace('"', "")
if resource[6]:
sub_list = build_resources(
"product", resource[2], flag_id, observation_id, practitioner_id,
resource[1], resource[4], visit_encounter, "Group/" + resource[6], value_string,
)
else:
logging.info("-- Missing Group, skipping resource: " + str(resource))
else:
logging.info("-- This entityType is not supported")

if len(sub_list) < 1:
sub_list = ""

final_string = final_string + sub_list
final_string = initial_string + final_string[:-1] + " ] } "
return final_string
2 changes: 1 addition & 1 deletion importer/importer/services/fhir_keycloak_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ def request(self, **kwargs):
# TODO - spread headers into kwargs.
headers = {"content-type": "application/json", "accept": "application/json"}
response = self.api_service.oauth.request(headers=headers, **kwargs)
if response.status_code == 401 or '<html class="login-pf">' in response.text:
if response.status_code == 401 or 'login' in response.text:
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Initial line only worked for a html response, this should work even for a json response

self.api_service.refresh_token()
return self.api_service.oauth.request(headers=headers, **kwargs)
return response
Expand Down
82 changes: 82 additions & 0 deletions importer/json_payloads/flag_encounter_payload.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
{
"request": {
"method": "PUT",
"url": "Encounter/$id",
"ifMatch": "$version"
},
"resource": {
"resourceType": "Encounter",
"id": "$id",
"identifier": [
{
"use": "usual",
"value": "$id"
}
],
"status": "finished",
"class": {
"system": "http://terminology.hl7.org/CodeSystem/v3-ActCode",
"code": "OBSENC",
"display": "Observation Encounter"
},
"type": [
{
"coding": [
{
"system": "http://smartregister.org/",
"code": "$category_code",
"display": "$category_display"
}
],
"text": "$category_text"
}
],
"priority": {
"coding": [
{
"system": "http://terminology.hl7.org/ValueSet/v3-ActPriority",
"code": "EL",
"display": "elective"
}
],
"text": "elective"
},
"subject": {
"reference": "$subject"
},
"participant": [
{
"individual": {
"reference": "Practitioner/$practitioner_id"
}
}
],
"period": {
"start": "$start",
"end": "$end"
},
"reasonCode": [
{
"coding": [
{
"system": "http://smartregister.org/",
"code": "$category_code",
"display": "$category_display"
}
],
"text": "$category_text"
}
],
"location": [
{
"location": {
"reference": "Location/$location"
},
"status": "active"
}
],
"partOf": {
"reference": "Encounter/$visit_encounter"
}
}
}
77 changes: 77 additions & 0 deletions importer/json_payloads/flag_observation_payload.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
{
"request": {
"method": "PUT",
"url": "Observation/$id",
"ifMatch": "$version"
},
"resource": {
"resourceType": "Observation",
"id": "$id",
"identifier": [
{
"use": "usual",
"value": "$id"
}
],
"status": "final",
"category": [
{
"coding": [
{
"system": "http://smartregister.org/",
"code": "$category_code",
"display": "$category_display"
}
],
"text": "$category_text"
}
],
"code": {
"coding": [
{
"system": "http://snomed.info/sct",
"code": "$code_code",
"display": "$code_display"
}
],
"text": "$code_text"
},
"subject": {
"reference": "$subject"
},
"focus": [
{
"reference": "Location/$location"
}
],
"encounter": {
"reference": "Encounter/$form_encounter"
},
"effectivePeriod": {
"start": "$start",
"end": "$end"
},
"performer": [
{
"reference": "Practitioner/$practitioner_id"
}
],
"valueCodeableConcept": {
"coding": [
{
"system": "http://snomed.info/sct",
"code": "$boolean_code",
"display": "$boolean_value"
}
],
"text": "$boolean_value"
},
"note": [
{
"time": "$start",
"text": "$note"
}
],
"valueString": "$value_string"
}
}
Loading